From 2fc5c50c9cc00ab18425b6648e544f544fc1af8d Mon Sep 17 00:00:00 2001 From: prekshivyas Date: Wed, 19 Aug 2026 14:31:33 -0700 Subject: [PATCH 01/10] fix(uninstall): preserve failed llama.cpp cleanup state Signed-off-by: prekshivyas --- .../run-plan-local-model-profile.test.ts | 27 ++++--- src/lib/actions/uninstall/run-plan.ts | 73 ++++++++++++++----- .../local-model-profile/cleanup.test.ts | 4 + .../inference/local-model-profile/cleanup.ts | 12 +++ 4 files changed, 90 insertions(+), 26 deletions(-) diff --git a/src/lib/actions/uninstall/run-plan-local-model-profile.test.ts b/src/lib/actions/uninstall/run-plan-local-model-profile.test.ts index f7b65d819e9..501cbeb7317 100644 --- a/src/lib/actions/uninstall/run-plan-local-model-profile.test.ts +++ b/src/lib/actions/uninstall/run-plan-local-model-profile.test.ts @@ -620,13 +620,17 @@ describe("uninstall local model profile cleanup", () => { } }); - it("preserves selected gateway authority when scoped cleanup leaves ownership state", () => { + it("continues unrelated uninstall after managed llama.cpp cleanup fails (#9575)", () => { const tmpHome = fs.realpathSync( fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-llama-fail-")), ); const stateDir = publishManagedLlamaOwner(tmpHome, 8080, "selected-sandbox"); writeScopedGatewayState(tmpHome); + const unrelatedState = path.join(tmpHome, ".nemoclaw", "unrelated-state.json"); + fs.writeFileSync(unrelatedState, "{}\n", { mode: 0o600 }); const errors: string[] = []; + const logs: string[] = []; + const runLocalModelRuntimeCleanup = vi.fn(() => ok()); try { const result = runUninstallPlan( { assumeYes: true, deleteModels: false, keepOpenShell: true }, @@ -637,19 +641,24 @@ describe("uninstall local model profile cleanup", () => { existsSync: fs.existsSync, isPortFree: () => true, isTty: false, - log: () => {}, - run: vi.fn((command: string, args: string[]) => - command === "openshell" && args[0] === "gateway" && args[1] === "list" - ? ok(JSON.stringify([{ name: "nemoclaw" }, { name: "nemoclaw-9000" }])) - : ok(), - ), - runManagedLlamaCppRuntimeCleanup: vi.fn(() => ok()), + log: (message) => logs.push(message), + run: vi.fn(okWithKnownGatewayList), + runLocalModelRuntimeCleanup, + runManagedLlamaCppRuntimeCleanup: vi.fn(() => ({ + status: 1, + stdout: "", + stderr: "qualified endpoint changed", + })), }), ); expect(result.exitCode).toBe(1); expect(fs.existsSync(stateDir)).toBe(true); - expect(errors.join("\n")).toContain("returned without retiring its ownership state"); + expect(fs.existsSync(path.join(stateDir, "owner.json"))).toBe(true); + expect(fs.existsSync(unrelatedState)).toBe(false); + expect(runLocalModelRuntimeCleanup).not.toHaveBeenCalled(); + expect(logs.some((message) => message.endsWith("State and binaries"))).toBe(true); + expect(errors.join("\n")).toContain("continue unrelated uninstall steps"); } finally { fs.rmSync(tmpHome, { recursive: true, force: true }); } diff --git a/src/lib/actions/uninstall/run-plan.ts b/src/lib/actions/uninstall/run-plan.ts index f74504d18e2..cc35364c05c 100644 --- a/src/lib/actions/uninstall/run-plan.ts +++ b/src/lib/actions/uninstall/run-plan.ts @@ -600,6 +600,7 @@ function buildRuntime(deps: UninstallRunDeps): UninstallRuntime { deps.runManagedLlamaCppRuntimeCleanup ?? ((sandboxName, gatewayPort) => { const result = cleanupManagedLlamaCppRuntimeForSandbox(sandboxName, { + env, gatewayPort, homeDir: env.HOME || os.homedir(), }); @@ -2027,49 +2028,59 @@ function managedLlamaCppCleanupTargets( } } +interface ManagedLlamaCppCleanupOutcome { + readonly failedStateDirs: readonly string[]; + readonly ok: boolean; +} + function removeManagedLlamaCppRuntimes( runtime: UninstallRuntime, scopedToSelectedGateway: boolean, -): boolean { +): ManagedLlamaCppCleanupOutcome { const targets = managedLlamaCppCleanupTargets(runtime, scopedToSelectedGateway); - if (targets === null) return false; + if (targets === null) return { failedStateDirs: [], ok: false }; for (const target of targets) { const result = runtime.runManagedLlamaCppRuntimeCleanup(target.sandboxName, target.gatewayPort); for (const removed of splitNonEmptyLines(result.stdout)) runtime.log(`Removed ${removed}`); if (result.status !== 0) { runtime.error( - `Managed llama.cpp cleanup for sandbox '${target.sandboxName}' on gateway port ${String(target.gatewayPort)} did not complete: ${result.stderr.trim() || "unknown cleanup error"}. NemoClaw did not start the remaining uninstall steps.`, + `Managed llama.cpp cleanup for sandbox '${target.sandboxName}' on gateway port ${String(target.gatewayPort)} did not complete: ${result.stderr.trim() || "unknown cleanup error"}. NemoClaw will preserve its ownership state and continue unrelated uninstall steps.`, ); - return false; + return { failedStateDirs: [target.stateDir], ok: true }; } if (fs.lstatSync(target.stateDir, { throwIfNoEntry: false }) !== undefined) { runtime.error( - `Managed llama.cpp cleanup for sandbox '${target.sandboxName}' on gateway port ${String(target.gatewayPort)} returned without retiring its ownership state. NemoClaw did not start the remaining uninstall steps.`, + `Managed llama.cpp cleanup for sandbox '${target.sandboxName}' on gateway port ${String(target.gatewayPort)} returned without retiring its ownership state. NemoClaw will preserve its ownership state and continue unrelated uninstall steps.`, ); - return false; + return { failedStateDirs: [target.stateDir], ok: true }; } } - return true; + return { failedStateDirs: [], ok: true }; } function removeManagedModelRuntimes( paths: UninstallPaths, runtime: UninstallRuntime, scopedToSelectedGateway: boolean, -): boolean { - if (!removeManagedLlamaCppRuntimes(runtime, scopedToSelectedGateway)) return false; - if (scopedToSelectedGateway) return true; +): ManagedLlamaCppCleanupOutcome { + const llama = removeManagedLlamaCppRuntimes(runtime, scopedToSelectedGateway); + if (!llama.ok || llama.failedStateDirs.length > 0) return llama; + if (scopedToSelectedGateway) return llama; const sharedRoot = path.dirname(paths.managedSwapMarkerPath); const hasDistributedReceipt = [ MANAGED_CLUSTER_VLLM_RUNTIME_RECEIPT_FILE, DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE, ].some((name) => runtime.existsSync(path.join(sharedRoot, name))); - if (!removeManagedDistributedVllmRuntime(paths, runtime, !hasDistributedReceipt)) return false; - if (!removeHostLocalModelRuntimes(paths, runtime)) return false; + if (!removeManagedDistributedVllmRuntime(paths, runtime, !hasDistributedReceipt)) { + return { failedStateDirs: [], ok: false }; + } + if (!removeHostLocalModelRuntimes(paths, runtime)) { + return { failedStateDirs: [], ok: false }; + } if (!hasDistributedReceipt) { removePath(path.join(sharedRoot, MANAGED_VLLM_API_KEY_FILE), runtime); } - if (!runtime.commandExists("docker")) return true; + if (!runtime.commandExists("docker")) return llama; const inventory = runtime.runDocker(["ps", "-a", "--format", "{{.Names}}"], { env: runtime.env, timeout: 10_000, @@ -2078,16 +2089,29 @@ function removeManagedModelRuntimes( runtime.error( "Docker could not inventory reserved managed inference container names. NemoClaw refused the remaining uninstall steps so it cannot report incomplete cleanup as success.", ); - return false; + return { failedStateDirs: [], ok: false }; } const residual = splitNonEmptyLines(inventory.stdout).find((name) => MANAGED_INFERENCE_CONTAINER_NAME_PATTERN.test(name), ); - if (!residual) return true; + if (!residual) return llama; runtime.error( `Managed inference container '${residual}' remains after ownership-aware cleanup. NemoClaw refused the remaining uninstall steps; restore its ownership state or remove it after manual review, then retry.`, ); - return false; + return { failedStateDirs: [], ok: false }; +} + +function recordManagedModelCleanup( + paths: UninstallPaths, + runtime: UninstallRuntime, + scopedToSelectedGateway: boolean, + failedStateDirs: string[], + onPartialFailure: () => void, +): boolean { + const result = removeManagedModelRuntimes(paths, runtime, scopedToSelectedGateway); + failedStateDirs.push(...result.failedStateDirs); + if (result.failedStateDirs.length > 0) onPartialFailure(); + return result.ok; } function removeDockerContainers(runtime: UninstallRuntime, gatewayName?: string): void { @@ -2752,6 +2776,7 @@ function executePlan( return { ok: false }; } let ok = true; + const failedManagedLlamaStateDirs: string[] = []; const branding = runtimeBranding(runtime); const preserveSharedOpenShell = options.keepOpenShell || externallySupervised || portableRuntimeCleanup; @@ -2793,7 +2818,15 @@ function executePlan( if (step.name === "Stopping services") { if ( !portableRuntimeCleanup && - !removeManagedModelRuntimes(paths, runtime, scopedToSelectedGateway) + !recordManagedModelCleanup( + paths, + runtime, + scopedToSelectedGateway, + failedManagedLlamaStateDirs, + () => { + ok = false; + }, + ) ) { return { ok: false }; } @@ -2957,6 +2990,12 @@ function executePlan( [ ...preserveUnderStateDir, ...portableStateEntries, + ...failedManagedLlamaStateDirs.flatMap((stateDir) => { + const relative = path.relative(paths.nemoclawStateDir, stateDir); + return relative && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative) + ? [relative.split(path.sep)[0]!] + : []; + }), ...(selectedIsDefault ? [GATEWAYS_SUBDIR, path.basename(paths.managedSwapMarkerPath)] : []), diff --git a/src/lib/inference/local-model-profile/cleanup.test.ts b/src/lib/inference/local-model-profile/cleanup.test.ts index 54cccdd109d..c3a44f18622 100644 --- a/src/lib/inference/local-model-profile/cleanup.test.ts +++ b/src/lib/inference/local-model-profile/cleanup.test.ts @@ -582,17 +582,21 @@ describe("host-local model cleanup", () => { const homeDir = temporaryHome(); const original = engineHarness({ authorityId: "docker:original" }); const changed = engineHarness({ authorityId: "docker:changed" }); + const privateBridge = privateBridgeFixture(); createManagedState(homeDir, original.engine); const result = cleanupLocalModelRuntimes({ homeDir, engine: changed.engine, + privateBridge, }); expect(result).toMatchObject({ ok: false, reason: expect.stringContaining("endpoint"), }); + expect(privateBridge.stopTransaction).toHaveBeenCalledExactlyOnceWith(TRANSACTION_ID); + expect(privateBridge.assertStopped).toHaveBeenCalledExactlyOnceWith(TRANSACTION_ID); expect(changed.capture).not.toHaveBeenCalled(); expect(fs.existsSync(managedLlamaCppStatePaths(homeDir).stateDir)).toBe(true); }); diff --git a/src/lib/inference/local-model-profile/cleanup.ts b/src/lib/inference/local-model-profile/cleanup.ts index 4b5df55cec2..bcfc054e124 100644 --- a/src/lib/inference/local-model-profile/cleanup.ts +++ b/src/lib/inference/local-model-profile/cleanup.ts @@ -79,6 +79,7 @@ export interface LocalModelRuntimeCleanupOptions { sandboxName?: string; env?: NodeJS.ProcessEnv; engine?: ContainerEngine; + privateBridge?: DockerLlamaCppPrivateBridgeController; deps?: Partial; } @@ -469,6 +470,7 @@ function cleanupLlamaCpp( sandboxName?: string; env?: NodeJS.ProcessEnv; engine?: ContainerEngine; + privateBridge?: DockerLlamaCppPrivateBridgeController; } = {}, ): boolean { const paths = managedLlamaCppStatePaths(homeDir, options.gatewayPort); @@ -547,6 +549,13 @@ function cleanupLlamaCpp( throw new Error("managed llama.cpp finalized receipt is missing"); } + const privateBridge = + options.privateBridge ?? + (process.platform === "linux" ? createDockerLlamaCppPrivateBridgeController() : undefined); + if (privateBridge) { + privateBridge.stopTransaction(journal.transactionId); + privateBridge.assertStopped(journal.transactionId); + } const engine = options.engine ?? createManagedLlamaCppEngine(options.env ?? process.env); requireQualifiedEngine(receipt?.engineAuthority ?? journal.engineAuthority, engine); requireEngineSuccess( @@ -627,6 +636,7 @@ export interface ManagedLlamaCppSandboxCleanupOptions { readonly gatewayPort?: number; readonly env?: NodeJS.ProcessEnv; readonly engine?: ContainerEngine; + readonly privateBridge?: DockerLlamaCppPrivateBridgeController; readonly deps?: Partial; } @@ -906,6 +916,7 @@ export function cleanupManagedLlamaCppRuntimeForSandbox( sandboxName, env: options.env, engine: options.engine, + privateBridge: options.privateBridge, }); preserveSharedHuggingFaceCache(homeDir, preserved); return { ok: true, removed, preserved }; @@ -955,6 +966,7 @@ export function cleanupLocalModelRuntimes( sandboxName: options.sandboxName, env: options.env, engine: options.engine, + privateBridge: options.privateBridge, }); } preserveSharedHuggingFaceCache(homeDir, preserved); From 6432af605b2e488da5b50f04621f892d6a6a245e Mon Sep 17 00:00:00 2001 From: prekshivyas Date: Wed, 19 Aug 2026 17:11:48 -0700 Subject: [PATCH 02/10] test(uninstall): preserve sibling model authority Signed-off-by: prekshivyas --- .../run-plan-local-model-profile.test.ts | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/lib/actions/uninstall/run-plan-local-model-profile.test.ts b/src/lib/actions/uninstall/run-plan-local-model-profile.test.ts index 501cbeb7317..690bc509d3c 100644 --- a/src/lib/actions/uninstall/run-plan-local-model-profile.test.ts +++ b/src/lib/actions/uninstall/run-plan-local-model-profile.test.ts @@ -625,12 +625,18 @@ describe("uninstall local model profile cleanup", () => { fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-llama-fail-")), ); const stateDir = publishManagedLlamaOwner(tmpHome, 8080, "selected-sandbox"); + const siblingStateDir = publishManagedLlamaOwner(tmpHome, 9000, "sibling-sandbox"); writeScopedGatewayState(tmpHome); const unrelatedState = path.join(tmpHome, ".nemoclaw", "unrelated-state.json"); fs.writeFileSync(unrelatedState, "{}\n", { mode: 0o600 }); const errors: string[] = []; const logs: string[] = []; const runLocalModelRuntimeCleanup = vi.fn(() => ok()); + const runManagedLlamaCppRuntimeCleanup = vi.fn(() => ({ + status: 1, + stdout: "", + stderr: "qualified endpoint changed", + })); try { const result = runUninstallPlan( { assumeYes: true, deleteModels: false, keepOpenShell: true }, @@ -644,17 +650,18 @@ describe("uninstall local model profile cleanup", () => { log: (message) => logs.push(message), run: vi.fn(okWithKnownGatewayList), runLocalModelRuntimeCleanup, - runManagedLlamaCppRuntimeCleanup: vi.fn(() => ({ - status: 1, - stdout: "", - stderr: "qualified endpoint changed", - })), + runManagedLlamaCppRuntimeCleanup, }), ); expect(result.exitCode).toBe(1); + expect(runManagedLlamaCppRuntimeCleanup).toHaveBeenCalledExactlyOnceWith( + "selected-sandbox", + 8080, + ); expect(fs.existsSync(stateDir)).toBe(true); expect(fs.existsSync(path.join(stateDir, "owner.json"))).toBe(true); + expect(fs.existsSync(path.join(siblingStateDir, "owner.json"))).toBe(true); expect(fs.existsSync(unrelatedState)).toBe(false); expect(runLocalModelRuntimeCleanup).not.toHaveBeenCalled(); expect(logs.some((message) => message.endsWith("State and binaries"))).toBe(true); From b0c52914c09025f4a7612b2fe05f57a66a434ed4 Mon Sep 17 00:00:00 2001 From: prekshivyas Date: Wed, 19 Aug 2026 17:38:08 -0700 Subject: [PATCH 03/10] test(inference): qualify vllm serving-port fixtures Signed-off-by: prekshivyas --- src/lib/inference/vllm-serving-port.test.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/lib/inference/vllm-serving-port.test.ts b/src/lib/inference/vllm-serving-port.test.ts index 298c6adaaaf..ba002e10160 100644 --- a/src/lib/inference/vllm-serving-port.test.ts +++ b/src/lib/inference/vllm-serving-port.test.ts @@ -63,15 +63,25 @@ vi.mock("./serving/vllm-managed-support", async (importOriginal) => { }; }); -import { detectVllmProfile, installVllm } from "./vllm"; +import { + detectVllmProfile, + installVllm as installVllmProduction, + type InstallVllmOptions, + type VllmProfile, +} from "./vllm"; import { applyVllmInstallProbeDefaults, createVllmInstallSpies, mockSuccessfulVllmInstall, resetVllmInstallEnv, type VllmInstallSpies, + withVllmInstallTestReadiness, } from "./vllm-install.test-support"; +function installVllm(profile: VllmProfile, options: InstallVllmOptions) { + return installVllmProduction(profile, withVllmInstallTestReadiness(profile, options)); +} + describe("managed vLLM serving-port guard (#8685)", () => { const originalEnv = { ...process.env }; let errSpy: VllmInstallSpies["errSpy"]; From 870496b341c17ff70b401a9dd6e2615c0c593067 Mon Sep 17 00:00:00 2001 From: prekshivyas Date: Wed, 19 Aug 2026 17:50:44 -0700 Subject: [PATCH 04/10] test(uninstall): assert preserved llama ownership --- .../actions/uninstall/run-plan-local-model-profile.test.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/lib/actions/uninstall/run-plan-local-model-profile.test.ts b/src/lib/actions/uninstall/run-plan-local-model-profile.test.ts index 690bc509d3c..7044c243bfa 100644 --- a/src/lib/actions/uninstall/run-plan-local-model-profile.test.ts +++ b/src/lib/actions/uninstall/run-plan-local-model-profile.test.ts @@ -12,6 +12,7 @@ import { } from "../../../../test/support/uninstall-managed-gateway-test-support"; import { + loadManagedLlamaCppOwner, managedLlamaCppStatePaths, reserveManagedLlamaCppOwner, } from "../../inference/llama-cpp/managed-state"; @@ -626,6 +627,10 @@ describe("uninstall local model profile cleanup", () => { ); const stateDir = publishManagedLlamaOwner(tmpHome, 8080, "selected-sandbox"); const siblingStateDir = publishManagedLlamaOwner(tmpHome, 9000, "sibling-sandbox"); + const selectedPaths = managedLlamaCppStatePaths(tmpHome, 8080); + const siblingPaths = managedLlamaCppStatePaths(tmpHome, 9000); + const selectedOwnerBefore = loadManagedLlamaCppOwner(selectedPaths); + const siblingOwnerBefore = loadManagedLlamaCppOwner(siblingPaths); writeScopedGatewayState(tmpHome); const unrelatedState = path.join(tmpHome, ".nemoclaw", "unrelated-state.json"); fs.writeFileSync(unrelatedState, "{}\n", { mode: 0o600 }); @@ -662,6 +667,8 @@ describe("uninstall local model profile cleanup", () => { expect(fs.existsSync(stateDir)).toBe(true); expect(fs.existsSync(path.join(stateDir, "owner.json"))).toBe(true); expect(fs.existsSync(path.join(siblingStateDir, "owner.json"))).toBe(true); + expect(loadManagedLlamaCppOwner(selectedPaths)).toEqual(selectedOwnerBefore); + expect(loadManagedLlamaCppOwner(siblingPaths)).toEqual(siblingOwnerBefore); expect(fs.existsSync(unrelatedState)).toBe(false); expect(runLocalModelRuntimeCleanup).not.toHaveBeenCalled(); expect(logs.some((message) => message.endsWith("State and binaries"))).toBe(true); From 78e300bc2c7950f97e2ed94bfa4abd9a262a9d96 Mon Sep 17 00:00:00 2001 From: prekshivyas Date: Wed, 19 Aug 2026 18:02:18 -0700 Subject: [PATCH 05/10] test(inference): qualify Spark onboarder fixtures --- .../local-model-profile/onboarder.test.ts | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/lib/onboard/local-model-profile/onboarder.test.ts b/src/lib/onboard/local-model-profile/onboarder.test.ts index f87ee3eb0b0..8356cbd01e2 100644 --- a/src/lib/onboard/local-model-profile/onboarder.test.ts +++ b/src/lib/onboard/local-model-profile/onboarder.test.ts @@ -56,7 +56,11 @@ describe("dedicated local model profile onboarder", () => { prompt: vi.fn(async () => ""), error: vi.fn(), }); - const vllmProfile = { name: "DGX Spark", platform: "spark" } as VllmProfile; + const vllmProfile = { + name: "DGX Spark", + platform: "spark", + architecture: "arm64", + } as VllmProfile; await expect( onboard( @@ -94,7 +98,11 @@ describe("dedicated local model profile onboarder", () => { { hasVllmImage: false, sparkHost: true, - vllmProfile: { name: "DGX Spark", platform: "spark" } as VllmProfile, + vllmProfile: { + name: "DGX Spark", + platform: "spark", + architecture: "arm64", + } as VllmProfile, vllmRunning: false, }, state(), @@ -123,7 +131,11 @@ describe("dedicated local model profile onboarder", () => { { hasVllmImage: false, sparkHost: true, - vllmProfile: { name: "DGX Spark", platform: "spark" } as VllmProfile, + vllmProfile: { + name: "DGX Spark", + platform: "spark", + architecture: "arm64", + } as VllmProfile, vllmRunning: false, }, state(), From 3cc81e446c039299bb0b6bd5347859ba5f978ffd Mon Sep 17 00:00:00 2001 From: prekshivyas Date: Wed, 19 Aug 2026 18:26:28 -0700 Subject: [PATCH 06/10] test(package): allow full CLI docs crawl --- test/package-contract/cli/public-cli-contracts.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/package-contract/cli/public-cli-contracts.test.ts b/test/package-contract/cli/public-cli-contracts.test.ts index 0c637f0b823..3b0814f30dc 100644 --- a/test/package-contract/cli/public-cli-contracts.test.ts +++ b/test/package-contract/cli/public-cli-contracts.test.ts @@ -11,6 +11,7 @@ import { describe, expect, it } from "vitest"; const REPO_ROOT = path.join(import.meta.dirname, "../../.."); const CLI_ENTRYPOINT = path.join(REPO_ROOT, "bin", "nemoclaw.js"); const CHECK_DOCS = path.join(REPO_ROOT, "test", "e2e", "e2e-cloud-experimental", "check-docs.sh"); +const CLI_DOCS_TIMEOUT_MS = 240_000; describe("public compiled CLI contracts", () => { it("prints the public NemoClaw version prefix (#7616)", () => { @@ -27,7 +28,7 @@ describe("public compiled CLI contracts", () => { }); it("keeps compiled CLI commands aligned with their documentation headings (#7616)", { - timeout: 150_000, + timeout: CLI_DOCS_TIMEOUT_MS + 30_000, }, () => { // `npm run test:package` builds the CLI before this project, so the shim // exercises the same compiled entrypoint shipped by the package. @@ -55,7 +56,7 @@ exec ${JSON.stringify(process.execPath)} ${JSON.stringify(CLI_ENTRYPOINT)} "$@" PATH: `${binDir}${path.delimiter}${process.env.PATH ?? ""}`, }, killSignal: "SIGKILL", - timeout: 120_000, + timeout: CLI_DOCS_TIMEOUT_MS, }); expect(result.error).toBeUndefined(); From e0501c8a40c2bb44009c076885b96359a90c80b3 Mon Sep 17 00:00:00 2001 From: prekshivyas Date: Wed, 19 Aug 2026 18:49:53 -0700 Subject: [PATCH 07/10] fix(inference): lease llama bridge cleanup --- .../local-model-profile/cleanup.test.ts | 4 +++ .../inference/local-model-profile/cleanup.ts | 29 ++++++++++--------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/src/lib/inference/local-model-profile/cleanup.test.ts b/src/lib/inference/local-model-profile/cleanup.test.ts index c3a44f18622..b2caf0bafe9 100644 --- a/src/lib/inference/local-model-profile/cleanup.test.ts +++ b/src/lib/inference/local-model-profile/cleanup.test.ts @@ -625,6 +625,7 @@ describe("host-local model cleanup", () => { it("does not race cleanup against a live lifecycle execution lease", () => { const homeDir = temporaryHome(); const harness = engineHarness(); + const privateBridge = privateBridgeFixture(); createManagedState(homeDir, harness.engine, { phase: "started" }); const store = createHostLocalCreateJournalStore(managedLlamaCppStatePaths(homeDir).stateDir); const lease = store.acquireExecution(TRANSACTION_ID); @@ -632,6 +633,7 @@ describe("host-local model cleanup", () => { const result = cleanupLocalModelRuntimes({ homeDir, engine: harness.engine, + privateBridge, }); expect(result).toMatchObject({ @@ -642,6 +644,8 @@ describe("host-local model cleanup", () => { ["rm", "--force", RUNTIME_ID], expect.any(Number), ); + expect(privateBridge.stopTransaction).not.toHaveBeenCalled(); + expect(privateBridge.assertStopped).not.toHaveBeenCalled(); expect(fs.existsSync(managedLlamaCppStatePaths(homeDir).stateDir)).toBe(true); } finally { store.releaseExecution(lease); diff --git a/src/lib/inference/local-model-profile/cleanup.ts b/src/lib/inference/local-model-profile/cleanup.ts index bcfc054e124..6072a61b4c8 100644 --- a/src/lib/inference/local-model-profile/cleanup.ts +++ b/src/lib/inference/local-model-profile/cleanup.ts @@ -549,22 +549,23 @@ function cleanupLlamaCpp( throw new Error("managed llama.cpp finalized receipt is missing"); } - const privateBridge = - options.privateBridge ?? - (process.platform === "linux" ? createDockerLlamaCppPrivateBridgeController() : undefined); - if (privateBridge) { - privateBridge.stopTransaction(journal.transactionId); - privateBridge.assertStopped(journal.transactionId); - } - const engine = options.engine ?? createManagedLlamaCppEngine(options.env ?? process.env); - requireQualifiedEngine(receipt?.engineAuthority ?? journal.engineAuthority, engine); - requireEngineSuccess( - "engine availability check", - engine.capture(["info"], DOCKER_INSPECT_TIMEOUT_MS), - ); - const lease = journalStore.acquireExecution(journal.transactionId); try { + journalStore.assertExecution(lease); + const privateBridge = + options.privateBridge ?? + (process.platform === "linux" ? createDockerLlamaCppPrivateBridgeController() : undefined); + if (privateBridge) { + privateBridge.stopTransaction(journal.transactionId); + privateBridge.assertStopped(journal.transactionId); + } + journalStore.assertExecution(lease); + const engine = options.engine ?? createManagedLlamaCppEngine(options.env ?? process.env); + requireQualifiedEngine(receipt?.engineAuthority ?? journal.engineAuthority, engine); + requireEngineSuccess( + "engine availability check", + engine.capture(["info"], DOCKER_INSPECT_TIMEOUT_MS), + ); journalStore.assertExecution(lease); removeExactContainerForJournal(engine, journal, removed); journalStore.assertExecution(lease); From c1bc00ca17528340b09a4d4b5612c317a434e0df Mon Sep 17 00:00:00 2001 From: prekshivyas Date: Wed, 19 Aug 2026 18:49:58 -0700 Subject: [PATCH 08/10] fix(ci): compile catalog for CPU delegation proof --- .github/workflows/podman-cpu-proof.yaml | 3 +++ .../support/podman-cpu-proof-workflow.test.ts | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/.github/workflows/podman-cpu-proof.yaml b/.github/workflows/podman-cpu-proof.yaml index 9b3cb7c5550..f349ac11d52 100644 --- a/.github/workflows/podman-cpu-proof.yaml +++ b/.github/workflows/podman-cpu-proof.yaml @@ -80,6 +80,9 @@ jobs: - name: Build shared sandbox-name contract run: npm run build:policy-boundary + - name: Compile managed inference catalog + run: npm run catalog:compile + - name: Prepare system and app slice CPU settings without service delegation shell: bash run: node --experimental-strip-types scripts/checks/run-portable-cpu-delegation-proof.mts prepare diff --git a/test/e2e/support/podman-cpu-proof-workflow.test.ts b/test/e2e/support/podman-cpu-proof-workflow.test.ts index 821bc7ea438..f57b332f24b 100644 --- a/test/e2e/support/podman-cpu-proof-workflow.test.ts +++ b/test/e2e/support/podman-cpu-proof-workflow.test.ts @@ -370,6 +370,22 @@ describe("native Podman CPU proof workflow", () => { expect(selectedPath).toBe(authorityPath); }); + it("compiles the managed inference catalog before the live delegation proof", () => { + const steps = delegationJob().steps ?? []; + const catalogIndex = steps.findIndex( + ({ name }) => name === "Compile managed inference catalog", + ); + const prepareIndex = steps.findIndex( + ({ name }) => name === "Prepare system and app slice CPU settings without service delegation", + ); + + expect(namedDelegationStep("Compile managed inference catalog").run).toBe( + "npm run catalog:compile", + ); + expect(catalogIndex).toBeGreaterThan(-1); + expect(catalogIndex).toBeLessThan(prepareIndex); + }); + it("executes the five typed proof modes with exact argv and durable cleanup receipts (#9188)", () => { withProofFixture((fixture) => { const modes: readonly PortableCpuDelegationProofMode[] = [ From 6a7c50908a925a5965159208930a84e3a9ab83a2 Mon Sep 17 00:00:00 2001 From: prekshivyas Date: Wed, 19 Aug 2026 18:55:08 -0700 Subject: [PATCH 09/10] test(ci): avoid shape-locking proof workflow --- .../support/podman-cpu-proof-workflow.test.ts | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/test/e2e/support/podman-cpu-proof-workflow.test.ts b/test/e2e/support/podman-cpu-proof-workflow.test.ts index f57b332f24b..821bc7ea438 100644 --- a/test/e2e/support/podman-cpu-proof-workflow.test.ts +++ b/test/e2e/support/podman-cpu-proof-workflow.test.ts @@ -370,22 +370,6 @@ describe("native Podman CPU proof workflow", () => { expect(selectedPath).toBe(authorityPath); }); - it("compiles the managed inference catalog before the live delegation proof", () => { - const steps = delegationJob().steps ?? []; - const catalogIndex = steps.findIndex( - ({ name }) => name === "Compile managed inference catalog", - ); - const prepareIndex = steps.findIndex( - ({ name }) => name === "Prepare system and app slice CPU settings without service delegation", - ); - - expect(namedDelegationStep("Compile managed inference catalog").run).toBe( - "npm run catalog:compile", - ); - expect(catalogIndex).toBeGreaterThan(-1); - expect(catalogIndex).toBeLessThan(prepareIndex); - }); - it("executes the five typed proof modes with exact argv and durable cleanup receipts (#9188)", () => { withProofFixture((fixture) => { const modes: readonly PortableCpuDelegationProofMode[] = [ From fbfe522b396df5369583557df90764d34b04e02c Mon Sep 17 00:00:00 2001 From: Senthil Ravichandran Date: Wed, 19 Aug 2026 19:15:06 -0700 Subject: [PATCH 10/10] fix(uninstall): preserve model stores after cleanup failure Signed-off-by: Senthil Ravichandran --- .../run-plan-local-model-profile.test.ts | 29 +++++++++++-------- src/lib/actions/uninstall/run-plan.ts | 15 +++++++++- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/src/lib/actions/uninstall/run-plan-local-model-profile.test.ts b/src/lib/actions/uninstall/run-plan-local-model-profile.test.ts index 7044c243bfa..fafefb2d584 100644 --- a/src/lib/actions/uninstall/run-plan-local-model-profile.test.ts +++ b/src/lib/actions/uninstall/run-plan-local-model-profile.test.ts @@ -103,13 +103,7 @@ function publishManagedLlamaOwner( } function writeScopedGatewayState(home: string): void { - const stateDir = path.join( - home, - ".local", - "state", - "nemoclaw", - "openshell-docker-gateway", - ); + const stateDir = path.join(home, ".local", "state", "nemoclaw", "openshell-docker-gateway"); const jwtBundle = ensureDockerDriverGatewayJwtBundle(stateDir); fs.writeFileSync( path.join(stateDir, "openshell-gateway.toml"), @@ -443,10 +437,12 @@ describe("uninstall local model profile cleanup", () => { ); expect(result.exitCode).toBe(0); - run.mock.calls.filter(([command]) => command === "ollama").forEach(([command, , options]) => { - expect(command).toBe("ollama"); - expect(options?.env?.OLLAMA_HOST).toBe("127.0.0.1:11434"); - }); + run.mock.calls + .filter(([command]) => command === "ollama") + .forEach(([command, , options]) => { + expect(command).toBe("ollama"); + expect(options?.env?.OLLAMA_HOST).toBe("127.0.0.1:11434"); + }); }); it("fails without deleting any Ollama model when inventory is malformed", () => { @@ -633,10 +629,13 @@ describe("uninstall local model profile cleanup", () => { const siblingOwnerBefore = loadManagedLlamaCppOwner(siblingPaths); writeScopedGatewayState(tmpHome); const unrelatedState = path.join(tmpHome, ".nemoclaw", "unrelated-state.json"); + const cacheDir = path.join(tmpHome, ".cache", "huggingface"); + fs.mkdirSync(cacheDir, { recursive: true }); fs.writeFileSync(unrelatedState, "{}\n", { mode: 0o600 }); const errors: string[] = []; const logs: string[] = []; const runLocalModelRuntimeCleanup = vi.fn(() => ok()); + const runHuggingFaceCacheDataCleanup = vi.fn(() => ok()); const runManagedLlamaCppRuntimeCleanup = vi.fn(() => ({ status: 1, stdout: "", @@ -644,7 +643,7 @@ describe("uninstall local model profile cleanup", () => { })); try { const result = runUninstallPlan( - { assumeYes: true, deleteModels: false, keepOpenShell: true }, + { assumeYes: true, deleteModels: true, keepOpenShell: true }, withProvenManagedGatewayProcess({ commandExists: (command) => command === "openshell", env: { HOME: tmpHome } as NodeJS.ProcessEnv, @@ -654,6 +653,7 @@ describe("uninstall local model profile cleanup", () => { isTty: false, log: (message) => logs.push(message), run: vi.fn(okWithKnownGatewayList), + runHuggingFaceCacheDataCleanup, runLocalModelRuntimeCleanup, runManagedLlamaCppRuntimeCleanup, }), @@ -670,7 +670,12 @@ describe("uninstall local model profile cleanup", () => { expect(loadManagedLlamaCppOwner(selectedPaths)).toEqual(selectedOwnerBefore); expect(loadManagedLlamaCppOwner(siblingPaths)).toEqual(siblingOwnerBefore); expect(fs.existsSync(unrelatedState)).toBe(false); + expect(fs.existsSync(cacheDir)).toBe(true); + expect(runHuggingFaceCacheDataCleanup).not.toHaveBeenCalled(); expect(runLocalModelRuntimeCleanup).not.toHaveBeenCalled(); + expect(logs).toContain( + "Managed llama.cpp cleanup did not complete. NemoClaw kept model stores for retry.", + ); expect(logs.some((message) => message.endsWith("State and binaries"))).toBe(true); expect(errors.join("\n")).toContain("continue unrelated uninstall steps"); } finally { diff --git a/src/lib/actions/uninstall/run-plan.ts b/src/lib/actions/uninstall/run-plan.ts index cc35364c05c..a36103a9864 100644 --- a/src/lib/actions/uninstall/run-plan.ts +++ b/src/lib/actions/uninstall/run-plan.ts @@ -2303,7 +2303,12 @@ function removeHostModelStores( options: UninstallRunOptions, runtime: UninstallRuntime, scopedToSelectedGateway: boolean, + preserveForFailedLlamaCleanup: boolean, ): boolean { + if (preserveForFailedLlamaCleanup) { + runtime.log("Managed llama.cpp cleanup did not complete. NemoClaw kept model stores for retry."); + return true; + } if (scopedToSelectedGateway) { runtime.log( "Sibling gateways remain; kept host-shared Ollama models and the Hugging Face model cache.", @@ -2947,7 +2952,15 @@ function executePlan( if (action.kind === "delete-docker-volume") removeDockerVolume(action.name, runtime); } } else if (step.name === "Model stores") { - if (!removeHostModelStores(paths, options, runtime, scopedToSelectedGateway)) { + if ( + !removeHostModelStores( + paths, + options, + runtime, + scopedToSelectedGateway, + failedManagedLlamaStateDirs.length > 0, + ) + ) { ok = false; } } else if (step.name === "State and binaries") {