fix(uninstall): preserve failed llama.cpp cleanup state

.github/workflows/podman-cpu-proof.yaml

@@ -80,6 +80,9 @@ jobs:
       - name: Build shared sandbox-name contract
         run: npm run build:policy-boundary
 
+      - name: Compile managed inference catalog
+        run: npm run catalog:compile
+
       - name: Prepare system and app slice CPU settings without service delegation
         shell: bash
         run: node --experimental-strip-types scripts/checks/run-portable-cpu-delegation-proof.mts prepare

src/lib/actions/uninstall/run-plan-local-model-profile.test.ts

@@ -12,6 +12,7 @@ import {
 } from "../../../../test/support/uninstall-managed-gateway-test-support";
 
 import {
+  loadManagedLlamaCppOwner,
   managedLlamaCppStatePaths,
   reserveManagedLlamaCppOwner,
 } from "../../inference/llama-cpp/managed-state";
@@ -102,13 +103,7 @@ function publishManagedLlamaOwner(
 }
 
 function writeScopedGatewayState(home: string): void {
-  const stateDir = path.join(
-    home,
-    ".local",
-    "state",
-    "nemoclaw",
-    "openshell-docker-gateway",
-  );
+  const stateDir = path.join(home, ".local", "state", "nemoclaw", "openshell-docker-gateway");
   const jwtBundle = ensureDockerDriverGatewayJwtBundle(stateDir);
   fs.writeFileSync(
     path.join(stateDir, "openshell-gateway.toml"),
@@ -442,10 +437,12 @@ describe("uninstall local model profile cleanup", () => {
     );
 
     expect(result.exitCode).toBe(0);
-    run.mock.calls.filter(([command]) => command === "ollama").forEach(([command, , options]) => {
-      expect(command).toBe("ollama");
-      expect(options?.env?.OLLAMA_HOST).toBe("127.0.0.1:11434");
-    });
+    run.mock.calls
+      .filter(([command]) => command === "ollama")
+      .forEach(([command, , options]) => {
+        expect(command).toBe("ollama");
+        expect(options?.env?.OLLAMA_HOST).toBe("127.0.0.1:11434");
+      });
   });
 
   it("fails without deleting any Ollama model when inventory is malformed", () => {
@@ -620,36 +617,67 @@ describe("uninstall local model profile cleanup", () => {
     }
   });
 
-  it("preserves selected gateway authority when scoped cleanup leaves ownership state", () => {
+  it("continues unrelated uninstall after managed llama.cpp cleanup fails (#9575)", () => {
     const tmpHome = fs.realpathSync(
       fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-llama-fail-")),
     );
     const stateDir = publishManagedLlamaOwner(tmpHome, 8080, "selected-sandbox");
+    const siblingStateDir = publishManagedLlamaOwner(tmpHome, 9000, "sibling-sandbox");
+    const selectedPaths = managedLlamaCppStatePaths(tmpHome, 8080);
+    const siblingPaths = managedLlamaCppStatePaths(tmpHome, 9000);
+    const selectedOwnerBefore = loadManagedLlamaCppOwner(selectedPaths);
+    const siblingOwnerBefore = loadManagedLlamaCppOwner(siblingPaths);
     writeScopedGatewayState(tmpHome);
+    const unrelatedState = path.join(tmpHome, ".nemoclaw", "unrelated-state.json");
+    const cacheDir = path.join(tmpHome, ".cache", "huggingface");
+    fs.mkdirSync(cacheDir, { recursive: true });
+    fs.writeFileSync(unrelatedState, "{}\n", { mode: 0o600 });
     const errors: string[] = [];
+    const logs: string[] = [];
+    const runLocalModelRuntimeCleanup = vi.fn(() => ok());
+    const runHuggingFaceCacheDataCleanup = vi.fn(() => ok());
+    const runManagedLlamaCppRuntimeCleanup = vi.fn(() => ({
+      status: 1,
+      stdout: "",
+      stderr: "qualified endpoint changed",
+    }));
     try {
       const result = runUninstallPlan(
-        { assumeYes: true, deleteModels: false, keepOpenShell: true },
+        { assumeYes: true, deleteModels: true, keepOpenShell: true },
         withProvenManagedGatewayProcess({
           commandExists: (command) => command === "openshell",
           env: { HOME: tmpHome } as NodeJS.ProcessEnv,
           error: (message) => errors.push(message),
           existsSync: fs.existsSync,
           isPortFree: () => true,
           isTty: false,
-          log: () => {},
-          run: vi.fn((command: string, args: string[]) =>
-            command === "openshell" && args[0] === "gateway" && args[1] === "list"
-              ? ok(JSON.stringify([{ name: "nemoclaw" }, { name: "nemoclaw-9000" }]))
-              : ok(),
-          ),
-          runManagedLlamaCppRuntimeCleanup: vi.fn(() => ok()),
+          log: (message) => logs.push(message),
+          run: vi.fn(okWithKnownGatewayList),
+          runHuggingFaceCacheDataCleanup,
+          runLocalModelRuntimeCleanup,
+          runManagedLlamaCppRuntimeCleanup,
         }),
       );
 
       expect(result.exitCode).toBe(1);
+      expect(runManagedLlamaCppRuntimeCleanup).toHaveBeenCalledExactlyOnceWith(
+        "selected-sandbox",
+        8080,
+      );
       expect(fs.existsSync(stateDir)).toBe(true);
-      expect(errors.join("\n")).toContain("returned without retiring its ownership state");
+      expect(fs.existsSync(path.join(stateDir, "owner.json"))).toBe(true);
+      expect(fs.existsSync(path.join(siblingStateDir, "owner.json"))).toBe(true);
+      expect(loadManagedLlamaCppOwner(selectedPaths)).toEqual(selectedOwnerBefore);
+      expect(loadManagedLlamaCppOwner(siblingPaths)).toEqual(siblingOwnerBefore);
+      expect(fs.existsSync(unrelatedState)).toBe(false);
+      expect(fs.existsSync(cacheDir)).toBe(true);
+      expect(runHuggingFaceCacheDataCleanup).not.toHaveBeenCalled();
+      expect(runLocalModelRuntimeCleanup).not.toHaveBeenCalled();
+      expect(logs).toContain(
+        "Managed llama.cpp cleanup did not complete. NemoClaw kept model stores for retry.",
+      );
+      expect(logs.some((message) => message.endsWith("State and binaries"))).toBe(true);
+      expect(errors.join("\n")).toContain("continue unrelated uninstall steps");
     } finally {
       fs.rmSync(tmpHome, { recursive: true, force: true });
     }

src/lib/actions/uninstall/run-plan.ts

@@ -600,6 +600,7 @@ function buildRuntime(deps: UninstallRunDeps): UninstallRuntime {
       deps.runManagedLlamaCppRuntimeCleanup ??
       ((sandboxName, gatewayPort) => {
         const result = cleanupManagedLlamaCppRuntimeForSandbox(sandboxName, {
+          env,
           gatewayPort,
           homeDir: env.HOME || os.homedir(),
         });
@@ -2027,49 +2028,59 @@ function managedLlamaCppCleanupTargets(
   }
 }
 
+interface ManagedLlamaCppCleanupOutcome {
+  readonly failedStateDirs: readonly string[];
+  readonly ok: boolean;
+}
+
 function removeManagedLlamaCppRuntimes(
   runtime: UninstallRuntime,
   scopedToSelectedGateway: boolean,
-): boolean {
+): ManagedLlamaCppCleanupOutcome {
   const targets = managedLlamaCppCleanupTargets(runtime, scopedToSelectedGateway);
-  if (targets === null) return false;
+  if (targets === null) return { failedStateDirs: [], ok: false };
   for (const target of targets) {
     const result = runtime.runManagedLlamaCppRuntimeCleanup(target.sandboxName, target.gatewayPort);
     for (const removed of splitNonEmptyLines(result.stdout)) runtime.log(`Removed ${removed}`);
     if (result.status !== 0) {
       runtime.error(
-        `Managed llama.cpp cleanup for sandbox '${target.sandboxName}' on gateway port ${String(target.gatewayPort)} did not complete: ${result.stderr.trim() || "unknown cleanup error"}. NemoClaw did not start the remaining uninstall steps.`,
+        `Managed llama.cpp cleanup for sandbox '${target.sandboxName}' on gateway port ${String(target.gatewayPort)} did not complete: ${result.stderr.trim() || "unknown cleanup error"}. NemoClaw will preserve its ownership state and continue unrelated uninstall steps.`,
       );
-      return false;
+      return { failedStateDirs: [target.stateDir], ok: true };
     }
     if (fs.lstatSync(target.stateDir, { throwIfNoEntry: false }) !== undefined) {
       runtime.error(
-        `Managed llama.cpp cleanup for sandbox '${target.sandboxName}' on gateway port ${String(target.gatewayPort)} returned without retiring its ownership state. NemoClaw did not start the remaining uninstall steps.`,
+        `Managed llama.cpp cleanup for sandbox '${target.sandboxName}' on gateway port ${String(target.gatewayPort)} returned without retiring its ownership state. NemoClaw will preserve its ownership state and continue unrelated uninstall steps.`,
       );
-      return false;
+      return { failedStateDirs: [target.stateDir], ok: true };
     }
   }
-  return true;
+  return { failedStateDirs: [], ok: true };
 }
 
 function removeManagedModelRuntimes(
   paths: UninstallPaths,
   runtime: UninstallRuntime,
   scopedToSelectedGateway: boolean,
-): boolean {
-  if (!removeManagedLlamaCppRuntimes(runtime, scopedToSelectedGateway)) return false;
-  if (scopedToSelectedGateway) return true;
+): ManagedLlamaCppCleanupOutcome {
+  const llama = removeManagedLlamaCppRuntimes(runtime, scopedToSelectedGateway);
+  if (!llama.ok || llama.failedStateDirs.length > 0) return llama;
+  if (scopedToSelectedGateway) return llama;
   const sharedRoot = path.dirname(paths.managedSwapMarkerPath);
   const hasDistributedReceipt = [
     MANAGED_CLUSTER_VLLM_RUNTIME_RECEIPT_FILE,
     DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE,
   ].some((name) => runtime.existsSync(path.join(sharedRoot, name)));
-  if (!removeManagedDistributedVllmRuntime(paths, runtime, !hasDistributedReceipt)) return false;
-  if (!removeHostLocalModelRuntimes(paths, runtime)) return false;
+  if (!removeManagedDistributedVllmRuntime(paths, runtime, !hasDistributedReceipt)) {
+    return { failedStateDirs: [], ok: false };
+  }
+  if (!removeHostLocalModelRuntimes(paths, runtime)) {
+    return { failedStateDirs: [], ok: false };
+  }
   if (!hasDistributedReceipt) {
     removePath(path.join(sharedRoot, MANAGED_VLLM_API_KEY_FILE), runtime);
   }
-  if (!runtime.commandExists("docker")) return true;
+  if (!runtime.commandExists("docker")) return llama;
   const inventory = runtime.runDocker(["ps", "-a", "--format", "{{.Names}}"], {
     env: runtime.env,
     timeout: 10_000,
@@ -2078,16 +2089,29 @@ function removeManagedModelRuntimes(
     runtime.error(
       "Docker could not inventory reserved managed inference container names. NemoClaw refused the remaining uninstall steps so it cannot report incomplete cleanup as success.",
     );
-    return false;
+    return { failedStateDirs: [], ok: false };
   }
   const residual = splitNonEmptyLines(inventory.stdout).find((name) =>
     MANAGED_INFERENCE_CONTAINER_NAME_PATTERN.test(name),
   );
-  if (!residual) return true;
+  if (!residual) return llama;
   runtime.error(
     `Managed inference container '${residual}' remains after ownership-aware cleanup. NemoClaw refused the remaining uninstall steps; restore its ownership state or remove it after manual review, then retry.`,
   );
-  return false;
+  return { failedStateDirs: [], ok: false };
+}
+
+function recordManagedModelCleanup(
+  paths: UninstallPaths,
+  runtime: UninstallRuntime,
+  scopedToSelectedGateway: boolean,
+  failedStateDirs: string[],
+  onPartialFailure: () => void,
+): boolean {
+  const result = removeManagedModelRuntimes(paths, runtime, scopedToSelectedGateway);
+  failedStateDirs.push(...result.failedStateDirs);
+  if (result.failedStateDirs.length > 0) onPartialFailure();
+  return result.ok;
 }
 
 function removeDockerContainers(runtime: UninstallRuntime, gatewayName?: string): void {
@@ -2279,7 +2303,12 @@ function removeHostModelStores(
   options: UninstallRunOptions,
   runtime: UninstallRuntime,
   scopedToSelectedGateway: boolean,
+  preserveForFailedLlamaCleanup: boolean,
 ): boolean {
+  if (preserveForFailedLlamaCleanup) {
+    runtime.log("Managed llama.cpp cleanup did not complete. NemoClaw kept model stores for retry.");
+    return true;
+  }
   if (scopedToSelectedGateway) {
     runtime.log(
       "Sibling gateways remain; kept host-shared Ollama models and the Hugging Face model cache.",
@@ -2752,6 +2781,7 @@ function executePlan(
     return { ok: false };
   }
   let ok = true;
+  const failedManagedLlamaStateDirs: string[] = [];
   const branding = runtimeBranding(runtime);
   const preserveSharedOpenShell =
     options.keepOpenShell || externallySupervised || portableRuntimeCleanup;
@@ -2793,7 +2823,15 @@ function executePlan(
     if (step.name === "Stopping services") {
       if (
         !portableRuntimeCleanup &&
-        !removeManagedModelRuntimes(paths, runtime, scopedToSelectedGateway)
+        !recordManagedModelCleanup(
+          paths,
+          runtime,
+          scopedToSelectedGateway,
+          failedManagedLlamaStateDirs,
+          () => {
+            ok = false;
+          },
+        )
       ) {
         return { ok: false };
       }
@@ -2914,7 +2952,15 @@ function executePlan(
           if (action.kind === "delete-docker-volume") removeDockerVolume(action.name, runtime);
       }
     } else if (step.name === "Model stores") {
-      if (!removeHostModelStores(paths, options, runtime, scopedToSelectedGateway)) {
+      if (
+        !removeHostModelStores(
+          paths,
+          options,
+          runtime,
+          scopedToSelectedGateway,
+          failedManagedLlamaStateDirs.length > 0,
+        )
+      ) {
         ok = false;
       }
     } else if (step.name === "State and binaries") {
@@ -2957,6 +3003,12 @@ function executePlan(
           [
             ...preserveUnderStateDir,
             ...portableStateEntries,
+            ...failedManagedLlamaStateDirs.flatMap((stateDir) => {
+              const relative = path.relative(paths.nemoclawStateDir, stateDir);
+              return relative && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative)
+                ? [relative.split(path.sep)[0]!]
+                : [];
+            }),
             ...(selectedIsDefault
               ? [GATEWAYS_SUBDIR, path.basename(paths.managedSwapMarkerPath)]
               : []),

src/lib/inference/local-model-profile/cleanup.test.ts

@@ -582,17 +582,21 @@ describe("host-local model cleanup", () => {
     const homeDir = temporaryHome();
     const original = engineHarness({ authorityId: "docker:original" });
     const changed = engineHarness({ authorityId: "docker:changed" });
+    const privateBridge = privateBridgeFixture();
     createManagedState(homeDir, original.engine);
 
     const result = cleanupLocalModelRuntimes({
       homeDir,
       engine: changed.engine,
+      privateBridge,
     });
 
     expect(result).toMatchObject({
       ok: false,
       reason: expect.stringContaining("endpoint"),
     });
+    expect(privateBridge.stopTransaction).toHaveBeenCalledExactlyOnceWith(TRANSACTION_ID);
+    expect(privateBridge.assertStopped).toHaveBeenCalledExactlyOnceWith(TRANSACTION_ID);
     expect(changed.capture).not.toHaveBeenCalled();
     expect(fs.existsSync(managedLlamaCppStatePaths(homeDir).stateDir)).toBe(true);
   });
@@ -621,13 +625,15 @@ describe("host-local model cleanup", () => {
   it("does not race cleanup against a live lifecycle execution lease", () => {
     const homeDir = temporaryHome();
     const harness = engineHarness();
+    const privateBridge = privateBridgeFixture();
     createManagedState(homeDir, harness.engine, { phase: "started" });
     const store = createHostLocalCreateJournalStore(managedLlamaCppStatePaths(homeDir).stateDir);
     const lease = store.acquireExecution(TRANSACTION_ID);
     try {
       const result = cleanupLocalModelRuntimes({
         homeDir,
         engine: harness.engine,
+        privateBridge,
       });
 
       expect(result).toMatchObject({
@@ -638,6 +644,8 @@ describe("host-local model cleanup", () => {
         ["rm", "--force", RUNTIME_ID],
         expect.any(Number),
       );
+      expect(privateBridge.stopTransaction).not.toHaveBeenCalled();
+      expect(privateBridge.assertStopped).not.toHaveBeenCalled();
       expect(fs.existsSync(managedLlamaCppStatePaths(homeDir).stateDir)).toBe(true);
     } finally {
       store.releaseExecution(lease);