From 3bf053e656679272ef94750568efd1631a91b9e4 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Wed, 12 Aug 2026 13:13:35 -0700 Subject: [PATCH 1/4] fix(uninstall): delete all installed host models Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- .../choose-local-inference-server.mdx | 6 +- docs/inference/set-up-llama-cpp.mdx | 5 +- docs/manage-sandboxes/uninstall-nemoclaw.mdx | 31 ++- docs/reference/commands.mdx | 13 +- docs/reference/host-files-and-state.mdx | 10 +- scripts/smoke-macos-install.sh | 2 +- src/commands/internal/uninstall/plan.ts | 5 +- src/commands/internal/uninstall/run-plan.ts | 2 +- src/lib/actions/root-help.ts | 4 +- .../uninstall/run-plan-dual-station.test.ts | 44 ++++ .../run-plan-local-model-profile.test.ts | 210 +++++++++++++++++- src/lib/actions/uninstall/run-plan.ts | 136 ++++++++++-- src/lib/domain/uninstall/paths.test.ts | 1 + src/lib/domain/uninstall/paths.ts | 3 +- src/lib/domain/uninstall/plan.test.ts | 19 +- src/lib/domain/uninstall/plan.ts | 25 ++- .../cleanup-path-safety.test.ts | 56 +++++ .../local-model-profile/cleanup.test.ts | 25 ++- .../inference/local-model-profile/cleanup.ts | 81 ++++++- test/uninstall.test.ts | 2 +- 20 files changed, 612 insertions(+), 68 deletions(-) diff --git a/docs/inference/choose-local-inference-server.mdx b/docs/inference/choose-local-inference-server.mdx index 359ae3e1e61..c24c30885f9 100644 --- a/docs/inference/choose-local-inference-server.mdx +++ b/docs/inference/choose-local-inference-server.mdx @@ -153,8 +153,10 @@ Refer to [Host Files and State](../../reference/host-files-and-state) before del When you intend to remove the entire NemoClaw installation, run `$$nemoclaw uninstall`. Full uninstall verifies exact managed container ownership before it removes the runtime and its state. -It preserves the shared Hugging Face cache used by vLLM. -Add `--delete-models` only when you also accept deletion of NemoClaw-pulled Ollama models. +It preserves the shared Hugging Face cache used by vLLM by default. +Add `--delete-models` only when you accept deletion of every model reported by the host's local Ollama inventory and all non-credential data in the current user's shared Hugging Face cache. +This opt-in can delete cached files that other applications installed or use. +It preserves the Hugging Face `token` and `stored_tokens` authentication files. If cleanup reports an ownership or Docker error, leave the state files in place, resolve the reported resource, and rerun uninstall. After a successful full uninstall, `docker container inspect nemoclaw-vllm` must report no object. diff --git a/docs/inference/set-up-llama-cpp.mdx b/docs/inference/set-up-llama-cpp.mdx index f47315995fe..7fdde948a2a 100644 --- a/docs/inference/set-up-llama-cpp.mdx +++ b/docs/inference/set-up-llama-cpp.mdx @@ -246,7 +246,10 @@ It preserves `~/.cache/huggingface/` because other applications can use that cac If exact cleanup fails, NemoClaw preserves the sandbox registry entry and ownership state for a retry. For full NemoClaw removal, run `$$nemoclaw uninstall`. -Full uninstall applies the same exact-ownership checks and preserves the shared Hugging Face cache. +Full uninstall applies the same exact-ownership checks and preserves the shared Hugging Face cache by default. +Pass `--delete-models` only when you accept deletion of all non-credential data in the current user's shared cache after managed model runtimes stop. +The same flag also deletes every model installed in the host's local Ollama inventory. +Uninstall preserves the Hugging Face `token` and `stored_tokens` authentication files. ## Related Topics diff --git a/docs/manage-sandboxes/uninstall-nemoclaw.mdx b/docs/manage-sandboxes/uninstall-nemoclaw.mdx index 3ba7e5f5c46..2c1e1829d56 100644 --- a/docs/manage-sandboxes/uninstall-nemoclaw.mdx +++ b/docs/manage-sandboxes/uninstall-nemoclaw.mdx @@ -24,7 +24,7 @@ It preserves resources from a separate OpenClaw installation while continuing to |---|---| | `--yes` | Skip the confirmation prompt. | | `--keep-openshell` | Leave OpenShell binaries, NemoClaw-managed gateway service files, and local gateway state in place, and do not stop the host gateway process. | -| `--delete-models` | Also remove NemoClaw-pulled Ollama models. | +| `--delete-models` | Delete every model reported by the host's local Ollama inventory and all non-credential data in the current user's shared `~/.cache/huggingface/` cache after managed model runtimes stop. Hugging Face authentication files remain. | | `--destroy-user-data` | Also remove preserved user data in `rebuild-backups/`, `backups/`, and `sandboxes.json`. | | `--all-gateway-ports` | Uninstall every gateway port on the host instead of only the selected one. | | `--gateway ` | Optional consistency check; must match the name derived from `NEMOCLAW_GATEWAY_PORT`. | @@ -33,6 +33,35 @@ It preserves resources from a separate OpenClaw installation while continuing to For example, `NEMOCLAW_GATEWAY_PORT=9123 $$nemoclaw uninstall` selects `nemoclaw-9123` and its port-scoped state. Do not use `--gateway` to select another instance; when supplied for compatibility, its value must match the derived name or uninstall stops before cleanup. +## Delete Host Models + +The `--delete-models` flag is an explicit opt-in for deleting host model files. + + +This flag deletes every model that the local Ollama inventory reports, including models installed or used by other applications. +It also deletes all non-credential data in the current user's `~/.cache/huggingface/` directory, which other applications can use for models, datasets, and other cached assets. +Affected applications must download and verify their cached files again. +The Hugging Face `token` and `stored_tokens` authentication files remain, so this cleanup does not sign the current user out. + + +Run full uninstall with model deletion: + +```bash +$$nemoclaw uninstall --delete-models +``` + +NemoClaw stops and verifies its managed local and distributed model runtimes before it deletes non-credential data from the local Hugging Face cache. +It validates that `~/.cache/` and `~/.cache/huggingface/` are real, current-user-owned directories that are not symlinks or group- or world-writable. +It inventories Ollama before it deletes any Ollama model. +An Ollama inventory error, model deletion error, unsafe cache path, or cache-data deletion error makes uninstall exit nonzero. +Cleanup can partially complete before an error. +Resolve the reported error, inspect the remaining models and runtimes, and rerun uninstall. + +Without `--delete-models`, uninstall preserves both model stores. +When sibling gateway environments remain, uninstall preserves both model stores even if you pass `--delete-models`. +An all-gateway-port sweep deletes them only after every selected gateway cleanup succeeds and no sibling environment remains. +The flag does not scan arbitrary directories or delete model caches on remote peers. + ## Uninstall Every Gateway Port One uninstall removes one gateway port. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index f3febfd1a94..bcc55f42feb 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -3954,8 +3954,15 @@ These host-local checks run before NemoClaw deletes their state. If Docker is unavailable or a resource does not match its persisted ownership state, uninstall exits nonzero before the remaining uninstall steps and preserves that state for recovery. Host-local cleanup can partially complete before an error. Restore Docker access or resolve the named ownership conflict, inspect the remaining container and network, and retry uninstall. -Managed llama.cpp and vLLM model files remain in the shared Hugging Face cache. -The `--delete-models` flag does not remove this shared cache. +Managed llama.cpp and vLLM model files remain in the shared Hugging Face cache by default. +With `--delete-models`, uninstall deletes every model in the local Ollama inventory and all non-credential data in the current user's shared `~/.cache/huggingface/` cache. +This opt-in can delete cached files that other applications installed or use. +It preserves the Hugging Face `token` and `stored_tokens` authentication files. +NemoClaw stops and verifies its managed local and distributed model runtimes before it deletes non-credential data from the local Hugging Face cache. +It does not scan arbitrary directories or delete model caches on remote peers. +When sibling gateway environments remain, uninstall preserves both model stores even if you pass `--delete-models`. +An Ollama inventory error, model deletion error, unsafe cache path, or cache-data deletion error makes uninstall exit nonzero. +Cleanup can partially complete before an error, so resolve the reported error and rerun uninstall. It does not use the legacy `gateway destroy` command for that gateway. @@ -3981,7 +3988,7 @@ Uninstall does not otherwise target the declared external directory. |---|---| | `--yes` | Skip the confirmation prompt | | `--keep-openshell` | Leave OpenShell binaries, NemoClaw-managed gateway service files, and local gateway state in place, and do not stop the host gateway process | -| `--delete-models` | Also remove NemoClaw-pulled Ollama models. The shared Hugging Face cache remains in place. | +| `--delete-models` | Delete every model reported by the host's local Ollama inventory and all non-credential data in the current user's shared `~/.cache/huggingface/` cache after managed model runtimes stop. Hugging Face authentication files remain. | | `--destroy-user-data` | Also remove preserved user data (`rebuild-backups/`, `backups/`, `sandboxes.json`) | | `--all-gateway-ports` | Uninstall every gateway port on the host, not only the port `NEMOCLAW_GATEWAY_PORT` selects | | `--gateway ` | Optional consistency check; must match the name derived from `NEMOCLAW_GATEWAY_PORT` | diff --git a/docs/reference/host-files-and-state.mdx b/docs/reference/host-files-and-state.mdx index 98cdd0d9334..ca8315f87b1 100644 --- a/docs/reference/host-files-and-state.mdx +++ b/docs/reference/host-files-and-state.mdx @@ -93,7 +93,7 @@ Run `$$nemoclaw onboard` to register a sandbox again. | `~/.nemoclaw/backups/` | Workspace backups written by legacy backup helpers and some recovery flows. | Only after confirming you no longer need those workspace archives. | | `~/.nemoclaw/mounts/` | Default local mount points created by share or mount commands. | Unmount first, then remove unused directories. | | `~/.nemoclaw/blueprints/` | Cached blueprint inputs used by onboarding and sandbox recreation. | Avoid manual deletion unless you plan to rerun onboarding from fresh inputs. | -| `~/.cache/huggingface/` | Shared Hugging Face cache used by managed vLLM and managed llama.cpp model acquisition. The llama.cpp path resolves the exact YAML-declared revision and GGUF file, then verifies its size, SHA-256 digest, and filesystem identity before launch. Other applications can use the same cache. | Only after stopping every process that uses the cache. Destroy and full uninstall preserve this shared cache, including when you pass `--delete-models`. Deleting it makes affected applications download and verify model files again. | +| `~/.cache/huggingface/` | Shared Hugging Face cache used by managed vLLM and managed llama.cpp model acquisition. The llama.cpp path resolves the exact YAML-declared revision and GGUF file, then verifies its size, SHA-256 digest, and filesystem identity before launch. Other applications can use the same cache. | Only after stopping every process that uses the cache. Sandbox destroy and full uninstall without `--delete-models` preserve it. Full uninstall with `--delete-models` deletes its non-credential data after managed model runtimes stop and only when no sibling gateway remains. The `token` and `stored_tokens` authentication files remain. Affected applications must download and verify their cached files again. | ## Migration Snapshot Retention @@ -137,8 +137,12 @@ Before full uninstall deletes shared state, it removes only an exact authenticat Catalog-selected host-local vLLM also requires its exact owner-only runtime receipt; legacy authenticated containers without catalog profile labels retain their existing cleanup behavior. It refuses the remaining uninstall steps when Docker is unavailable or persisted ownership cannot be proved. Resolve the reported Docker or ownership error and rerun uninstall with the state still present. -The shared Hugging Face cache used by managed vLLM and managed llama.cpp remains in place. -The `--delete-models` flag removes NemoClaw-pulled Ollama models, not the shared Hugging Face cache. +The shared Hugging Face cache used by managed vLLM and managed llama.cpp remains in place by default. +With `--delete-models`, full uninstall deletes every model in the local Ollama inventory and all non-credential data in the current user's shared Hugging Face cache. +The flag can delete cached files that other applications installed or use. +It preserves the Hugging Face `token` and `stored_tokens` authentication files. +NemoClaw deletes the cache only after managed model runtimes stop and only when no sibling gateway remains. +An Ollama inventory error, model deletion error, unsafe cache path, or cache-data deletion error makes uninstall exit nonzero. For operational uninstall steps, refer to: diff --git a/scripts/smoke-macos-install.sh b/scripts/smoke-macos-install.sh index a85217269f5..b8877a936a1 100755 --- a/scripts/smoke-macos-install.sh +++ b/scripts/smoke-macos-install.sh @@ -59,7 +59,7 @@ Options: --allow-existing-state Allow running even if NemoClaw/OpenShell state already exists --keep-logs Preserve log files after success --remove-openshell Allow uninstall.sh to remove openshell - --delete-models Allow uninstall.sh to delete Ollama models + --delete-models Allow uninstall.sh to delete all Ollama models and Hugging Face cache data -h, --help Show this help Environment: diff --git a/src/commands/internal/uninstall/plan.ts b/src/commands/internal/uninstall/plan.ts index ad94012087b..e9d8c11a630 100644 --- a/src/commands/internal/uninstall/plan.ts +++ b/src/commands/internal/uninstall/plan.ts @@ -2,12 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 import { Flags } from "@oclif/core"; +import { buildHostUninstallPlan } from "../../../lib/actions/uninstall/plan"; import { CLI_DISPLAY_NAME, CLI_NAME } from "../../../lib/cli/branding"; import { jsonFlag } from "../../../lib/cli/common-flags"; import { NemoClawCommand } from "../../../lib/cli/nemoclaw-oclif-command"; -import { buildHostUninstallPlan } from "../../../lib/actions/uninstall/plan"; - export default class InternalUninstallPlanCommand extends NemoClawCommand { static hidden = true; static strict = true; @@ -21,7 +20,7 @@ export default class InternalUninstallPlanCommand extends NemoClawCommand { description: "Accepted for parity with run-plan; ignored while planning", }), "delete-models": Flags.boolean({ - description: `Plan removal of ${CLI_DISPLAY_NAME}-pulled Ollama models`, + description: "Plan removal of all Ollama models and Hugging Face cache data", }), "keep-openshell": Flags.boolean({ description: "Keep the openshell binary installed" }), gateway: Flags.string({ description: "Gateway name", default: "nemoclaw" }), diff --git a/src/commands/internal/uninstall/run-plan.ts b/src/commands/internal/uninstall/run-plan.ts index d3237ecc13a..889a892e476 100644 --- a/src/commands/internal/uninstall/run-plan.ts +++ b/src/commands/internal/uninstall/run-plan.ts @@ -30,7 +30,7 @@ export default class InternalUninstallRunPlanCommand extends NemoClawCommand { "all-gateway-ports-child": Flags.boolean({ hidden: true }), "keep-openshell": Flags.boolean({ description: "Leave the openshell binary installed" }), "delete-models": Flags.boolean({ - description: `Remove ${CLI_DISPLAY_NAME}-pulled Ollama models`, + description: "Remove all Ollama models and Hugging Face cache data", }), "destroy-user-data": Flags.boolean({ description: diff --git a/src/lib/actions/root-help.ts b/src/lib/actions/root-help.ts index 90e1b9d258e..3645400c8fc 100644 --- a/src/lib/actions/root-help.ts +++ b/src/lib/actions/root-help.ts @@ -81,7 +81,9 @@ export function help(): void { lines.push(` ${G}Uninstall flags:${R}`); lines.push(` --yes${" ".repeat(29)}Skip the confirmation prompt`); lines.push(` --keep-openshell${" ".repeat(18)}Leave the openshell binary installed`); - lines.push(` --delete-models${" ".repeat(19)}Remove ${CLI_DISPLAY_NAME}-pulled Ollama models`); + lines.push( + ` --delete-models${" ".repeat(19)}Remove all Ollama models and Hugging Face cache data`, + ); lines.push( ` --destroy-user-data${" ".repeat(15)}Remove preserved ~/.nemoclaw/ user data (rebuild-backups/, backups/, sandboxes.json)`, ); diff --git a/src/lib/actions/uninstall/run-plan-dual-station.test.ts b/src/lib/actions/uninstall/run-plan-dual-station.test.ts index 352ec597809..8cda16b1406 100644 --- a/src/lib/actions/uninstall/run-plan-dual-station.test.ts +++ b/src/lib/actions/uninstall/run-plan-dual-station.test.ts @@ -7,6 +7,8 @@ import path from "node:path"; import { describe, expect, it, vi } from "vitest"; +import { DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE } from "../../inference/serving/managed-runtime-receipts"; + import { type RunResult, runUninstallPlan as runUninstallPlanBase, @@ -90,6 +92,48 @@ describe("managed distributed vLLM runtime uninstall", () => { } }); + it("stops a distributed runtime before deleting the shared Hugging Face cache", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-cache-order-")); + const stateDir = path.join(home, ".nemoclaw"); + const receiptPath = path.join(stateDir, DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE); + const cacheDir = path.join(home, ".cache", "huggingface"); + fs.mkdirSync(stateDir, { mode: 0o700 }); + fs.writeFileSync(receiptPath, "{}\n", { mode: 0o600 }); + fs.mkdirSync(`${receiptPath}.ssh-binding`, { mode: 0o700 }); + fs.mkdirSync(cacheDir, { recursive: true }); + const runDualStationRuntimeCleanup = vi.fn(() => ok()); + const runLocalModelRuntimeCleanup = vi.fn(() => ok()); + + try { + const result = runUninstallPlan( + { assumeYes: true, deleteModels: true, keepOpenShell: true }, + { + commandExists: (command) => command === "openshell", + env: { HOME: home, TMPDIR: home } as NodeJS.ProcessEnv, + existsSync: fs.existsSync, + isTty: false, + log: vi.fn(), + rmSync: vi.fn(), + run: okWithKnownGatewayList, + runDualStationRuntimeCleanup, + runLocalModelRuntimeCleanup, + }, + ); + + expect(result.exitCode).toBe(0); + expect(runDualStationRuntimeCleanup).toHaveBeenCalledOnce(); + expect(runLocalModelRuntimeCleanup).toHaveBeenCalledWith( + true, + expect.objectContaining({ stdio: "inherit" }), + ); + expect(runDualStationRuntimeCleanup.mock.invocationCallOrder[0]).toBeLessThan( + runLocalModelRuntimeCleanup.mock.invocationCallOrder[0], + ); + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } + }); + it("finds host-global pair ownership when the final gateway uses a non-default port", async () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-dual-port-")); const port = 9123; diff --git a/src/lib/actions/uninstall/run-plan-local-model-profile.test.ts b/src/lib/actions/uninstall/run-plan-local-model-profile.test.ts index 2620fbf1eac..2a4417e2501 100644 --- a/src/lib/actions/uninstall/run-plan-local-model-profile.test.ts +++ b/src/lib/actions/uninstall/run-plan-local-model-profile.test.ts @@ -133,27 +133,30 @@ describe("uninstall local model profile cleanup", () => { expect(errors.join("\n")).toContain("could not inventory reserved managed inference"); }); - it("states that uninstall preserves the shared Hugging Face cache", () => { + it("states that model deletion removes every Ollama model and the Hugging Face cache", () => { const logs: string[] = []; const result = runUninstallPlan( { assumeYes: false, deleteModels: true, keepOpenShell: true }, { - commandExists: () => false, + commandExists: (command) => command === "openshell", env: { HOME: "/tmp/nemoclaw-uninstall-model-confirmation" } as NodeJS.ProcessEnv, existsSync: () => false, isTty: true, log: (line) => logs.push(line), readLine: () => "no", - run: vi.fn(), + run: vi.fn(okWithKnownGatewayList), }, ); expect(result.exitCode).toBe(0); - expect(logs).toContain(" · Shared Hugging Face model cache: kept"); + expect(logs).toContain(" · All installed Ollama models"); + expect(logs).toContain( + " · Shared Hugging Face cache data: deleted; authentication files kept", + ); expect(logs).toContain("Aborted."); }); - it("does not run managed cleanup for a shared Hugging Face cache without runtime state", () => { + it("removes a shared Hugging Face cache even when no managed runtime state remains", () => { const tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-llama-cache-")); const cacheDir = path.join(tmpHome, ".cache", "huggingface"); const runLocalModelRuntimeCleanup = vi.fn(() => ok()); @@ -172,12 +175,205 @@ describe("uninstall local model profile cleanup", () => { ); expect(result.exitCode).toBe(0); - expect(runLocalModelRuntimeCleanup).not.toHaveBeenCalled(); + expect(runLocalModelRuntimeCleanup).toHaveBeenCalledWith( + true, + expect.objectContaining({ stdio: "inherit" }), + ); } finally { fs.rmSync(tmpHome, { recursive: true, force: true }); } }); + it("deletes every model returned by Ollama inventory", () => { + const run = vi.fn((command: string, args: string[], _options?: { env?: NodeJS.ProcessEnv }) => { + if (command === "openshell" && args[0] === "gateway" && args[1] === "list") { + return ok(JSON.stringify([{ name: "nemoclaw" }])); + } + if (command === "ollama" && args[0] === "list") { + return ok( + [ + "NAME ID SIZE MODIFIED", + "team/first:latest 111111111111 5 GB 1 hour ago", + "second:q4 222222222222 3 GB 2 hours ago", + ].join("\n"), + ); + } + return ok(); + }); + + const result = runUninstallPlan( + { assumeYes: true, deleteModels: true, keepOpenShell: true }, + { + commandExists: (command) => command === "openshell" || command === "ollama", + env: { HOME: "/tmp/nemoclaw-uninstall-all-ollama-models" } as NodeJS.ProcessEnv, + existsSync: () => false, + isTty: false, + log: () => {}, + run, + }, + ); + + expect(result.exitCode).toBe(0); + expect(run).toHaveBeenCalledWith( + "ollama", + ["list"], + expect.objectContaining({ + env: expect.objectContaining({ OLLAMA_HOST: "127.0.0.1:11434" }), + timeout: 10_000, + }), + ); + expect( + run.mock.calls + .filter(([command, args]) => command === "ollama" && args[0] === "rm") + .map(([, args]) => args[1]), + ).toEqual(["team/first:latest", "second:q4"]); + }); + + it("ignores a remote Ollama environment override during model cleanup", () => { + const run = vi.fn((command: string, args: string[], _options?: { env?: NodeJS.ProcessEnv }) => { + if (command === "openshell" && args[0] === "gateway" && args[1] === "list") { + return ok(JSON.stringify([{ name: "nemoclaw" }])); + } + if (command === "ollama" && args[0] === "list") { + return ok("NAME ID SIZE MODIFIED\nlocal-model 111 1 GB now\n"); + } + return ok(); + }); + + const result = runUninstallPlan( + { assumeYes: true, deleteModels: true, keepOpenShell: true }, + { + commandExists: (command) => command === "openshell" || command === "ollama", + env: { + HOME: "/tmp/nemoclaw-uninstall-local-ollama-only", + OLLAMA_HOST: "https://remote.example.test:11434", + } as NodeJS.ProcessEnv, + existsSync: () => false, + isTty: false, + log: () => {}, + run, + }, + ); + + expect(result.exitCode).toBe(0); + for (const [command, , options] of run.mock.calls.filter(([command]) => command === "ollama")) { + expect(command).toBe("ollama"); + expect(options?.env?.OLLAMA_HOST).toBe("127.0.0.1:11434"); + } + }); + + it("fails without deleting any Ollama model when inventory is malformed", () => { + const errors: string[] = []; + const run = vi.fn((command: string, args: string[]) => { + if (command === "openshell" && args[0] === "gateway" && args[1] === "list") { + return ok(JSON.stringify([{ name: "nemoclaw" }])); + } + if (command === "ollama" && args[0] === "list") { + return ok("NAME ID SIZE MODIFIED\n--unsafe 111111111111 1 GB now\n"); + } + return ok(); + }); + + const result = runUninstallPlan( + { assumeYes: true, deleteModels: true, keepOpenShell: true }, + { + commandExists: (command) => command === "openshell" || command === "ollama", + env: { HOME: "/tmp/nemoclaw-uninstall-malformed-ollama" } as NodeJS.ProcessEnv, + error: (message) => errors.push(message), + existsSync: () => false, + isTty: false, + log: () => {}, + run, + }, + ); + + expect(result.exitCode).toBe(1); + expect(run.mock.calls.some(([command, args]) => command === "ollama" && args[0] === "rm")).toBe( + false, + ); + expect(errors.join("\n")).toContain("No Ollama models were removed"); + }); + + it("fails without deleting any Ollama model when inventory execution fails", () => { + const run = vi.fn((command: string, args: string[]) => { + if (command === "openshell" && args[0] === "gateway" && args[1] === "list") { + return ok(JSON.stringify([{ name: "nemoclaw" }])); + } + if (command === "ollama" && args[0] === "list") { + return { status: 1, stdout: "", stderr: "daemon unavailable" }; + } + return ok(); + }); + + const result = runUninstallPlan( + { assumeYes: true, deleteModels: true, keepOpenShell: true }, + { + commandExists: (command) => command === "openshell" || command === "ollama", + env: { HOME: "/tmp/nemoclaw-uninstall-failed-ollama-inventory" } as NodeJS.ProcessEnv, + existsSync: () => false, + isTty: false, + log: () => {}, + run, + }, + ); + + expect(result.exitCode).toBe(1); + expect(run.mock.calls.some(([command, args]) => command === "ollama" && args[0] === "rm")).toBe( + false, + ); + }); + + it("attempts every inventoried Ollama removal and fails when one removal fails", () => { + const run = vi.fn((command: string, args: string[]) => { + if (command === "openshell" && args[0] === "gateway" && args[1] === "list") { + return ok(JSON.stringify([{ name: "nemoclaw" }])); + } + if (command === "ollama" && args[0] === "list") { + return ok("NAME ID SIZE MODIFIED\nfirst 111 1 GB now\nsecond 222 1 GB now\n"); + } + if (command === "ollama" && args[0] === "rm" && args[1] === "first") return notFound(); + return ok(); + }); + + const result = runUninstallPlan( + { assumeYes: true, deleteModels: true, keepOpenShell: true }, + { + commandExists: (command) => command === "openshell" || command === "ollama", + env: { HOME: "/tmp/nemoclaw-uninstall-partial-ollama" } as NodeJS.ProcessEnv, + existsSync: () => false, + isTty: false, + log: () => {}, + run, + }, + ); + + expect(result.exitCode).toBe(1); + expect( + run.mock.calls + .filter(([command, args]) => command === "ollama" && args[0] === "rm") + .map(([, args]) => args[1]), + ).toEqual(["first", "second"]); + }); + + it("does not inventory or remove Ollama models without delete-models", () => { + const run = vi.fn(okWithKnownGatewayList); + + const result = runUninstallPlan( + { assumeYes: true, deleteModels: false, keepOpenShell: true }, + { + commandExists: (command) => command === "openshell" || command === "ollama", + env: { HOME: "/tmp/nemoclaw-uninstall-keep-models" } as NodeJS.ProcessEnv, + existsSync: () => false, + isTty: false, + log: () => {}, + run, + }, + ); + + expect(result.exitCode).toBe(0); + expect(run.mock.calls.some(([command]) => command === "ollama")).toBe(false); + }); + it("cleans selected gateway-owned llama.cpp state before scoped uninstall removes state", () => { const tmpHome = fs.realpathSync( fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-llama-scoped-")), @@ -304,7 +500,7 @@ describe("uninstall local model profile cleanup", () => { ); expect(result.exitCode).toBe(1); - expect(errors.join("\n")).toContain("Host-local model cleanup did not complete"); + expect(errors.join("\n")).toContain("Host-local model and cache cleanup did not complete"); expect(runDocker.mock.calls.some(([args]) => args[0] === "rm")).toBe(false); }); }); diff --git a/src/lib/actions/uninstall/run-plan.ts b/src/lib/actions/uninstall/run-plan.ts index e5808f3940b..744ab9e10a4 100644 --- a/src/lib/actions/uninstall/run-plan.ts +++ b/src/lib/actions/uninstall/run-plan.ts @@ -24,7 +24,6 @@ import { } from "../../domain/uninstall/messaging"; import { defaultUninstallPaths, - NEMOCLAW_OLLAMA_MODELS, NEMOCLAW_PROVIDERS, type UninstallPaths, } from "../../domain/uninstall/paths"; @@ -592,12 +591,19 @@ function confirm( runtime.log(userDataDispositionLine(options, runtime, paths)); runtime.log(" · ~/.config/openshell ~/.config/nemoclaw"); runtime.log(` · Global ${branding.display} CLI (npm package: nemoclaw)`); - runtime.log( - options.deleteModels - ? ` · Ollama models: ${NEMOCLAW_OLLAMA_MODELS.join(" ")}` - : " · Ollama models: kept", - ); - runtime.log(" · Shared Hugging Face model cache: kept"); + if (scopedToSelectedGateway) { + runtime.log(" · Ollama models: kept while sibling gateways remain"); + runtime.log(" · Shared Hugging Face model cache: kept while sibling gateways remain"); + } else { + runtime.log( + options.deleteModels ? " · All installed Ollama models" : " · Ollama models: kept", + ); + runtime.log( + options.deleteModels + ? " · Shared Hugging Face cache data: deleted; authentication files kept" + : " · Shared Hugging Face model cache: kept", + ); + } runtime.log("Proceed? [y/N]"); const reply = runtime.readLine(); if (reply && /^(y|yes)$/i.test(reply.trim())) return true; @@ -1428,6 +1434,7 @@ function managedDistributedVllmStateRootStatus( function removeManagedDistributedVllmRuntime( paths: UninstallPaths, runtime: UninstallRuntime, + preserveApiKeyWithoutReceipt = false, ): boolean { const rootStatus = managedDistributedVllmStateRootStatus(paths, runtime); if (rootStatus !== "directory") return rootStatus === "absent"; @@ -1475,7 +1482,7 @@ function removeManagedDistributedVllmRuntime( return false; } if (receipts.length === 0) { - removePath(apiKeyPath, runtime); + if (!preserveApiKeyWithoutReceipt) removePath(apiKeyPath, runtime); return true; } if (state.managedClusterPath && state.stationPaths.length > 0) { @@ -1516,7 +1523,12 @@ function removeHostLocalModelRuntimes( MANAGED_CLUSTER_VLLM_RUNTIME_RECEIPT_FILE, DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE, ].some((name) => runtime.existsSync(path.join(sharedRoot, name))); - if (!hasLlamaState && (!hasManagedKey || hasDistributedReceipt)) { + const hasSharedModelCache = runtime.existsSync(paths.huggingFaceModelCacheDir); + if ( + !hasLlamaState && + (!hasManagedKey || hasDistributedReceipt) && + (!deleteModels || !hasSharedModelCache) + ) { return true; } const result = runtime.runLocalModelRuntimeCleanup(deleteModels, { @@ -1525,7 +1537,7 @@ function removeHostLocalModelRuntimes( }); if (result.status === 0) return true; runtime.error( - "Host-local model cleanup did not complete. NemoClaw did not start the remaining uninstall steps. Resolve the reported ownership or Docker error and retry uninstall.", + "Host-local model and cache cleanup did not complete. NemoClaw did not start the remaining uninstall steps. Resolve the reported ownership, path, or Docker error and retry uninstall.", ); return false; } @@ -1590,8 +1602,16 @@ function removeManagedModelRuntimes( ): boolean { if (!removeManagedLlamaCppRuntimes(runtime, scopedToSelectedGateway)) return false; if (scopedToSelectedGateway) return true; + const sharedRoot = path.dirname(paths.managedSwapMarkerPath); + const hasDistributedReceipt = [ + MANAGED_CLUSTER_VLLM_RUNTIME_RECEIPT_FILE, + DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE, + ].some((name) => runtime.existsSync(path.join(sharedRoot, name))); + if (!removeManagedDistributedVllmRuntime(paths, runtime, !hasDistributedReceipt)) return false; if (!removeHostLocalModelRuntimes(paths, deleteModels, runtime)) return false; - if (!removeManagedDistributedVllmRuntime(paths, runtime)) return false; + if (!hasDistributedReceipt) { + removePath(path.join(sharedRoot, MANAGED_VLLM_API_KEY_FILE), runtime); + } if (!runtime.commandExists("docker")) return true; const inventory = runtime.runDocker(["ps", "-a", "--format", "{{.Names}}"], { env: runtime.env, @@ -1691,20 +1711,92 @@ function removeDockerVolume(name: string, runtime: UninstallRuntime): void { else runtime.warn(`Failed to remove Docker volume ${name}`); } -function removeOllamaModels(options: UninstallRunOptions, runtime: UninstallRuntime): void { +function parseOllamaModelInventory(output: string): string[] { + const rows = output + .split(/\r?\n/u) + .map((row) => row.trim()) + .filter(Boolean); + const header = rows.shift()?.split(/\s+/u) ?? []; + if (header.length < 2 || header[0] !== "NAME" || header[1] !== "ID") { + throw new Error("Ollama model inventory did not contain the expected NAME and ID columns"); + } + const models = new Set(); + for (const row of rows) { + const columns = row.split(/\s+/u); + const model = columns[0] ?? ""; + if ( + columns.length < 2 || + model.length === 0 || + model.length > 512 || + model.startsWith("-") || + /[\u0000-\u001f\u007f]/u.test(model) + ) { + throw new Error("Ollama model inventory contained an unsafe or malformed model name"); + } + models.add(model); + } + return [...models]; +} + +function localOllamaEnvironment(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + return { ...env, OLLAMA_HOST: "127.0.0.1:11434" }; +} + +function removeOllamaModels(options: UninstallRunOptions, runtime: UninstallRuntime): boolean { if (!options.deleteModels) { runtime.log("Keeping Ollama models as requested."); - return; + return true; } if (!runtime.commandExists("ollama")) { - runtime.warn("ollama not found; skipping model cleanup."); - return; + runtime.log("Ollama is not installed; no Ollama model inventory is available to remove."); + return true; } - for (const model of NEMOCLAW_OLLAMA_MODELS) { - if (runtime.run("ollama", ["rm", model], { env: runtime.env, stdio: "ignore" }).status === 0) + const ollamaEnv = localOllamaEnvironment(runtime.env); + const inventory = runtime.run("ollama", ["list"], { + env: ollamaEnv, + timeout: 10_000, + }); + if (inventory.status !== 0) { + runtime.error( + `Ollama model inventory failed${inventory.stderr.trim() ? `: ${inventory.stderr.trim()}` : "."}`, + ); + return false; + } + let models: string[]; + try { + models = parseOllamaModelInventory(inventory.stdout); + } catch (error) { + runtime.error(`${formatError(error)}. No Ollama models were removed.`); + return false; + } + if (models.length === 0) { + runtime.log("No installed Ollama models found."); + return true; + } + let ok = true; + for (const model of models) { + if (runtime.run("ollama", ["rm", model], { env: ollamaEnv, stdio: "ignore" }).status === 0) runtime.log(`Removed Ollama model '${model}'`); - else runtime.warn(`Ollama model '${model}' not found or already removed`); + else { + runtime.error(`Failed to remove Ollama model '${model}'`); + ok = false; + } } + return ok; +} + +function removeHostModelStores( + options: UninstallRunOptions, + runtime: UninstallRuntime, + scopedToSelectedGateway: boolean, +): boolean { + if (scopedToSelectedGateway) { + runtime.log( + "Sibling gateways remain; kept host-shared Ollama models and the Hugging Face model cache.", + ); + return true; + } + return removeOllamaModels(options, runtime); } interface OtherGatewayInspection { @@ -2190,12 +2282,8 @@ function executePlan( for (const action of step.actions) if (action.kind === "delete-docker-volume") removeDockerVolume(action.name, runtime); } - } else if (step.name === "Ollama models") { - if (scopedToSelectedGateway) { - runtime.log("Sibling gateways remain; kept host-shared Ollama models."); - } else { - removeOllamaModels(options, runtime); - } + } else if (step.name === "Model stores") { + if (!removeHostModelStores(options, runtime, scopedToSelectedGateway)) ok = false; } else if (step.name === "State and binaries") { removeManagedSwap(paths, runtime, scopedToSelectedGateway); if (!scopedToSelectedGateway) { diff --git a/src/lib/domain/uninstall/paths.test.ts b/src/lib/domain/uninstall/paths.test.ts index 77b371e026b..e8cb2770fad 100644 --- a/src/lib/domain/uninstall/paths.test.ts +++ b/src/lib/domain/uninstall/paths.test.ts @@ -32,6 +32,7 @@ describe("uninstall paths", () => { ...OPENSHELL_MANAGED_BINARIES.map((binary) => path.join("/xdg/bin", binary)), ]); expect(paths.helperServiceGlob).toBe(path.join("/tmp/nemo", "nemoclaw-services-*")); + expect(paths.huggingFaceModelCacheDir).toBe(path.join("/home/test", ".cache", "huggingface")); expect(paths.runtimeTempGlobs).toEqual([ path.join("/tmp/nemo", "nemoclaw-create-*.log"), path.join("/tmp/nemo", "nemoclaw-tg-ssh-*.conf"), diff --git a/src/lib/domain/uninstall/paths.ts b/src/lib/domain/uninstall/paths.ts index 37ef50e095f..f847395173b 100644 --- a/src/lib/domain/uninstall/paths.ts +++ b/src/lib/domain/uninstall/paths.ts @@ -15,7 +15,6 @@ export const NEMOCLAW_PROVIDERS = [ "nvidia-ncp", "nim-local", ] as const; -export const NEMOCLAW_OLLAMA_MODELS = ["nemotron-3-super:120b", "nemotron-3-nano:30b"] as const; export const OPENSHELL_MANAGED_BINARIES = [ "openshell", "openshell-gateway", @@ -35,6 +34,7 @@ export const AGENT_ALIAS_CLI_BINARIES = ["nemohermes", "nemo-deepagents"] as con export interface UninstallPaths { helperServiceGlob: string; + huggingFaceModelCacheDir: string; managedSwapMarkerPath: string; nemoclawConfigDir: string; nemoclawShimPath: string; @@ -67,6 +67,7 @@ export function defaultUninstallPaths(options: UninstallPathOptions): UninstallP const gatewayLocalStateDir = path.join(options.home, ".local", "state", "nemoclaw"); return { helperServiceGlob: path.join(tmpDir, "nemoclaw-services-*"), + huggingFaceModelCacheDir: path.join(options.home, ".cache", "huggingface"), managedSwapMarkerPath: path.join(options.home, ".nemoclaw", "managed_swap"), nemoclawConfigDir: path.join(options.home, ".config", "nemoclaw"), nemoclawShimPath: path.join(options.home, ".local", "bin", "nemoclaw"), diff --git a/src/lib/domain/uninstall/plan.test.ts b/src/lib/domain/uninstall/plan.test.ts index b7a18f74bc3..6f60bf1b17c 100644 --- a/src/lib/domain/uninstall/plan.test.ts +++ b/src/lib/domain/uninstall/plan.test.ts @@ -1,9 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; - import path from "node:path"; +import { describe, expect, it } from "vitest"; import { defaultUninstallPaths, OPENSHELL_MANAGED_BINARIES } from "./paths"; import { buildUninstallPlan, flattenUninstallPlan } from "./plan"; @@ -22,7 +21,7 @@ describe("uninstall plan", () => { "OpenShell resources", "NemoClaw CLI", "Docker resources", - "Ollama models", + "Model stores", "State and binaries", ]); expect(flattenUninstallPlan(plan)).toEqual( @@ -33,7 +32,11 @@ describe("uninstall plan", () => { { kind: "delete-related-docker-containers" }, { kind: "delete-related-docker-images" }, { kind: "delete-docker-volume", name: "openshell-cluster-nemoclaw" }, - { kind: "preserve-ollama-models", names: ["nemotron-3-super:120b", "nemotron-3-nano:30b"] }, + { kind: "preserve-ollama-models" }, + { + kind: "preserve-hugging-face-cache-data", + path: path.join("/home/test", ".cache", "huggingface"), + }, { kind: "delete-managed-swap" }, ...OPENSHELL_MANAGED_BINARIES.map((binary) => ({ kind: "delete-openshell-install-path" as const, @@ -73,8 +76,14 @@ describe("uninstall plan", () => { expect(actions).toEqual( expect.arrayContaining([{ kind: "delete-docker-volume", name: "openshell-cluster-custom" }]), ); + expect(actions).toEqual(expect.arrayContaining([{ kind: "delete-all-ollama-models" }])); expect(actions).toEqual( - expect.arrayContaining([{ kind: "delete-ollama-model", name: "nemotron-3-super:120b" }]), + expect.arrayContaining([ + { + kind: "delete-hugging-face-cache-data", + path: path.join("/home/test", ".cache", "huggingface"), + }, + ]), ); expect(actions).toEqual( expect.arrayContaining([ diff --git a/src/lib/domain/uninstall/plan.ts b/src/lib/domain/uninstall/plan.ts index 758be879f50..933b7900338 100644 --- a/src/lib/domain/uninstall/plan.ts +++ b/src/lib/domain/uninstall/plan.ts @@ -4,7 +4,6 @@ import { DEFAULT_GATEWAY_NAME, gatewayVolumeCandidates, - NEMOCLAW_OLLAMA_MODELS, NEMOCLAW_PROVIDERS, type UninstallPaths, uninstallStatePaths, @@ -20,8 +19,9 @@ export interface UninstallPlanOptions { export type UninstallPlanAction = | { kind: "delete-docker-volume"; name: string } + | { kind: "delete-all-ollama-models" } + | { kind: "delete-hugging-face-cache-data"; path: string } | { kind: "delete-managed-swap" } - | { kind: "delete-ollama-model"; name: string } | { kind: "delete-related-docker-containers" } | { kind: "delete-related-docker-images" } | { kind: "delete-openshell-install-path"; path: string } @@ -30,7 +30,8 @@ export type UninstallPlanAction = | { kind: "delete-runtime-glob"; pattern: string } | { kind: "delete-shim"; reason: string } | { kind: "destroy-openshell-gateway"; name: string } - | { kind: "preserve-ollama-models"; names: string[] } + | { kind: "preserve-hugging-face-cache-data"; path: string } + | { kind: "preserve-ollama-models" } | { kind: "preserve-openshell-install-paths"; paths: string[] } | { kind: "preserve-shim"; reason: string } | { kind: "stop-helper-services" } @@ -106,10 +107,22 @@ export function buildUninstallPlan( ], }, { - name: "Ollama models", + name: "Model stores", actions: options.deleteModels - ? NEMOCLAW_OLLAMA_MODELS.map((name) => ({ kind: "delete-ollama-model" as const, name })) - : [{ kind: "preserve-ollama-models", names: [...NEMOCLAW_OLLAMA_MODELS] }], + ? [ + { kind: "delete-all-ollama-models" }, + { + kind: "delete-hugging-face-cache-data", + path: paths.huggingFaceModelCacheDir, + }, + ] + : [ + { kind: "preserve-ollama-models" }, + { + kind: "preserve-hugging-face-cache-data", + path: paths.huggingFaceModelCacheDir, + }, + ], }, { name: "State and binaries", diff --git a/src/lib/inference/local-model-profile/cleanup-path-safety.test.ts b/src/lib/inference/local-model-profile/cleanup-path-safety.test.ts index f99068091a9..780b5d4de15 100644 --- a/src/lib/inference/local-model-profile/cleanup-path-safety.test.ts +++ b/src/lib/inference/local-model-profile/cleanup-path-safety.test.ts @@ -35,6 +35,62 @@ afterEach(() => { }); describe("host-local model cleanup path safety", () => { + it.skipIf(process.platform === "win32")( + "fails closed when the Hugging Face model cache is a symlink", + () => { + const homeDir = home(); + const cacheParent = path.join(homeDir, ".cache"); + const target = path.join(homeDir, "substituted-cache"); + fs.mkdirSync(cacheParent); + fs.mkdirSync(target); + fs.symlinkSync(target, path.join(cacheParent, "huggingface"), "dir"); + + expect(cleanupLocalModelRuntimes({ deleteModels: true, homeDir })).toMatchObject({ + ok: false, + reason: expect.stringContaining("model cache is a symlink"), + }); + expect(fs.existsSync(target)).toBe(true); + }, + ); + + it.skipIf(typeof process.getuid !== "function")( + "fails closed when the Hugging Face model cache has an unexpected owner", + () => { + const homeDir = home(); + const cache = path.join(homeDir, ".cache", "huggingface"); + fs.mkdirSync(cache, { recursive: true }); + const observedOwner = fs.lstatSync(cache).uid; + + expect( + cleanupLocalModelRuntimes({ + deleteModels: true, + homeDir, + deps: dockerDeps({ currentUserId: observedOwner + 1 }), + }), + ).toMatchObject({ + ok: false, + reason: expect.stringContaining("cache parent is not owned by the current user"), + }); + expect(fs.existsSync(cache)).toBe(true); + }, + ); + + it.skipIf(process.platform === "win32")( + "fails closed when the Hugging Face model cache is group-writable", + () => { + const homeDir = home(); + const cache = path.join(homeDir, ".cache", "huggingface"); + fs.mkdirSync(cache, { recursive: true }); + fs.chmodSync(cache, 0o770); + + expect(cleanupLocalModelRuntimes({ deleteModels: true, homeDir })).toMatchObject({ + ok: false, + reason: expect.stringContaining("model cache is not current-user filesystem authority"), + }); + expect(fs.existsSync(cache)).toBe(true); + }, + ); + it.skipIf(process.platform === "win32")( "fails closed when managed llama.cpp state is a symlink", () => { diff --git a/src/lib/inference/local-model-profile/cleanup.test.ts b/src/lib/inference/local-model-profile/cleanup.test.ts index 4602ebb325e..694771a641c 100644 --- a/src/lib/inference/local-model-profile/cleanup.test.ts +++ b/src/lib/inference/local-model-profile/cleanup.test.ts @@ -358,7 +358,9 @@ describe("host-local model cleanup", () => { createManagedState(homeDir, harness.engine); const cache = path.join(homeDir, ".cache", "huggingface"); fs.mkdirSync(cache, { recursive: true }); - fs.writeFileSync(path.join(cache, "shared-model"), "keep"); + fs.writeFileSync(path.join(cache, "shared-model"), "delete"); + fs.writeFileSync(path.join(cache, "token"), "keep-credential", { mode: 0o600 }); + fs.writeFileSync(path.join(cache, "stored_tokens"), "keep-stored-credentials", { mode: 0o600 }); const ambientCapture = vi.fn(() => "") as never; const ambientForceRm = vi.fn(() => ({ status: 0 })) as never; const ambientRun = vi.fn(() => ({ status: 0 })) as never; @@ -379,8 +381,29 @@ describe("host-local model cleanup", () => { expect(ambientCapture).not.toHaveBeenCalled(); expect(ambientForceRm).not.toHaveBeenCalled(); expect(ambientRun).not.toHaveBeenCalled(); + expect(fs.existsSync(path.join(cache, "shared-model"))).toBe(false); + expect(fs.readFileSync(path.join(cache, "token"), "utf8")).toBe("keep-credential"); + expect(fs.readFileSync(path.join(cache, "stored_tokens"), "utf8")).toBe( + "keep-stored-credentials", + ); + expect(result.removed).toContain(`cache-contents:${cache}`); + expect(result.preserved).toEqual( + expect.arrayContaining([path.join(cache, "token"), path.join(cache, "stored_tokens")]), + ); + }); + + it("preserves the shared Hugging Face cache unless model deletion is requested", () => { + const homeDir = temporaryHome(); + const cache = path.join(homeDir, ".cache", "huggingface"); + fs.mkdirSync(cache, { recursive: true }); + fs.writeFileSync(path.join(cache, "shared-model"), "keep"); + + const result = cleanupLocalModelRuntimes({ deleteModels: false, homeDir }); + + expect(result).toMatchObject({ ok: true }); expect(fs.existsSync(path.join(cache, "shared-model"))).toBe(true); expect(result.preserved).toContain(cache); + expect(result.removed).not.toContain(cache); }); it("canonicalizes a symlink HOME alias before exact managed llama.cpp cleanup", () => { diff --git a/src/lib/inference/local-model-profile/cleanup.ts b/src/lib/inference/local-model-profile/cleanup.ts index c00ffc5ba16..98b4d6e31c4 100644 --- a/src/lib/inference/local-model-profile/cleanup.ts +++ b/src/lib/inference/local-model-profile/cleanup.ts @@ -53,6 +53,7 @@ const LLAMA_SPEC_LABEL = "io.nvidia.nemoclaw.host-local-inference.spec-sha256"; const LLAMA_TRANSACTION_LABEL = "io.nvidia.nemoclaw.host-local-inference.transaction-sha256"; const LLAMA_NETWORK_TRANSACTION_LABEL = "io.nvidia.nemoclaw.host-local-inference.network-transaction-sha256"; +const HUGGING_FACE_CREDENTIAL_ENTRIES = new Set(["stored_tokens", "token"]); interface CleanupDeps { capture: typeof dockerCapture; @@ -208,6 +209,71 @@ function canonicalCleanupHomeDir(homeDir: string): string { return statePathExists(homeDir) ? fs.realpathSync(homeDir) : path.resolve(homeDir); } +function sharedHuggingFaceCacheDir(homeDir: string): string { + return path.join(homeDir, ".cache", "huggingface"); +} + +function requireCurrentUserCacheDirectory( + directory: string, + label: string, + currentUserId: number | null, +): void { + if (currentUserId === null) { + throw new Error(`${label} ownership cannot be verified on this host`); + } + const status = fs.lstatSync(directory); + if (status.isSymbolicLink()) throw new Error(`${label} is a symlink`); + if (!status.isDirectory()) throw new Error(`${label} is not a directory`); + if (status.uid !== currentUserId) { + throw new Error(`${label} is not owned by the current user`); + } + const expectedPath = path.join( + fs.realpathSync(path.dirname(directory)), + path.basename(directory), + ); + if ((status.mode & 0o022) !== 0 || fs.realpathSync(directory) !== expectedPath) { + throw new Error(`${label} is not current-user filesystem authority`); + } +} + +function removeSharedHuggingFaceCacheData( + homeDir: string, + currentUserId: number | null, + removed: string[], + preserved: string[], +): void { + const cacheDir = sharedHuggingFaceCacheDir(homeDir); + if (!statePathExists(cacheDir)) return; + const cacheParent = path.dirname(cacheDir); + requireCurrentUserCacheDirectory(cacheParent, "Hugging Face cache parent", currentUserId); + requireCurrentUserCacheDirectory(cacheDir, "Hugging Face model cache", currentUserId); + const entries = fs.readdirSync(cacheDir); + let deletedCacheData = false; + for (const entry of entries) { + const target = path.join(cacheDir, entry); + if (HUGGING_FACE_CREDENTIAL_ENTRIES.has(entry)) { + preserved.push(target); + continue; + } + fs.rmSync(target, { force: true, recursive: true }); + deletedCacheData = true; + } + const unexpectedEntry = fs + .readdirSync(cacheDir) + .find((entry) => !HUGGING_FACE_CREDENTIAL_ENTRIES.has(entry)); + if (unexpectedEntry) { + throw new Error( + `Hugging Face cache cleanup left an unexpected entry at ${path.join(cacheDir, unexpectedEntry)}`, + ); + } + if (deletedCacheData) removed.push(`cache-contents:${cacheDir}`); +} + +function preserveSharedHuggingFaceCache(homeDir: string, preserved: string[]): void { + const cacheDir = sharedHuggingFaceCacheDir(homeDir); + if (statePathExists(cacheDir)) preserved.push(cacheDir); +} + function requireEngineSuccess( label: string, result: ReturnType, @@ -383,7 +449,6 @@ function cleanupLlamaCpp( homeDir: string, deps: CleanupDeps, removed: string[], - preserved: string[], options: { gatewayPort?: number; sandboxName?: string; @@ -441,8 +506,6 @@ function cleanupLlamaCpp( } fs.rmSync(paths.stateDir, { recursive: true }); removed.push(`state:${paths.stateDir}`); - const sharedCache = path.join(homeDir, ".cache", "huggingface"); - if (fs.existsSync(sharedCache)) preserved.push(sharedCache); return true; } if (journals.length > 1) { @@ -541,8 +604,6 @@ function cleanupLlamaCpp( } fs.rmSync(paths.stateDir, { recursive: true }); removed.push(`state:${paths.stateDir}`); - const sharedCache = path.join(homeDir, ".cache", "huggingface"); - if (fs.existsSync(sharedCache)) preserved.push(sharedCache); return true; } @@ -604,12 +665,13 @@ export function cleanupManagedLlamaCppRuntimeForSandbox( if (!fs.existsSync(paths.ownerPath)) return { ok: true, removed, preserved }; const owner = loadManagedLlamaCppOwner(paths); if (!owner || owner.sandboxName !== sandboxName) return { ok: true, removed, preserved }; - cleanupLlamaCpp(homeDir, deps, removed, preserved, { + cleanupLlamaCpp(homeDir, deps, removed, { gatewayPort: options.gatewayPort, sandboxName, env: options.env, engine: options.engine, }); + preserveSharedHuggingFaceCache(homeDir, preserved); return { ok: true, removed, preserved }; } catch (error) { return { ok: false, reason: (error as Error).message, removed, preserved }; @@ -652,13 +714,18 @@ export function cleanupLocalModelRuntimes( cleanupHostLocalVllm(vllmStateDir, deps, removed); } if (statePathExists(llamaPaths.stateDir)) { - cleanupLlamaCpp(homeDir, deps, removed, preserved, { + cleanupLlamaCpp(homeDir, deps, removed, { gatewayPort: options.gatewayPort, sandboxName: options.sandboxName, env: options.env, engine: options.engine, }); } + if (options.deleteModels) { + removeSharedHuggingFaceCacheData(homeDir, deps.currentUserId, removed, preserved); + } else { + preserveSharedHuggingFaceCache(homeDir, preserved); + } return { ok: true, removed, preserved }; } catch (error) { return { ok: false, reason: (error as Error).message, removed, preserved }; diff --git a/test/uninstall.test.ts b/test/uninstall.test.ts index 0da4703f7f3..5f269b47911 100644 --- a/test/uninstall.test.ts +++ b/test/uninstall.test.ts @@ -100,7 +100,7 @@ exit 0 const output = `${result.stdout}${result.stderr}`; expect(output).toMatch(/NemoHermes Uninstaller/); expect(output).toMatch(/Remove host-side NemoHermes resources/); - expect(output).toMatch(/Remove NemoHermes-pulled Ollama models/); + expect(output).toMatch(/Remove all Ollama models and Hugging Face cache data/); expect(output).not.toMatch(/NemoClaw Uninstaller/); }); From 5b8a4923f131d038b6c2c713c43e3f469ee19c88 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Wed, 12 Aug 2026 13:34:08 -0700 Subject: [PATCH 2/4] fix(uninstall): address review findings Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- scripts/smoke-macos-install.sh | 2 +- src/commands/internal/uninstall/plan.ts | 3 +- src/commands/internal/uninstall/run-plan.ts | 3 +- src/lib/actions/root-help.ts | 2 +- .../uninstall/run-plan-dual-station.test.ts | 2 +- .../run-plan-local-model-profile.test.ts | 94 ++++++++----------- src/lib/actions/uninstall/run-plan.ts | 8 +- src/lib/domain/uninstall/plan.test.ts | 45 ++++++--- src/lib/domain/uninstall/plan.ts | 29 +++--- test/uninstall.test.ts | 4 +- 10 files changed, 105 insertions(+), 87 deletions(-) diff --git a/scripts/smoke-macos-install.sh b/scripts/smoke-macos-install.sh index b8877a936a1..c5bff9c8a5a 100755 --- a/scripts/smoke-macos-install.sh +++ b/scripts/smoke-macos-install.sh @@ -59,7 +59,7 @@ Options: --allow-existing-state Allow running even if NemoClaw/OpenShell state already exists --keep-logs Preserve log files after success --remove-openshell Allow uninstall.sh to remove openshell - --delete-models Allow uninstall.sh to delete all Ollama models and Hugging Face cache data + --delete-models Allow uninstall.sh to delete all Ollama models and non-credential Hugging Face cache data (authentication files remain) -h, --help Show this help Environment: diff --git a/src/commands/internal/uninstall/plan.ts b/src/commands/internal/uninstall/plan.ts index e9d8c11a630..931bceb0dba 100644 --- a/src/commands/internal/uninstall/plan.ts +++ b/src/commands/internal/uninstall/plan.ts @@ -20,7 +20,8 @@ export default class InternalUninstallPlanCommand extends NemoClawCommand { description: "Accepted for parity with run-plan; ignored while planning", }), "delete-models": Flags.boolean({ - description: "Plan removal of all Ollama models and Hugging Face cache data", + description: + "Plan removal of all Ollama models and non-credential Hugging Face cache data (authentication files remain)", }), "keep-openshell": Flags.boolean({ description: "Keep the openshell binary installed" }), gateway: Flags.string({ description: "Gateway name", default: "nemoclaw" }), diff --git a/src/commands/internal/uninstall/run-plan.ts b/src/commands/internal/uninstall/run-plan.ts index 889a892e476..5ee8b54ad42 100644 --- a/src/commands/internal/uninstall/run-plan.ts +++ b/src/commands/internal/uninstall/run-plan.ts @@ -30,7 +30,8 @@ export default class InternalUninstallRunPlanCommand extends NemoClawCommand { "all-gateway-ports-child": Flags.boolean({ hidden: true }), "keep-openshell": Flags.boolean({ description: "Leave the openshell binary installed" }), "delete-models": Flags.boolean({ - description: "Remove all Ollama models and Hugging Face cache data", + description: + "Remove all Ollama models and non-credential Hugging Face cache data (authentication files remain)", }), "destroy-user-data": Flags.boolean({ description: diff --git a/src/lib/actions/root-help.ts b/src/lib/actions/root-help.ts index 3645400c8fc..4bf463fd7f0 100644 --- a/src/lib/actions/root-help.ts +++ b/src/lib/actions/root-help.ts @@ -82,7 +82,7 @@ export function help(): void { lines.push(` --yes${" ".repeat(29)}Skip the confirmation prompt`); lines.push(` --keep-openshell${" ".repeat(18)}Leave the openshell binary installed`); lines.push( - ` --delete-models${" ".repeat(19)}Remove all Ollama models and Hugging Face cache data`, + ` --delete-models${" ".repeat(19)}Remove all Ollama models and non-credential Hugging Face cache data (authentication files remain)`, ); lines.push( ` --destroy-user-data${" ".repeat(15)}Remove preserved ~/.nemoclaw/ user data (rebuild-backups/, backups/, sandboxes.json)`, diff --git a/src/lib/actions/uninstall/run-plan-dual-station.test.ts b/src/lib/actions/uninstall/run-plan-dual-station.test.ts index 8cda16b1406..a0b5324d45c 100644 --- a/src/lib/actions/uninstall/run-plan-dual-station.test.ts +++ b/src/lib/actions/uninstall/run-plan-dual-station.test.ts @@ -92,7 +92,7 @@ describe("managed distributed vLLM runtime uninstall", () => { } }); - it("stops a distributed runtime before deleting the shared Hugging Face cache", () => { + it("stops a distributed runtime before requesting shared Hugging Face cache-data cleanup", () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-cache-order-")); const stateDir = path.join(home, ".nemoclaw"); const receiptPath = path.join(stateDir, DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE); diff --git a/src/lib/actions/uninstall/run-plan-local-model-profile.test.ts b/src/lib/actions/uninstall/run-plan-local-model-profile.test.ts index 2a4417e2501..5d8b6ade5d8 100644 --- a/src/lib/actions/uninstall/run-plan-local-model-profile.test.ts +++ b/src/lib/actions/uninstall/run-plan-local-model-profile.test.ts @@ -48,6 +48,19 @@ function okWithKnownGatewayList(command: string, args: readonly string[]): RunRe : ok(); } +function runWithOllamaInventory(inventory: RunResult, failedModels: readonly string[] = []) { + const failures = new Set(failedModels); + const run: NonNullable = (command, args) => + command === "openshell" && args[0] === "gateway" && args[1] === "list" + ? ok(JSON.stringify([{ name: "nemoclaw" }])) + : command === "ollama" && args[0] === "list" + ? inventory + : command === "ollama" && args[0] === "rm" && failures.has(args[1] ?? "") + ? notFound() + : ok(); + return vi.fn(run); +} + function publishManagedLlamaOwner( homeDir: string, gatewayPort: number, @@ -133,7 +146,7 @@ describe("uninstall local model profile cleanup", () => { expect(errors.join("\n")).toContain("could not inventory reserved managed inference"); }); - it("states that model deletion removes every Ollama model and the Hugging Face cache", () => { + it("states that model deletion removes every Ollama model and non-credential Hugging Face cache data", () => { const logs: string[] = []; const result = runUninstallPlan( { assumeYes: false, deleteModels: true, keepOpenShell: true }, @@ -156,7 +169,7 @@ describe("uninstall local model profile cleanup", () => { expect(logs).toContain("Aborted."); }); - it("removes a shared Hugging Face cache even when no managed runtime state remains", () => { + it("requests shared Hugging Face cache-data cleanup when no managed runtime state remains", () => { const tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-llama-cache-")); const cacheDir = path.join(tmpHome, ".cache", "huggingface"); const runLocalModelRuntimeCleanup = vi.fn(() => ok()); @@ -185,21 +198,15 @@ describe("uninstall local model profile cleanup", () => { }); it("deletes every model returned by Ollama inventory", () => { - const run = vi.fn((command: string, args: string[], _options?: { env?: NodeJS.ProcessEnv }) => { - if (command === "openshell" && args[0] === "gateway" && args[1] === "list") { - return ok(JSON.stringify([{ name: "nemoclaw" }])); - } - if (command === "ollama" && args[0] === "list") { - return ok( - [ - "NAME ID SIZE MODIFIED", - "team/first:latest 111111111111 5 GB 1 hour ago", - "second:q4 222222222222 3 GB 2 hours ago", - ].join("\n"), - ); - } - return ok(); - }); + const run = runWithOllamaInventory( + ok( + [ + "NAME ID SIZE MODIFIED", + "team/first:latest 111111111111 5 GB 1 hour ago", + "second:q4 222222222222 3 GB 2 hours ago", + ].join("\n"), + ), + ); const result = runUninstallPlan( { assumeYes: true, deleteModels: true, keepOpenShell: true }, @@ -227,18 +234,15 @@ describe("uninstall local model profile cleanup", () => { .filter(([command, args]) => command === "ollama" && args[0] === "rm") .map(([, args]) => args[1]), ).toEqual(["team/first:latest", "second:q4"]); + expect( + run.mock.calls + .filter(([command, args]) => command === "ollama" && args[0] === "rm") + .map(([, , options]) => options?.timeout), + ).toEqual([60_000, 60_000]); }); it("ignores a remote Ollama environment override during model cleanup", () => { - const run = vi.fn((command: string, args: string[], _options?: { env?: NodeJS.ProcessEnv }) => { - if (command === "openshell" && args[0] === "gateway" && args[1] === "list") { - return ok(JSON.stringify([{ name: "nemoclaw" }])); - } - if (command === "ollama" && args[0] === "list") { - return ok("NAME ID SIZE MODIFIED\nlocal-model 111 1 GB now\n"); - } - return ok(); - }); + const run = runWithOllamaInventory(ok("NAME ID SIZE MODIFIED\nlocal-model 111 1 GB now\n")); const result = runUninstallPlan( { assumeYes: true, deleteModels: true, keepOpenShell: true }, @@ -264,15 +268,9 @@ describe("uninstall local model profile cleanup", () => { it("fails without deleting any Ollama model when inventory is malformed", () => { const errors: string[] = []; - const run = vi.fn((command: string, args: string[]) => { - if (command === "openshell" && args[0] === "gateway" && args[1] === "list") { - return ok(JSON.stringify([{ name: "nemoclaw" }])); - } - if (command === "ollama" && args[0] === "list") { - return ok("NAME ID SIZE MODIFIED\n--unsafe 111111111111 1 GB now\n"); - } - return ok(); - }); + const run = runWithOllamaInventory( + ok("NAME ID SIZE MODIFIED\n--unsafe 111111111111 1 GB now\n"), + ); const result = runUninstallPlan( { assumeYes: true, deleteModels: true, keepOpenShell: true }, @@ -295,14 +293,10 @@ describe("uninstall local model profile cleanup", () => { }); it("fails without deleting any Ollama model when inventory execution fails", () => { - const run = vi.fn((command: string, args: string[]) => { - if (command === "openshell" && args[0] === "gateway" && args[1] === "list") { - return ok(JSON.stringify([{ name: "nemoclaw" }])); - } - if (command === "ollama" && args[0] === "list") { - return { status: 1, stdout: "", stderr: "daemon unavailable" }; - } - return ok(); + const run = runWithOllamaInventory({ + status: 1, + stdout: "", + stderr: "daemon unavailable", }); const result = runUninstallPlan( @@ -324,16 +318,10 @@ describe("uninstall local model profile cleanup", () => { }); it("attempts every inventoried Ollama removal and fails when one removal fails", () => { - const run = vi.fn((command: string, args: string[]) => { - if (command === "openshell" && args[0] === "gateway" && args[1] === "list") { - return ok(JSON.stringify([{ name: "nemoclaw" }])); - } - if (command === "ollama" && args[0] === "list") { - return ok("NAME ID SIZE MODIFIED\nfirst 111 1 GB now\nsecond 222 1 GB now\n"); - } - if (command === "ollama" && args[0] === "rm" && args[1] === "first") return notFound(); - return ok(); - }); + const run = runWithOllamaInventory( + ok("NAME ID SIZE MODIFIED\nfirst 111 1 GB now\nsecond 222 1 GB now\n"), + ["first"], + ); const result = runUninstallPlan( { assumeYes: true, deleteModels: true, keepOpenShell: true }, diff --git a/src/lib/actions/uninstall/run-plan.ts b/src/lib/actions/uninstall/run-plan.ts index 744ab9e10a4..205e2c03d5f 100644 --- a/src/lib/actions/uninstall/run-plan.ts +++ b/src/lib/actions/uninstall/run-plan.ts @@ -1775,7 +1775,13 @@ function removeOllamaModels(options: UninstallRunOptions, runtime: UninstallRunt } let ok = true; for (const model of models) { - if (runtime.run("ollama", ["rm", model], { env: ollamaEnv, stdio: "ignore" }).status === 0) + if ( + runtime.run("ollama", ["rm", model], { + env: ollamaEnv, + stdio: "ignore", + timeout: 60_000, + }).status === 0 + ) runtime.log(`Removed Ollama model '${model}'`); else { runtime.error(`Failed to remove Ollama model '${model}'`); diff --git a/src/lib/domain/uninstall/plan.test.ts b/src/lib/domain/uninstall/plan.test.ts index 6f60bf1b17c..2a2e069def1 100644 --- a/src/lib/domain/uninstall/plan.test.ts +++ b/src/lib/domain/uninstall/plan.test.ts @@ -54,24 +54,32 @@ describe("uninstall plan", () => { const stoppingServicesStep = plan.steps.find((step) => step.name === "Stopping services"); expect(stoppingServicesStep).toBeTruthy(); expect(stoppingServicesStep?.actions).toEqual( - expect.arrayContaining([{ kind: "stop-ollama-auth-proxy" }, { kind: "stop-model-router" }]), + expect.arrayContaining([ + { kind: "stop-ollama-auth-proxy" }, + { kind: "stop-model-router" }, + { + kind: "preserve-hugging-face-cache-data", + path: path.join("/home/test", ".cache", "huggingface"), + }, + ]), ); + const modelStoresStep = plan.steps.find((step) => step.name === "Model stores"); + expect(modelStoresStep?.actions).toEqual([{ kind: "preserve-ollama-models" }]); }); it("respects delete-models, keep-openshell, custom gateway, and foreign shim decisions", () => { const paths = defaultUninstallPaths({ home: "/home/test", xdgBinHome: "/bin" }); - const actions = flattenUninstallPlan( - buildUninstallPlan(paths, { - deleteModels: true, - gatewayName: "custom", - keepOpenShell: true, - shim: { - kind: "preserve-foreign-file", - reason: "regular file is not an installer-managed shim", - remove: false, - }, - }), - ); + const plan = buildUninstallPlan(paths, { + deleteModels: true, + gatewayName: "custom", + keepOpenShell: true, + shim: { + kind: "preserve-foreign-file", + reason: "regular file is not an installer-managed shim", + remove: false, + }, + }); + const actions = flattenUninstallPlan(plan); expect(actions).toEqual( expect.arrayContaining([{ kind: "delete-docker-volume", name: "openshell-cluster-custom" }]), @@ -85,6 +93,17 @@ describe("uninstall plan", () => { }, ]), ); + expect(plan.steps.find((step) => step.name === "Stopping services")?.actions).toEqual( + expect.arrayContaining([ + { + kind: "delete-hugging-face-cache-data", + path: path.join("/home/test", ".cache", "huggingface"), + }, + ]), + ); + expect(plan.steps.find((step) => step.name === "Model stores")?.actions).toEqual([ + { kind: "delete-all-ollama-models" }, + ]); expect(actions).toEqual( expect.arrayContaining([ { diff --git a/src/lib/domain/uninstall/plan.ts b/src/lib/domain/uninstall/plan.ts index 933b7900338..04a879ed394 100644 --- a/src/lib/domain/uninstall/plan.ts +++ b/src/lib/domain/uninstall/plan.ts @@ -79,6 +79,19 @@ export function buildUninstallPlan( { kind: "stop-orphaned-openshell-processes" }, { kind: "stop-ollama-auth-proxy" }, { kind: "stop-model-router" }, + ...(options.deleteModels + ? [ + { + kind: "delete-hugging-face-cache-data" as const, + path: paths.huggingFaceModelCacheDir, + }, + ] + : [ + { + kind: "preserve-hugging-face-cache-data" as const, + path: paths.huggingFaceModelCacheDir, + }, + ]), ], }, { @@ -109,20 +122,8 @@ export function buildUninstallPlan( { name: "Model stores", actions: options.deleteModels - ? [ - { kind: "delete-all-ollama-models" }, - { - kind: "delete-hugging-face-cache-data", - path: paths.huggingFaceModelCacheDir, - }, - ] - : [ - { kind: "preserve-ollama-models" }, - { - kind: "preserve-hugging-face-cache-data", - path: paths.huggingFaceModelCacheDir, - }, - ], + ? [{ kind: "delete-all-ollama-models" }] + : [{ kind: "preserve-ollama-models" }], }, { name: "State and binaries", diff --git a/test/uninstall.test.ts b/test/uninstall.test.ts index 5f269b47911..4f7660e5d20 100644 --- a/test/uninstall.test.ts +++ b/test/uninstall.test.ts @@ -100,7 +100,9 @@ exit 0 const output = `${result.stdout}${result.stderr}`; expect(output).toMatch(/NemoHermes Uninstaller/); expect(output).toMatch(/Remove host-side NemoHermes resources/); - expect(output).toMatch(/Remove all Ollama models and Hugging Face cache data/); + expect(output).toMatch( + /Remove all Ollama models and non-credential Hugging\s+Face cache data \(authentication files remain\)/, + ); expect(output).not.toMatch(/NemoClaw Uninstaller/); }); From a1b8884eaa42505af3e32c59141bca5552154e6e Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Wed, 12 Aug 2026 13:34:08 -0700 Subject: [PATCH 3/4] fix(uninstall): address review findings Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- docs/reference/host-files-and-state.mdx | 2 +- .../uninstall/run-plan-dual-station.test.ts | 9 +-- .../run-plan-local-model-profile.test.ts | 21 ++++-- src/lib/actions/uninstall/run-plan.ts | 74 +++++++++++++------ src/lib/domain/uninstall/plan.test.ts | 39 +++++----- src/lib/domain/uninstall/plan.ts | 29 ++++---- .../local-model-profile/cleanup-entry.ts | 17 +++-- .../cleanup-path-safety.test.ts | 21 +++--- .../local-model-profile/cleanup.test.ts | 43 ++++++----- .../inference/local-model-profile/cleanup.ts | 33 +++++++-- 10 files changed, 178 insertions(+), 110 deletions(-) diff --git a/docs/reference/host-files-and-state.mdx b/docs/reference/host-files-and-state.mdx index ca8315f87b1..d09b9c2afec 100644 --- a/docs/reference/host-files-and-state.mdx +++ b/docs/reference/host-files-and-state.mdx @@ -141,7 +141,7 @@ The shared Hugging Face cache used by managed vLLM and managed llama.cpp remains With `--delete-models`, full uninstall deletes every model in the local Ollama inventory and all non-credential data in the current user's shared Hugging Face cache. The flag can delete cached files that other applications installed or use. It preserves the Hugging Face `token` and `stored_tokens` authentication files. -NemoClaw deletes the cache only after managed model runtimes stop and only when no sibling gateway remains. +NemoClaw deletes the non-credential cache data only after managed model runtimes stop and only when no sibling gateway remains. An Ollama inventory error, model deletion error, unsafe cache path, or cache-data deletion error makes uninstall exit nonzero. For operational uninstall steps, refer to: diff --git a/src/lib/actions/uninstall/run-plan-dual-station.test.ts b/src/lib/actions/uninstall/run-plan-dual-station.test.ts index a0b5324d45c..95dcccff4bd 100644 --- a/src/lib/actions/uninstall/run-plan-dual-station.test.ts +++ b/src/lib/actions/uninstall/run-plan-dual-station.test.ts @@ -102,7 +102,7 @@ describe("managed distributed vLLM runtime uninstall", () => { fs.mkdirSync(`${receiptPath}.ssh-binding`, { mode: 0o700 }); fs.mkdirSync(cacheDir, { recursive: true }); const runDualStationRuntimeCleanup = vi.fn(() => ok()); - const runLocalModelRuntimeCleanup = vi.fn(() => ok()); + const runHuggingFaceCacheDataCleanup = vi.fn(() => ok()); try { const result = runUninstallPlan( @@ -116,18 +116,17 @@ describe("managed distributed vLLM runtime uninstall", () => { rmSync: vi.fn(), run: okWithKnownGatewayList, runDualStationRuntimeCleanup, - runLocalModelRuntimeCleanup, + runHuggingFaceCacheDataCleanup, }, ); expect(result.exitCode).toBe(0); expect(runDualStationRuntimeCleanup).toHaveBeenCalledOnce(); - expect(runLocalModelRuntimeCleanup).toHaveBeenCalledWith( - true, + expect(runHuggingFaceCacheDataCleanup).toHaveBeenCalledWith( expect.objectContaining({ stdio: "inherit" }), ); expect(runDualStationRuntimeCleanup.mock.invocationCallOrder[0]).toBeLessThan( - runLocalModelRuntimeCleanup.mock.invocationCallOrder[0], + runHuggingFaceCacheDataCleanup.mock.invocationCallOrder[0], ); } finally { fs.rmSync(home, { recursive: true, force: true }); diff --git a/src/lib/actions/uninstall/run-plan-local-model-profile.test.ts b/src/lib/actions/uninstall/run-plan-local-model-profile.test.ts index 5d8b6ade5d8..1bc42c7b92b 100644 --- a/src/lib/actions/uninstall/run-plan-local-model-profile.test.ts +++ b/src/lib/actions/uninstall/run-plan-local-model-profile.test.ts @@ -169,10 +169,11 @@ describe("uninstall local model profile cleanup", () => { expect(logs).toContain("Aborted."); }); - it("requests shared Hugging Face cache-data cleanup when no managed runtime state remains", () => { + it("requests shared Hugging Face cache-data cleanup during Model stores", () => { const tmpHome = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-llama-cache-")); const cacheDir = path.join(tmpHome, ".cache", "huggingface"); - const runLocalModelRuntimeCleanup = vi.fn(() => ok()); + const log = vi.fn(); + const runHuggingFaceCacheDataCleanup = vi.fn(() => ok()); try { const result = runUninstallPlan( { assumeYes: true, deleteModels: true, keepOpenShell: true }, @@ -181,17 +182,23 @@ describe("uninstall local model profile cleanup", () => { env: { HOME: tmpHome } as NodeJS.ProcessEnv, existsSync: (target) => target === cacheDir, isTty: false, - log: () => {}, + log, run: vi.fn(okWithKnownGatewayList), - runLocalModelRuntimeCleanup, + runHuggingFaceCacheDataCleanup, }, ); expect(result.exitCode).toBe(0); - expect(runLocalModelRuntimeCleanup).toHaveBeenCalledWith( - true, + expect(runHuggingFaceCacheDataCleanup).toHaveBeenCalledWith( expect.objectContaining({ stdio: "inherit" }), ); + const modelStoresLogIndex = log.mock.calls.findIndex( + ([line]) => line === "[5/6] Model stores", + ); + expect(modelStoresLogIndex).toBeGreaterThanOrEqual(0); + expect(log.mock.invocationCallOrder[modelStoresLogIndex]).toBeLessThan( + runHuggingFaceCacheDataCleanup.mock.invocationCallOrder[0], + ); } finally { fs.rmSync(tmpHome, { recursive: true, force: true }); } @@ -488,7 +495,7 @@ describe("uninstall local model profile cleanup", () => { ); expect(result.exitCode).toBe(1); - expect(errors.join("\n")).toContain("Host-local model and cache cleanup did not complete"); + expect(errors.join("\n")).toContain("Host-local model runtime cleanup did not complete"); expect(runDocker.mock.calls.some(([args]) => args[0] === "rm")).toBe(false); }); }); diff --git a/src/lib/actions/uninstall/run-plan.ts b/src/lib/actions/uninstall/run-plan.ts index 205e2c03d5f..0cad7884d17 100644 --- a/src/lib/actions/uninstall/run-plan.ts +++ b/src/lib/actions/uninstall/run-plan.ts @@ -115,7 +115,8 @@ export interface UninstallRunDeps { run?: (command: string, args: string[], options?: SpawnSyncOptions) => RunResult; runDocker?: (args: string[], options?: SpawnSyncOptions) => RunResult; runDualStationRuntimeCleanup?: (receiptPath: string, options?: SpawnSyncOptions) => RunResult; - runLocalModelRuntimeCleanup?: (deleteModels: boolean, options?: SpawnSyncOptions) => RunResult; + runHuggingFaceCacheDataCleanup?: (options?: SpawnSyncOptions) => RunResult; + runLocalModelRuntimeCleanup?: (options?: SpawnSyncOptions) => RunResult; runManagedLlamaCppRuntimeCleanup?: (sandboxName: string, gatewayPort: number) => RunResult; } @@ -425,7 +426,8 @@ interface UninstallRuntime { run: (command: string, args: string[], options?: SpawnSyncOptions) => RunResult; runDocker: (args: string[], options?: SpawnSyncOptions) => RunResult; runDualStationRuntimeCleanup: (receiptPath: string, options?: SpawnSyncOptions) => RunResult; - runLocalModelRuntimeCleanup: (deleteModels: boolean, options?: SpawnSyncOptions) => RunResult; + runHuggingFaceCacheDataCleanup: (options?: SpawnSyncOptions) => RunResult; + runLocalModelRuntimeCleanup: (options?: SpawnSyncOptions) => RunResult; runManagedLlamaCppRuntimeCleanup: (sandboxName: string, gatewayPort: number) => RunResult; warn: (message: string) => void; } @@ -481,9 +483,27 @@ function buildRuntime(deps: UninstallRunDeps): UninstallRuntime { ], options, )), + runHuggingFaceCacheDataCleanup: + deps.runHuggingFaceCacheDataCleanup ?? + ((options = {}) => + defaultRun( + process.execPath, + [ + path.resolve( + __dirname, + "..", + "..", + "inference", + "local-model-profile", + "cleanup-entry.js", + ), + "--delete-cache-data", + ], + options, + )), runLocalModelRuntimeCleanup: deps.runLocalModelRuntimeCleanup ?? - ((deleteModels, options = {}) => + ((options = {}) => defaultRun( process.execPath, [ @@ -495,7 +515,7 @@ function buildRuntime(deps: UninstallRunDeps): UninstallRuntime { "local-model-profile", "cleanup-entry.js", ), - deleteModels ? "--delete-models" : "--keep-models", + "--clean-runtimes", ], options, )), @@ -1511,11 +1531,7 @@ function removeManagedDistributedVllmRuntime( return false; } -function removeHostLocalModelRuntimes( - paths: UninstallPaths, - deleteModels: boolean, - runtime: UninstallRuntime, -): boolean { +function removeHostLocalModelRuntimes(paths: UninstallPaths, runtime: UninstallRuntime): boolean { const sharedRoot = path.dirname(paths.managedSwapMarkerPath); const hasLlamaState = runtime.existsSync(path.join(sharedRoot, "managed-llama-cpp")); const hasManagedKey = runtime.existsSync(path.join(sharedRoot, MANAGED_VLLM_API_KEY_FILE)); @@ -1523,21 +1539,16 @@ function removeHostLocalModelRuntimes( MANAGED_CLUSTER_VLLM_RUNTIME_RECEIPT_FILE, DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE, ].some((name) => runtime.existsSync(path.join(sharedRoot, name))); - const hasSharedModelCache = runtime.existsSync(paths.huggingFaceModelCacheDir); - if ( - !hasLlamaState && - (!hasManagedKey || hasDistributedReceipt) && - (!deleteModels || !hasSharedModelCache) - ) { + if (!hasLlamaState && (!hasManagedKey || hasDistributedReceipt)) { return true; } - const result = runtime.runLocalModelRuntimeCleanup(deleteModels, { + const result = runtime.runLocalModelRuntimeCleanup({ env: runtime.env, stdio: "inherit", }); if (result.status === 0) return true; runtime.error( - "Host-local model and cache cleanup did not complete. NemoClaw did not start the remaining uninstall steps. Resolve the reported ownership, path, or Docker error and retry uninstall.", + "Host-local model runtime cleanup did not complete. NemoClaw did not start the remaining uninstall steps. Resolve the reported ownership or Docker error and retry uninstall.", ); return false; } @@ -1596,7 +1607,6 @@ function removeManagedLlamaCppRuntimes( function removeManagedModelRuntimes( paths: UninstallPaths, - deleteModels: boolean, runtime: UninstallRuntime, scopedToSelectedGateway: boolean, ): boolean { @@ -1608,7 +1618,7 @@ function removeManagedModelRuntimes( DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE, ].some((name) => runtime.existsSync(path.join(sharedRoot, name))); if (!removeManagedDistributedVllmRuntime(paths, runtime, !hasDistributedReceipt)) return false; - if (!removeHostLocalModelRuntimes(paths, deleteModels, runtime)) return false; + if (!removeHostLocalModelRuntimes(paths, runtime)) return false; if (!hasDistributedReceipt) { removePath(path.join(sharedRoot, MANAGED_VLLM_API_KEY_FILE), runtime); } @@ -1792,6 +1802,7 @@ function removeOllamaModels(options: UninstallRunOptions, runtime: UninstallRunt } function removeHostModelStores( + paths: UninstallPaths, options: UninstallRunOptions, runtime: UninstallRuntime, scopedToSelectedGateway: boolean, @@ -1802,7 +1813,24 @@ function removeHostModelStores( ); return true; } - return removeOllamaModels(options, runtime); + const ollamaOk = removeOllamaModels(options, runtime); + if (!options.deleteModels) { + runtime.log("Keeping Hugging Face cache data as requested."); + return ollamaOk; + } + if (!runtime.existsSync(paths.huggingFaceModelCacheDir)) { + runtime.log("No Hugging Face cache data found."); + return ollamaOk; + } + const result = runtime.runHuggingFaceCacheDataCleanup({ + env: runtime.env, + stdio: "inherit", + }); + if (result.status === 0) return ollamaOk; + runtime.error( + "Hugging Face cache-data cleanup did not complete during Model stores. Resolve the reported ownership or path error and retry uninstall.", + ); + return false; } interface OtherGatewayInspection { @@ -2150,9 +2178,7 @@ function executePlan( for (const [index, step] of plan.steps.entries()) { runtime.log(`[${index + 1}/${plan.steps.length}] ${planStepDisplayName(step.name, branding)}`); if (step.name === "Stopping services") { - if ( - !removeManagedModelRuntimes(paths, options.deleteModels, runtime, scopedToSelectedGateway) - ) { + if (!removeManagedModelRuntimes(paths, runtime, scopedToSelectedGateway)) { return { ok: false }; } // #8220: a gateway-scoped uninstall still needs the selected OpenShell @@ -2289,7 +2315,7 @@ function executePlan( if (action.kind === "delete-docker-volume") removeDockerVolume(action.name, runtime); } } else if (step.name === "Model stores") { - if (!removeHostModelStores(options, runtime, scopedToSelectedGateway)) ok = false; + if (!removeHostModelStores(paths, options, runtime, scopedToSelectedGateway)) ok = false; } else if (step.name === "State and binaries") { removeManagedSwap(paths, runtime, scopedToSelectedGateway); if (!scopedToSelectedGateway) { diff --git a/src/lib/domain/uninstall/plan.test.ts b/src/lib/domain/uninstall/plan.test.ts index 2a2e069def1..9b31d51faf6 100644 --- a/src/lib/domain/uninstall/plan.test.ts +++ b/src/lib/domain/uninstall/plan.test.ts @@ -54,17 +54,21 @@ describe("uninstall plan", () => { const stoppingServicesStep = plan.steps.find((step) => step.name === "Stopping services"); expect(stoppingServicesStep).toBeTruthy(); expect(stoppingServicesStep?.actions).toEqual( - expect.arrayContaining([ - { kind: "stop-ollama-auth-proxy" }, - { kind: "stop-model-router" }, - { - kind: "preserve-hugging-face-cache-data", - path: path.join("/home/test", ".cache", "huggingface"), - }, - ]), + expect.arrayContaining([{ kind: "stop-ollama-auth-proxy" }, { kind: "stop-model-router" }]), ); + expect( + stoppingServicesStep?.actions.some( + (action) => action.kind === "preserve-hugging-face-cache-data", + ), + ).toBe(false); const modelStoresStep = plan.steps.find((step) => step.name === "Model stores"); - expect(modelStoresStep?.actions).toEqual([{ kind: "preserve-ollama-models" }]); + expect(modelStoresStep?.actions).toEqual([ + { kind: "preserve-ollama-models" }, + { + kind: "preserve-hugging-face-cache-data", + path: path.join("/home/test", ".cache", "huggingface"), + }, + ]); }); it("respects delete-models, keep-openshell, custom gateway, and foreign shim decisions", () => { @@ -93,17 +97,18 @@ describe("uninstall plan", () => { }, ]), ); - expect(plan.steps.find((step) => step.name === "Stopping services")?.actions).toEqual( - expect.arrayContaining([ - { - kind: "delete-hugging-face-cache-data", - path: path.join("/home/test", ".cache", "huggingface"), - }, - ]), - ); expect(plan.steps.find((step) => step.name === "Model stores")?.actions).toEqual([ { kind: "delete-all-ollama-models" }, + { + kind: "delete-hugging-face-cache-data", + path: path.join("/home/test", ".cache", "huggingface"), + }, ]); + expect( + plan.steps + .find((step) => step.name === "Stopping services") + ?.actions.some((action) => action.kind === "delete-hugging-face-cache-data"), + ).toBe(false); expect(actions).toEqual( expect.arrayContaining([ { diff --git a/src/lib/domain/uninstall/plan.ts b/src/lib/domain/uninstall/plan.ts index 04a879ed394..933b7900338 100644 --- a/src/lib/domain/uninstall/plan.ts +++ b/src/lib/domain/uninstall/plan.ts @@ -79,19 +79,6 @@ export function buildUninstallPlan( { kind: "stop-orphaned-openshell-processes" }, { kind: "stop-ollama-auth-proxy" }, { kind: "stop-model-router" }, - ...(options.deleteModels - ? [ - { - kind: "delete-hugging-face-cache-data" as const, - path: paths.huggingFaceModelCacheDir, - }, - ] - : [ - { - kind: "preserve-hugging-face-cache-data" as const, - path: paths.huggingFaceModelCacheDir, - }, - ]), ], }, { @@ -122,8 +109,20 @@ export function buildUninstallPlan( { name: "Model stores", actions: options.deleteModels - ? [{ kind: "delete-all-ollama-models" }] - : [{ kind: "preserve-ollama-models" }], + ? [ + { kind: "delete-all-ollama-models" }, + { + kind: "delete-hugging-face-cache-data", + path: paths.huggingFaceModelCacheDir, + }, + ] + : [ + { kind: "preserve-ollama-models" }, + { + kind: "preserve-hugging-face-cache-data", + path: paths.huggingFaceModelCacheDir, + }, + ], }, { name: "State and binaries", diff --git a/src/lib/inference/local-model-profile/cleanup-entry.ts b/src/lib/inference/local-model-profile/cleanup-entry.ts index c5eee869049..7854a763b6a 100644 --- a/src/lib/inference/local-model-profile/cleanup-entry.ts +++ b/src/lib/inference/local-model-profile/cleanup-entry.ts @@ -1,22 +1,25 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { cleanupLocalModelRuntimes } from "./cleanup"; +import { cleanupHuggingFaceCacheData, cleanupLocalModelRuntimes } from "./cleanup"; try { const args = process.argv.slice(2); - if (args.length !== 1 || !["--delete-models", "--keep-models"].includes(args[0]!)) { - throw new Error("host-local model cleanup requires --delete-models or --keep-models"); + if (args.length !== 1 || !["--clean-runtimes", "--delete-cache-data"].includes(args[0]!)) { + throw new Error("host model cleanup requires --clean-runtimes or --delete-cache-data"); } - const result = cleanupLocalModelRuntimes({ deleteModels: args[0] === "--delete-models" }); + const result = + args[0] === "--clean-runtimes" ? cleanupLocalModelRuntimes({}) : cleanupHuggingFaceCacheData(); for (const resource of result.removed) console.log(`Removed ${resource}`); for (const resource of result.preserved) console.log(`Preserved ${resource}`); if (!result.ok) throw new Error(result.reason); } catch (error) { + const operation = + process.argv.at(-1) === "--delete-cache-data" + ? "Hugging Face cache-data cleanup" + : "Host-local model runtime cleanup"; console.error( - `Refusing uninstall before host-local model cleanup: ${ - error instanceof Error ? error.message : String(error) - }`, + `${operation} failed: ${error instanceof Error ? error.message : String(error)}`, ); process.exitCode = 1; } diff --git a/src/lib/inference/local-model-profile/cleanup-path-safety.test.ts b/src/lib/inference/local-model-profile/cleanup-path-safety.test.ts index 780b5d4de15..0be329b2e45 100644 --- a/src/lib/inference/local-model-profile/cleanup-path-safety.test.ts +++ b/src/lib/inference/local-model-profile/cleanup-path-safety.test.ts @@ -7,7 +7,11 @@ import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { cleanupLocalModelRuntimes, type LocalModelRuntimeCleanupOptions } from "./cleanup"; +import { + cleanupHuggingFaceCacheData, + cleanupLocalModelRuntimes, + type LocalModelRuntimeCleanupOptions, +} from "./cleanup"; const temporaryDirectories: string[] = []; @@ -45,7 +49,7 @@ describe("host-local model cleanup path safety", () => { fs.mkdirSync(target); fs.symlinkSync(target, path.join(cacheParent, "huggingface"), "dir"); - expect(cleanupLocalModelRuntimes({ deleteModels: true, homeDir })).toMatchObject({ + expect(cleanupHuggingFaceCacheData({ homeDir })).toMatchObject({ ok: false, reason: expect.stringContaining("model cache is a symlink"), }); @@ -62,10 +66,9 @@ describe("host-local model cleanup path safety", () => { const observedOwner = fs.lstatSync(cache).uid; expect( - cleanupLocalModelRuntimes({ - deleteModels: true, + cleanupHuggingFaceCacheData({ homeDir, - deps: dockerDeps({ currentUserId: observedOwner + 1 }), + currentUserId: observedOwner + 1, }), ).toMatchObject({ ok: false, @@ -83,7 +86,7 @@ describe("host-local model cleanup path safety", () => { fs.mkdirSync(cache, { recursive: true }); fs.chmodSync(cache, 0o770); - expect(cleanupLocalModelRuntimes({ deleteModels: true, homeDir })).toMatchObject({ + expect(cleanupHuggingFaceCacheData({ homeDir })).toMatchObject({ ok: false, reason: expect.stringContaining("model cache is not current-user filesystem authority"), }); @@ -102,7 +105,7 @@ describe("host-local model cleanup path safety", () => { fs.symlinkSync(target, path.join(stateRoot, "managed-llama-cpp"), "dir"); const deps = dockerDeps(); - expect(cleanupLocalModelRuntimes({ deleteModels: false, homeDir, deps })).toMatchObject({ + expect(cleanupLocalModelRuntimes({ homeDir, deps })).toMatchObject({ ok: false, reason: expect.stringContaining("symlink"), }); @@ -120,7 +123,7 @@ describe("host-local model cleanup path safety", () => { const observedOwner = fs.lstatSync(stateDir).uid; const deps = dockerDeps({ currentUserId: observedOwner + 1 }); - expect(cleanupLocalModelRuntimes({ deleteModels: false, homeDir, deps })).toMatchObject({ + expect(cleanupLocalModelRuntimes({ homeDir, deps })).toMatchObject({ ok: false, reason: expect.stringContaining("not owned by the current user"), }); @@ -136,7 +139,7 @@ describe("host-local model cleanup path safety", () => { fs.writeFileSync(path.join(stateRoot, "managed-llama-cpp"), "unexpected\n"); const deps = dockerDeps(); - expect(cleanupLocalModelRuntimes({ deleteModels: false, homeDir, deps })).toMatchObject({ + expect(cleanupLocalModelRuntimes({ homeDir, deps })).toMatchObject({ ok: false, reason: expect.stringContaining("not a directory"), }); diff --git a/src/lib/inference/local-model-profile/cleanup.test.ts b/src/lib/inference/local-model-profile/cleanup.test.ts index 694771a641c..40f4f648ce6 100644 --- a/src/lib/inference/local-model-profile/cleanup.test.ts +++ b/src/lib/inference/local-model-profile/cleanup.test.ts @@ -24,6 +24,7 @@ import { } from "../serving/vllm-host-local-lifecycle"; import { managedVllmStateDir } from "../vllm-api-key"; import { + cleanupHuggingFaceCacheData, cleanupLocalModelRuntimes, cleanupManagedLlamaCppRuntimeForSandbox, resolveManagedLlamaCppCleanupTarget, @@ -71,7 +72,7 @@ describe("host-local model cleanup", () => { preserved: [], removed: [], }); - expect(cleanupLocalModelRuntimes({ deleteModels: false, homeDir })).toEqual({ + expect(cleanupLocalModelRuntimes({ homeDir })).toEqual({ ok: true, preserved: [], removed: [], @@ -84,7 +85,6 @@ describe("host-local model cleanup", () => { createPreStartManagedState(homeDir, harness.engine); const result = cleanupLocalModelRuntimes({ - deleteModels: false, homeDir, engine: harness.engine, }); @@ -107,7 +107,6 @@ describe("host-local model cleanup", () => { createPreStartManagedState(homeDir, harness.engine); const result = cleanupLocalModelRuntimes({ - deleteModels: false, homeDir, engine: harness.engine, }); @@ -147,7 +146,6 @@ describe("host-local model cleanup", () => { expect( cleanupLocalModelRuntimes({ - deleteModels: false, homeDir, deps: { capture: capture as never, @@ -202,7 +200,6 @@ describe("host-local model cleanup", () => { expect( cleanupLocalModelRuntimes({ - deleteModels: false, gatewayPort, homeDir, deps: { @@ -270,7 +267,6 @@ describe("host-local model cleanup", () => { expect( cleanupLocalModelRuntimes({ - deleteModels: false, homeDir, deps: { capture: capture as never, @@ -304,7 +300,6 @@ describe("host-local model cleanup", () => { expect( cleanupLocalModelRuntimes({ - deleteModels: false, homeDir, deps: { capture: vi.fn((argv: readonly string[]) => @@ -339,7 +334,6 @@ describe("host-local model cleanup", () => { expect( cleanupLocalModelRuntimes({ - deleteModels: false, homeDir, deps: { capture: capture as never, @@ -366,7 +360,6 @@ describe("host-local model cleanup", () => { const ambientRun = vi.fn(() => ({ status: 0 })) as never; const result = cleanupLocalModelRuntimes({ - deleteModels: true, homeDir, engine: harness.engine, deps: { @@ -381,6 +374,26 @@ describe("host-local model cleanup", () => { expect(ambientCapture).not.toHaveBeenCalled(); expect(ambientForceRm).not.toHaveBeenCalled(); expect(ambientRun).not.toHaveBeenCalled(); + expect(fs.readFileSync(path.join(cache, "shared-model"), "utf8")).toBe("delete"); + expect(fs.readFileSync(path.join(cache, "token"), "utf8")).toBe("keep-credential"); + expect(fs.readFileSync(path.join(cache, "stored_tokens"), "utf8")).toBe( + "keep-stored-credentials", + ); + expect(result.removed).not.toContain(`cache-contents:${cache}`); + expect(result.preserved).toContain(cache); + }); + + it("removes non-credential Hugging Face cache data without running runtime cleanup", () => { + const homeDir = temporaryHome(); + const cache = path.join(homeDir, ".cache", "huggingface"); + fs.mkdirSync(cache, { recursive: true }); + fs.writeFileSync(path.join(cache, "shared-model"), "delete"); + fs.writeFileSync(path.join(cache, "token"), "keep-credential", { mode: 0o600 }); + fs.writeFileSync(path.join(cache, "stored_tokens"), "keep-stored-credentials", { mode: 0o600 }); + + const result = cleanupHuggingFaceCacheData({ homeDir }); + + expect(result).toMatchObject({ ok: true }); expect(fs.existsSync(path.join(cache, "shared-model"))).toBe(false); expect(fs.readFileSync(path.join(cache, "token"), "utf8")).toBe("keep-credential"); expect(fs.readFileSync(path.join(cache, "stored_tokens"), "utf8")).toBe( @@ -392,13 +405,13 @@ describe("host-local model cleanup", () => { ); }); - it("preserves the shared Hugging Face cache unless model deletion is requested", () => { + it("preserves the shared Hugging Face cache during runtime cleanup", () => { const homeDir = temporaryHome(); const cache = path.join(homeDir, ".cache", "huggingface"); fs.mkdirSync(cache, { recursive: true }); fs.writeFileSync(path.join(cache, "shared-model"), "keep"); - const result = cleanupLocalModelRuntimes({ deleteModels: false, homeDir }); + const result = cleanupLocalModelRuntimes({ homeDir }); expect(result).toMatchObject({ ok: true }); expect(fs.existsSync(path.join(cache, "shared-model"))).toBe(true); @@ -414,7 +427,6 @@ describe("host-local model cleanup", () => { createManagedState(homeDir, harness.engine); const result = cleanupLocalModelRuntimes({ - deleteModels: false, homeDir: homeAlias, engine: harness.engine, }); @@ -435,7 +447,6 @@ describe("host-local model cleanup", () => { const harness = engineHarness(); const result = cleanupLocalModelRuntimes({ - deleteModels: false, homeDir, engine: harness.engine, }); @@ -460,7 +471,6 @@ describe("host-local model cleanup", () => { createManagedState(homeDir, harness.engine, { phase }); const result = cleanupLocalModelRuntimes({ - deleteModels: false, homeDir, engine: harness.engine, }); @@ -478,7 +488,6 @@ describe("host-local model cleanup", () => { }); const result = cleanupLocalModelRuntimes({ - deleteModels: false, homeDir, engine: harness.engine, }); @@ -497,7 +506,6 @@ describe("host-local model cleanup", () => { createManagedState(homeDir, original.engine); const result = cleanupLocalModelRuntimes({ - deleteModels: false, homeDir, engine: changed.engine, }); @@ -516,7 +524,6 @@ describe("host-local model cleanup", () => { createManagedState(homeDir, harness.engine); const result = cleanupLocalModelRuntimes({ - deleteModels: false, homeDir, engine: harness.engine, }); @@ -540,7 +547,6 @@ describe("host-local model cleanup", () => { const lease = store.acquireExecution(TRANSACTION_ID); try { const result = cleanupLocalModelRuntimes({ - deleteModels: false, homeDir, engine: harness.engine, }); @@ -565,7 +571,6 @@ describe("host-local model cleanup", () => { createManagedState(homeDir, harness.engine); const result = cleanupLocalModelRuntimes({ - deleteModels: false, homeDir, engine: harness.engine, }); diff --git a/src/lib/inference/local-model-profile/cleanup.ts b/src/lib/inference/local-model-profile/cleanup.ts index 98b4d6e31c4..e0797cb7668 100644 --- a/src/lib/inference/local-model-profile/cleanup.ts +++ b/src/lib/inference/local-model-profile/cleanup.ts @@ -63,7 +63,6 @@ interface CleanupDeps { } export interface LocalModelRuntimeCleanupOptions { - deleteModels: boolean; gatewayPort?: number; homeDir?: string; sandboxName?: string; @@ -72,6 +71,11 @@ export interface LocalModelRuntimeCleanupOptions { deps?: Partial; } +export interface HuggingFaceCacheDataCleanupOptions { + homeDir?: string; + currentUserId?: number | null; +} + export type LocalModelRuntimeCleanupResult = | { ok: true; removed: string[]; preserved: string[] } | { ok: false; reason: string; removed: string[]; preserved: string[] }; @@ -721,11 +725,28 @@ export function cleanupLocalModelRuntimes( engine: options.engine, }); } - if (options.deleteModels) { - removeSharedHuggingFaceCacheData(homeDir, deps.currentUserId, removed, preserved); - } else { - preserveSharedHuggingFaceCache(homeDir, preserved); - } + preserveSharedHuggingFaceCache(homeDir, preserved); + return { ok: true, removed, preserved }; + } catch (error) { + return { ok: false, reason: (error as Error).message, removed, preserved }; + } +} + +/** Remove non-credential Hugging Face cache data after model runtimes have stopped. */ +export function cleanupHuggingFaceCacheData( + options: HuggingFaceCacheDataCleanupOptions = {}, +): LocalModelRuntimeCleanupResult { + const removed: string[] = []; + const preserved: string[] = []; + try { + const homeDir = canonicalCleanupHomeDir(options.homeDir ?? os.homedir()); + const currentUserId = + options.currentUserId === undefined + ? typeof process.getuid === "function" + ? process.getuid() + : null + : options.currentUserId; + removeSharedHuggingFaceCacheData(homeDir, currentUserId, removed, preserved); return { ok: true, removed, preserved }; } catch (error) { return { ok: false, reason: (error as Error).message, removed, preserved }; From 7fe855f7d00b9a63b1d30b402841cd7fd3c3b6c2 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Wed, 12 Aug 2026 15:13:18 -0700 Subject: [PATCH 4/4] test(uninstall): make cache cleanup isolation observable Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- src/lib/inference/local-model-profile/cleanup-entry.ts | 4 +--- src/lib/inference/local-model-profile/cleanup.test.ts | 6 ++++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/lib/inference/local-model-profile/cleanup-entry.ts b/src/lib/inference/local-model-profile/cleanup-entry.ts index 7854a763b6a..8df7aab4caf 100644 --- a/src/lib/inference/local-model-profile/cleanup-entry.ts +++ b/src/lib/inference/local-model-profile/cleanup-entry.ts @@ -18,8 +18,6 @@ try { process.argv.at(-1) === "--delete-cache-data" ? "Hugging Face cache-data cleanup" : "Host-local model runtime cleanup"; - console.error( - `${operation} failed: ${error instanceof Error ? error.message : String(error)}`, - ); + console.error(`${operation} failed: ${error instanceof Error ? error.message : String(error)}`); process.exitCode = 1; } diff --git a/src/lib/inference/local-model-profile/cleanup.test.ts b/src/lib/inference/local-model-profile/cleanup.test.ts index 40f4f648ce6..b73636a7aac 100644 --- a/src/lib/inference/local-model-profile/cleanup.test.ts +++ b/src/lib/inference/local-model-profile/cleanup.test.ts @@ -386,6 +386,10 @@ describe("host-local model cleanup", () => { it("removes non-credential Hugging Face cache data without running runtime cleanup", () => { const homeDir = temporaryHome(); const cache = path.join(homeDir, ".cache", "huggingface"); + const runtimePaths = managedLlamaCppStatePaths(homeDir); + createManagedState(homeDir, engineHarness().engine); + const runtimeOwner = fs.readFileSync(runtimePaths.ownerPath, "utf8"); + const runtimeReceipt = fs.readFileSync(runtimePaths.receiptPath, "utf8"); fs.mkdirSync(cache, { recursive: true }); fs.writeFileSync(path.join(cache, "shared-model"), "delete"); fs.writeFileSync(path.join(cache, "token"), "keep-credential", { mode: 0o600 }); @@ -399,6 +403,8 @@ describe("host-local model cleanup", () => { expect(fs.readFileSync(path.join(cache, "stored_tokens"), "utf8")).toBe( "keep-stored-credentials", ); + expect(fs.readFileSync(runtimePaths.ownerPath, "utf8")).toBe(runtimeOwner); + expect(fs.readFileSync(runtimePaths.receiptPath, "utf8")).toBe(runtimeReceipt); expect(result.removed).toContain(`cache-contents:${cache}`); expect(result.preserved).toEqual( expect.arrayContaining([path.join(cache, "token"), path.join(cache, "stored_tokens")]),