diff --git a/.github/workflows/podman-cpu-proof.yaml b/.github/workflows/podman-cpu-proof.yaml index dd45c85d2d0..ab985add712 100644 --- a/.github/workflows/podman-cpu-proof.yaml +++ b/.github/workflows/podman-cpu-proof.yaml @@ -16,6 +16,15 @@ on: - "src/lib/onboard/managed-bootstrap/podman-*.ts" - "src/lib/onboard/experimental/portable-cpu-delegation-preflight*.ts" - "src/lib/onboard/experimental/portable-demo-lifecycle.ts" + - "src/lib/onboard/experimental/portable-runtime-*.ts" + - "src/lib/onboard/portable-retirement-authority.ts" + - "src/lib/onboard.ts" + - "src/lib/actions/uninstall/portable-runtime-cleanup.ts" + - "src/lib/actions/uninstall/all-gateway-ports.ts" + - "src/lib/actions/uninstall/run-plan.ts" + - "src/commands/internal/uninstall/run-plan.ts" + - "src/lib/state/portable-uninstall-retirement.ts" + - "src/lib/state/registry/lock.ts" - "src/lib/onboard/runtime-provider/container-state-mutation.ts" - "src/lib/onboard/runtime-provider/docker-state-mutation.ts" - "src/lib/onboard/experimental/portable-host-preparation*.ts" @@ -26,6 +35,7 @@ on: - "test/e2e/live/podman-cpu-lifecycle-helpers.ts" - "test/e2e/live/podman-cpu-lifecycle-policy.yaml" - "test/e2e/live/podman-cpu-lifecycle.test.ts" + - "test/e2e/live/podman-portable-uninstall.test.ts" - "test/e2e/registry/native-runtime-qualification.ts" - "test/e2e/support/native-runtime-qualification.test.ts" - "test/e2e/live/portable-cpu-delegation-proof.test.ts" @@ -130,8 +140,11 @@ jobs: - name: Install locked test dependencies run: npm ci --ignore-scripts - - name: Build shared sandbox-name contract - run: npm run build:policy-boundary + - name: Build and link candidate CLI + run: | + npm run build:cli + npm link --ignore-scripts + nemoclaw --version - name: Install pinned OpenShell runtime shell: bash @@ -321,8 +334,28 @@ jobs: printf 'XDG_RUNTIME_DIR=%s\n' "$runtime_dir" } >>"$GITHUB_ENV" + - name: Start the exact connected gateway required by portable retirement + shell: bash + run: | + set -euo pipefail + gateway_state="$RUNNER_TEMP/nemoclaw-podman-uninstall-gateway" + export DOCKER_HOST="unix://$E2E_PODMAN_SOCKET" + export NEMOCLAW_EXPERIMENTAL_PROFILE=portable + export NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR="$gateway_state" + { + printf 'NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR=%s\n' "$gateway_state" + printf 'OPENSHELL_LOCAL_TLS_DIR=%s/tls\n' "$gateway_state" + } >>"$GITHUB_ENV" + node --input-type=module --eval ' + const { startDockerDriverGateway } = (await import("./dist/lib/onboard.js")).default; + await startDockerDriverGateway({ skipSandboxBridgeReachability: true }); + ' + - name: Prove pinned OpenShell activation and registered-agent Podman CPU lifecycle - run: npx vitest run --project e2e-live test/e2e/live/podman-cpu-lifecycle.test.ts + run: | + npx vitest run --project e2e-live \ + test/e2e/live/podman-cpu-lifecycle.test.ts \ + test/e2e/live/podman-portable-uninstall.test.ts - name: Verify Docker stayed unavailable if: always() @@ -367,6 +400,36 @@ jobs: podman --url "$endpoint" secret ls --format '{{.Name}}' \ >"$diagnostic_dir/podman-secrets.txt" 2>&1 || true + - name: Stop the exact portable-retirement proof gateway + if: always() + shell: bash + env: + E2E_PORTABLE_GATEWAY_STOP_SCOPE: full + run: | + set -euo pipefail + gateway_state="$RUNNER_TEMP/nemoclaw-podman-uninstall-gateway" + test "${NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR:-$gateway_state}" = "$gateway_state" + export NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR="$gateway_state" + gateway_bin="$(command -v openshell-gateway)" + E2E_OPENSHELL_GATEWAY_BIN="$gateway_bin" node --input-type=module --eval ' + const { stopHostGatewayProcesses } = + await import("./dist/lib/onboard/host-gateway-process.js"); + const result = stopHostGatewayProcesses({}, { + gatewayBin: process.env.E2E_OPENSHELL_GATEWAY_BIN, + openShellGatewayName: "nemoclaw", + openShellGatewayPort: 8080, + scopedGatewayStop: process.env.E2E_PORTABLE_GATEWAY_STOP_SCOPE !== "full", + stateDir: process.env.NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR, + usePgrepFallback: false, + }); + if (result.failed.length || result.ownershipFailures?.length) { + console.error(JSON.stringify(result)); + process.exitCode = 1; + } + ' + openshell gateway remove nemoclaw + rm -rf -- "$gateway_state" + - name: Clean up rootless Podman runtime if: always() shell: bash diff --git a/SECURITY.md b/SECURITY.md index 67bd5858ac9..d069d455062 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -61,6 +61,28 @@ Subscribe to notifications on that page to receive alerts when new bulletins are Component-level threat models for security-critical NemoClaw subsystems are documented here so a reviewer or auditor can understand what each subsystem is designed to prevent, which surfaces it protects, and where its guarantees end. +### Portable Uninstall Retirement Record (`#9189`) + +**Summary.** Portable uninstall retains `~/.nemoclaw/portable-uninstall-retirement.json` after `--destroy-user-data`. +The current user owns its mode-`0700` parent and the mode-`0600` record. +A later completed onboarding removes the record only after its new authority is durable. + +**Threat.** A host crash can interrupt receipt, sandbox registry, or portable configuration removal. +Without a durable discriminator, a retry can enter generic Docker, OpenShell, or model cleanup and remove resources outside the portable receipts. + +**Guarantee.** The record keeps every retry on receipt-owned portable cleanup. +It contains a random cleanup ID, receipt basenames derived from SHA-256 hashes of sandbox names, safe relative target identities, and length-framed transaction-scoped SHA-256 content fingerprints. +The fingerprints are dictionary-testable pseudonymous local data. +The record contains no raw sandbox or gateway names, absolute paths, environment values, configuration bytes, credentials, or secrets. +NemoClaw holds one process-bound host fence across every cooperative onboarding, rebuild, and uninstall writer while it publishes or supersedes the record. + +**Where the guarantee ends.** NemoClaw state owned by the same operating-system user is not a trust boundary against a malicious process running as that user. +Such a process can change the state before or after a checked filesystem operation. +NemoClaw detects mismatched file identities and fingerprints and exits without restoring or removing the ambiguous generation. +This control covers cooperating NemoClaw processes, crashes, retries, and recycled process IDs. + +**Enforced by:** `src/lib/state/portable-uninstall-retirement.test.ts`, `src/lib/actions/uninstall/portable-runtime-cleanup.test.ts`, `src/lib/onboard/portable-resume-lock-boundary.test.ts`, and `src/lib/state/registry-lock.test.ts` cover crash boundaries, async ownership, lock generations, record contents, and completed-onboarding supersession. + ### Ollama Auth Proxy Loopback Bind Probe (`#6014`) **Summary.** The Ollama auth proxy is the token-authenticated network gate in front of a locally-running Ollama backend on every topology where `shouldFrontOllamaWithProxy()` returns true (native Linux, macOS, WSL with a native dockerd runtime). Ollama itself has no built-in authentication. The proxy adds a bearer-token check on its own listen port and forwards to Ollama on the backend port. diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index f29f248e4fa..fc0295c448a 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -26,7 +26,7 @@ "src/lib/onboard/gateway-binding.ts": 52, "src/lib/runner.ts": 87, "src/lib/security/redact.ts": 52, - "src/lib/state/onboard-session.ts": 36, + "src/lib/state/onboard-session.ts": 37, "src/lib/state/registry.ts": 101, "src/lib/state/state-root.ts": 21, "src/lib/subprocess-env.ts": 24, @@ -43,14 +43,13 @@ "src/lib/actions/sandbox/status-snapshot.ts": 21, "src/lib/actions/sandbox/policy-channel.ts": 30, "src/lib/actions/sandbox/process-recovery.ts": 21, - "src/lib/actions/sandbox/rebuild-pipeline.ts": 28, + "src/lib/actions/sandbox/rebuild-pipeline.ts": 29, "src/lib/actions/sandbox/snapshot.ts": 40, - "src/lib/actions/uninstall/run-plan.ts": 26, - + "src/lib/actions/uninstall/run-plan.ts": 25, "src/lib/inference/local.ts": 21, "src/lib/inference/onboard-probes.ts": 21, "src/lib/inference/vllm.ts": 21, - "src/lib/onboard.ts": 202, + "src/lib/onboard.ts": 203, "src/lib/onboard/machine/handlers/sandbox.ts": 21, "src/lib/sandbox/config.ts": 22, "src/lib/shields/index.ts": 23 @@ -58,10 +57,10 @@ }, "allowedCycles": [], "maxRootFiles": { - "src/lib/onboard": 307, + "src/lib/onboard": 308, "src/lib/actions": 19, "src/lib/actions/sandbox": 183, - "src/lib/state": 38, + "src/lib/state": 39, "src/lib/inference": 63, "scripts": 47 } diff --git a/docs/manage-sandboxes/uninstall-nemoclaw.mdx b/docs/manage-sandboxes/uninstall-nemoclaw.mdx index d5096882598..32c0f687604 100644 --- a/docs/manage-sandboxes/uninstall-nemoclaw.mdx +++ b/docs/manage-sandboxes/uninstall-nemoclaw.mdx @@ -60,6 +60,7 @@ Resolve the reported error, inspect the remaining models and runtimes, and rerun Without `--delete-models`, uninstall preserves both model stores. When sibling gateway environments remain, uninstall preserves both model stores even if you pass `--delete-models`. An all-gateway-port sweep deletes them only after every selected gateway cleanup succeeds and no sibling environment remains. +For a full uninstall of the portable experimental profile, `--delete-models` preserves every Podman image. The flag does not scan arbitrary directories or delete model caches on remote peers. ## Uninstall Every Gateway Port @@ -91,12 +92,13 @@ $$nemoclaw uninstall --all-gateway-ports When the sweep finds more than one port, it lists the ports, confirms once, then uninstalls each port in turn and the currently selected port last. When it finds only the selected port, it uses the standard uninstall confirmation and runs that port once. -Running the selected port last lets that final pass remove the shared CLI, services, images, providers, configuration, models, and swap once no other environment needs them. +Outside portable cleanup, running the selected port last lets that final pass remove the shared CLI, services, images, providers, configuration, models, and swap once no other environment needs them. `--delete-models`, `--destroy-user-data`, and `--keep-openshell` apply to every port in the sweep. If the sweep cannot enumerate the gateway state roots safely, it exits before uninstalling any port. The sweep cannot select an unidentified environment until its gateway port can be determined. If one port fails to uninstall, the sweep reports that port, continues with the remaining ports, and exits nonzero. The failed port still counts as a live environment, so the final pass keeps the shared host resources instead of removing state that the surviving environment needs. +For the portable experimental profile, this preservation includes the shared Podman resources, current-user selectors, `~/.config/nemoclaw/portable/containers.conf`, and lifecycle receipts. The sweep does not roll back cleanup that completed before a failure. Resolve the reported error, inspect the remaining gateways with `openshell gateway list`, and rerun the sweep or the reported per-port command. Default-port uninstall removes NemoClaw-managed entries in `openshell/gateway.env`. @@ -140,7 +142,56 @@ Fresh state roots and already scoped configurations continue to use the state-ro The legacy `default` namespace is not isolated across gateways, so it cannot satisfy the scoped-uninstall proof while sibling gateways remain. Scoped uninstall stops before it deletes a sandbox, registry row, or gateway registration and preserves the selected gateway's runtime evidence and local state. Because the supported OpenShell Podman schema does not expose `sandbox_namespace`, scoped Podman uninstall fails closed before signaling and preserves the gateway runtime evidence and local state. -Full single-gateway Podman uninstall continues to use normal graceful teardown. +A full uninstall for the portable experimental profile detects ownership only from durable lifecycle state. +It records the initial lifecycle receipts and sandbox registry state for comparison. +NemoClaw first acquires the process-bound portable host lock at `~/.nemoclaw-portable-host.lock`. +This host-wide fence coordinates portable onboarding, rebuild, and uninstall. +It then acquires every receipt lifecycle lock in sorted order before it acquires the process-bound sandbox registry lock. +Portable lock acquisition fails before cleanup unless NemoClaw can establish the current Linux process's exact boot identity and process start tick. +A live lock owner with that exact identity is never displaced because of the lock's age. +A contender waits only for the bounded acquisition interval and exits nonzero if the lock remains held. +With every lock held, NemoClaw verifies that the recorded receipts and registry state have not changed. +This one-to-one ownership check applies only to NemoClaw's host `sandboxes.json`. +Each receipt must have one current matching row in that file, and the file must contain no extra sandbox row. +Missing, mismatched, or extra rows make NemoClaw exit nonzero before it changes a container or host state. +An unrelated row, including a non-portable row, makes ownership ambiguous and triggers this fail-closed result before any effect, even with `--destroy-user-data`. +An unrelated sandbox registered only with OpenShell is outside that host registry and remains usable. +NemoClaw holds those same sorted lifecycle locks and the process-bound registry lock throughout exact Podman container removal, exact-name OpenShell sandbox deletion, portable registry removal, selector cleanup, and the final retirement operation for the receipts, sandbox registry rows, and `~/.config/nemoclaw/portable/containers.conf`. +Before it retires `~/.config/nemoclaw/portable/containers.conf` or lifecycle receipts, NemoClaw revalidates the current user's local Podman socket authority and lifecycle generation. +It also revalidates the exact receipt-owned sandbox containers, their expected OpenShell labels and current running state, and the managed portable registry container. +NemoClaw uses the pinned local Podman socket to remove only the exact receipt-owned sandbox containers. +Before every exact receipt-name OpenShell sandbox deletion, NemoClaw runs `openshell status -g ` to prove that the gateway name recorded by the matching registry row is connected. +It then runs `openshell sandbox delete -g ` for that exact gateway and sandbox name. +After deletion, bounded `openshell sandbox get -g ` probes must verify that the exact sandbox is absent. +It never relies on or switches the globally selected gateway, and it never falls back to implicit deletion. +A same-name sandbox on another gateway remains untouched. +It never uses `openshell sandbox delete --all` during portable OpenShell cleanup. +If explicit deletion reports an exact receipt sandbox as absent, cleanup continues only because the gateway-scoped status probe proved that exact gateway is connected and reachable. +A missing gateway, missing provider, transport failure, verification timeout, generic or ambiguous `NotFound` result, or absence that the status probe cannot prove is fatal. +Uninstall exits nonzero and preserves the portable registry row and container, selectors, receipts, `~/.config/nemoclaw/portable/containers.conf`, gateway state, shared OpenShell provider and gateway registrations, and unrelated OpenShell-only sandboxes. +Portable cleanup retires only the exact receipt-owned portable resources. +It preserves model stores and images, shared OpenShell provider and gateway registrations, the gateway service and process, shared OpenShell binaries and configuration, and unrelated OpenShell-registered sandboxes. +Before it removes the portable registry container or selectors, NemoClaw verifies that the original lifecycle receipts and sandbox registry state have not changed. +If either changed, uninstall exits nonzero before later gateway-state or `~/.config/nemoclaw/portable/containers.conf` deletion. +After that verification, NemoClaw removes the prevalidated portable registry container by its exact ID. +It unsets the current-user systemd manager values that the portable profile installs, `CONTAINERS_CONF` and `NETAVARK_FW`, only when their current values match NemoClaw's recorded or derived portable projection; it inspects and preserves `CONTAINER_HOST`, `CONTAINER_CONNECTION`, and `CONTAINER_SSHKEY` because the portable profile does not own those values. +NemoClaw preserves unrelated or changed selector values. +It skips the generic Docker availability probe and all generic Docker container, image, and volume cleanup. +It does not remove the user's Podman installation, storage, networks, or unrelated containers. +It does not disable a user-managed socket. +On every successful portable cleanup, the final retirement operation removes the exact portable lifecycle receipts, their matching `sandboxes.json` rows, and `~/.config/nemoclaw/portable/containers.conf`, regardless of `--destroy-user-data`. +Other preserved user data follows the normal `--destroy-user-data` behavior. +After NemoClaw releases the locks, later uninstall-plan cleanup never recursively revisits the canonical receipt, sandbox registry, or `~/.config/nemoclaw/portable/containers.conf` paths. +This preserves any new lifecycle generation published after lock release. +After successful portable cleanup with `--destroy-user-data`, the sole residual regular file under `~/.nemoclaw/` is `~/.nemoclaw/portable-uninstall-retirement.json`, written at mode `0600` under a current-UID-owned, mode-`0700` `~/.nemoclaw/` directory. +Another same-user process can change the record, so this protection assumes cooperative NemoClaw writers under that UID. +NemoClaw retains the record until a later completed onboarding durably supersedes it. +It contains a random local cleanup ID, receipt basenames derived from SHA-256 hashes of sandbox names, and length-framed, transaction-scoped, dictionary-testable pseudonymous fingerprints, but no raw sandbox or gateway names, absolute paths, environment values, configuration bytes, or secrets. +A repeat invocation remains on portable cleanup and performs no Docker probe or mutation, implicit OpenShell cleanup, or model cleanup. +If NemoClaw cannot prove the required portable authority or complete a cleanup step, uninstall exits nonzero. +It preserves any existing durable retirement record and any remaining receipts, `~/.config/nemoclaw/portable/containers.conf`, gateway state, and other retry evidence, even with `--destroy-user-data`. +Resolve the reported authority or cleanup error, keep the preserved state intact, and rerun the same full uninstall command. +This cleanup does not change Docker uninstall, externally supervised gateways, `--keep-openshell`, scoped sibling-gateway cleanup, or non-portable profiles. For Docker, if any proof is absent, uninstall exits nonzero before it signals the host gateway. NemoClaw preserves the gateway runtime evidence and local state. Keep that state intact. @@ -164,7 +215,7 @@ Do not add `sandbox_namespace` manually to a live gateway configuration because In this section, `` is `~/.nemoclaw/` for the default gateway or `~/.nemoclaw/gateways//` for a non-default gateway. For the default gateway, the uninstall command preserves `~/.nemoclaw/rebuild-backups/`, `~/.nemoclaw/backups/`, and `~/.nemoclaw/sandboxes.json` by default. A non-default gateway uses the corresponding entries under `~/.nemoclaw/gateways//`. -When uninstall confirms that no sibling gateways remain, it also removes the shared CLI, services, images, providers, configuration, models, and swap. +Outside portable cleanup, when uninstall confirms that no sibling gateways remain, it also removes the shared CLI, services, images, providers, configuration, models, and swap. During full uninstall, a managed distributed-vLLM receipt makes NemoClaw first revalidate the recorded plan and every cluster node, GPU, container, and SSH peer identity. It then removes only the exact receipt-owned containers. The managed-cluster receipt is host-global under `~/.nemoclaw/`. @@ -196,7 +247,7 @@ For non-interactive runs using `--yes`, `NEMOCLAW_NON_INTERACTIVE=1`, or a non-T `--yes` stays non-destructive by design and never purges preserved user data on its own. After a successful or already-absent sandbox deletion, preserving other `sandboxes.json` entries does not preserve the selected sandbox entry or its gateway registration. -After uninstall confirms that no sibling gateways remain, it also removes the provider registrations and Docker image that the recorded sandboxes depend on. +Outside portable cleanup, after uninstall confirms that no sibling gateways remain, it also removes the provider registrations and Docker image that the recorded sandboxes depend on. Uninstall warns that those records cannot be recovered automatically on reinstall, and the remediation is `$$nemoclaw destroy` followed by `$$nemoclaw onboard`. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index aa56e91fc1f..7bc36c4bfd6 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -4252,7 +4252,7 @@ $$nemoclaw gc [--dry-run] [--yes|-y|--force] ### `$$nemoclaw uninstall` -Run `uninstall.sh` to remove NemoClaw sandboxes, local gateway resources, related images and containers, and local state. +Run `uninstall.sh` to uninstall NemoClaw. Unless this section explicitly describes portable cleanup, its resource-removal statements apply outside portable cleanup. The CLI runs the local `uninstall.sh` shipped with the installed npm package. If that local script is missing, the CLI does not auto-fetch a remote copy. It prints the versioned URL of the matching `uninstall.sh` so you can download, review, and run it manually. @@ -4272,7 +4272,7 @@ If Docker is unavailable or a resource does not match its persisted ownership st Host-local cleanup can partially complete before an error. Restore Docker access or resolve the named ownership conflict, inspect the remaining container and network, and retry uninstall. Managed llama.cpp and vLLM model files remain in the shared Hugging Face cache by default. -With `--delete-models`, uninstall deletes every model in the local Ollama inventory and all non-credential data in the current user's shared `~/.cache/huggingface/` cache. +Outside portable cleanup, `--delete-models` deletes every model in the local Ollama inventory and all non-credential data in the current user's shared `~/.cache/huggingface/` cache. This opt-in can delete cached files that other applications installed or use. It preserves the Hugging Face `token` and `stored_tokens` authentication files. NemoClaw stops and verifies its managed local and distributed model runtimes before it deletes non-credential data from the local Hugging Face cache. @@ -4287,16 +4287,16 @@ Refer to [Declare the OpenShell Gateway Lifecycle Authority](../deployment/gatew -Uninstall also stops any orphaned `openshell` host processes left behind by previous onboard or destroy cycles, including `openshell sandbox create`, `openshell ssh-proxy`, and SSH sessions spawned by OpenShell. +Outside portable cleanup, uninstall also stops any orphaned `openshell` host processes left behind by previous onboard or destroy cycles, including `openshell sandbox create`, `openshell ssh-proxy`, and SSH sessions spawned by OpenShell. Earlier releases only stopped `openshell forward` processes, so those orphans accumulated across runs. -For Local Ollama setups, uninstall also stops matching Ollama auth proxy processes before deleting `~/.nemoclaw` state so stale proxy listeners do not block a later reinstall. +Outside portable cleanup, uninstall also stops matching Ollama auth proxy processes before deleting `~/.nemoclaw` state so stale proxy listeners do not block a later reinstall. When sibling gateways remain, uninstall leaves the shared proxy running for them. For Hermes setups, uninstall inspects the selected gateway's managed port-forward watcher state, stops each verified watcher process and its sandbox-scoped forward, and leaves sibling gateway state untouched. If any watcher or forward cleanup cannot be confirmed, uninstall exits nonzero and preserves the selected gateway's watcher state so you can retry cleanup. -On Linux, uninstall removes `~/.local/state/nemoclaw` unless you pass `--keep-openshell`, the gateway is externally supervised, or another gateway-port environment remains on the host. +Outside portable cleanup, Linux uninstall removes `~/.local/state/nemoclaw` unless you pass `--keep-openshell`, the gateway is externally supervised, or another gateway-port environment remains on the host. That directory contains NemoClaw-owned Docker-driver gateway configuration and SQLite data, audit logs, VM-driver state, and standalone-fallback gateway PID files. Uninstall preserves it when the managed or externally supervised gateway process remains because that process depends on the state. When another gateway-port environment remains, uninstall removes only the selected gateway port's subdirectory of that directory and keeps the other ports' subdirectories. @@ -4308,8 +4308,8 @@ Uninstall does not otherwise target the declared external directory. |---|---| | `--yes` | Skip the confirmation prompt | | `--keep-openshell` | Leave OpenShell binaries, NemoClaw-managed gateway service files, and local gateway state in place, and do not stop the host gateway process | -| `--delete-models` | Delete every model reported by the host's local Ollama inventory and all non-credential data in the current user's shared `~/.cache/huggingface/` cache after managed model runtimes stop. Hugging Face authentication files remain. | -| `--destroy-user-data` | Also remove preserved user data (`rebuild-backups/`, `backups/`, `sandboxes.json`) | +| `--delete-models` | Outside portable cleanup, delete every model reported by the host's local Ollama inventory and all non-credential data in the current user's shared `~/.cache/huggingface/` cache after managed model runtimes stop. Hugging Face authentication files remain. Portable cleanup preserves both model stores and every Podman image. | +| `--destroy-user-data` | Also remove preserved user data (`rebuild-backups/`, `backups/`, `sandboxes.json`). Portable cleanup still retains its recovery record until later completed onboarding durably supersedes it. | | `--all-gateway-ports` | Uninstall every gateway port on the host, not only the port `NEMOCLAW_GATEWAY_PORT` selects | | `--gateway ` | Optional consistency check; must match the name derived from `NEMOCLAW_GATEWAY_PORT` | @@ -4362,7 +4362,7 @@ Fresh state roots and already scoped configurations continue to use the state-ro The legacy `default` namespace is not isolated across gateways, so it cannot satisfy the scoped-uninstall proof while sibling gateways remain. Scoped uninstall stops before it deletes a sandbox, registry row, or gateway registration and preserves the selected gateway's runtime evidence and local state. Because the supported OpenShell Podman schema does not expose `sandbox_namespace`, scoped Podman uninstall fails closed before signaling and preserves the gateway runtime evidence and local state. -Full single-gateway Podman uninstall continues to use normal graceful teardown. +Full single-gateway Podman uninstall for the portable experimental profile uses exact receipt-owned durable retirement and preserves shared OpenShell providers, the gateway service and process, binaries, configuration, model stores, and Podman images under the [portable authority, retry, and preservation contract](../manage-sandboxes/operate-sandboxes/uninstall-nemoclaw). For Docker, if any proof is absent, uninstall exits nonzero before it signals the host gateway. NemoClaw preserves the gateway runtime evidence and local state. Keep that state intact. @@ -4393,8 +4393,8 @@ The sweep enumerates the default state root and the non-default roots under `~/. When the sweep finds more than one port, it confirms once against the resulting port list, then uninstalls each other port before the port `NEMOCLAW_GATEWAY_PORT` selects. When it finds only the selected port, it uses the standard uninstall confirmation without a port list and runs that port once. Each port runs as its own uninstall so that every port-scoped value, including the state root, registry file, gateway name, and Docker resource names, resolves from that port rather than from the calling environment. -The selected port runs last so its pass can remove the shared host resources once no other environment remains. -`--delete-models`, `--destroy-user-data`, and `--keep-openshell` apply to every port; `--gateway` remains a check against the selected port only. +Outside portable cleanup, the selected port runs last so its pass can remove the shared host resources once no other environment remains. +`--delete-models`, `--destroy-user-data`, and `--keep-openshell` apply to every port, subject to the portable preservation contract; `--gateway` remains a check against the selected port only. A failure to enumerate the gateway state roots safely stops the sweep before any port uninstall begins. The sweep cannot select an unidentified environment until its gateway port can be determined. A port that fails to uninstall is reported, the sweep continues, and the exit code is nonzero. @@ -4413,7 +4413,7 @@ The default gateway uses `~/.nemoclaw/`; a non-default gateway uses `~/.nemoclaw | `backups/` | Host-side workspace backups that `scripts/backup-workspace.sh` writes. Refer to [Transfer State Manually](../manage-sandboxes/state-and-backups/transfer-state-manually). | | `sandboxes.json` | Host-side sandbox registry. NemoClaw uses it to map sandbox names back to their persistence directories when you reinstall. | -When uninstall confirms that no sibling gateways remain, it also removes shared host resources such as the gateway source clone, runtime state, and the Ollama auth proxy PID file. +Outside portable cleanup, when uninstall confirms that no sibling gateways remain, it also removes shared host resources such as the gateway source clone, runtime state, and the Ollama auth proxy PID file. When sibling gateways remain, it removes only the selected gateway's resources and port-scoped state while preserving those shared host resources. If the OpenShell command is unavailable or its gateway list cannot be read, uninstall cannot confirm that the selected gateway is the last one, so it uses the same scoped path and preserves the shared resources. When the command itself is unavailable, uninstall exits nonzero before OpenShell cleanup so you can restore the command and retry. @@ -4428,7 +4428,7 @@ Decision matrix: | Context | Behaviour | |---|---| | Interactive TTY, preserved entries present, no env override | Prompts `Also remove them? [y/N]`. Default `N` keeps the entries. | -| Interactive TTY, user answers `y` | Removes the preserved entries in the selected gateway's state root; a single-gateway uninstall also removes the remaining shared state. | +| Interactive TTY, user answers `y` | Removes the preserved entries in the selected gateway's state root; outside portable cleanup, a single-gateway uninstall also removes the remaining shared state. Portable cleanup preserves shared OpenShell resources and its retirement record. | | Non-interactive (`--yes`, `NEMOCLAW_NON_INTERACTIVE=1`, or non-TTY shell) | Preserves the entries and prints a one-line notice. | | `--destroy-user-data` | Skips the secondary user-data prompt and removes the preserved entries in the selected gateway's state root. The global `Proceed?` confirmation still applies unless `--yes` is also passed. | | `NEMOCLAW_UNINSTALL_DESTROY_USER_DATA=1` | Skips the secondary user-data prompt and removes the preserved entries. The global `Proceed?` confirmation still applies unless `--yes` is also passed. | @@ -4438,8 +4438,8 @@ Reinstall NemoClaw and re-onboard the sandbox before `$$nemoclaw snapshot The preserved `sandboxes.json` file does not make the recorded sandboxes recoverable on its own. Uninstall deletes the selected sandboxes and attempts to remove the local gateway registration. -After uninstall confirms that no sibling gateways remain, it also deletes provider registrations. -For a NemoClaw-managed gateway, it also removes the Docker image. +Outside portable cleanup, after uninstall confirms that no sibling gateways remain, it also deletes provider registrations. +Outside portable cleanup, a NemoClaw-managed gateway also removes the Docker image. For an externally supervised gateway, it preserves Docker resources, but the registry still cannot recover deleted sandbox and provider resources. Uninstall warns about this at preserve time. After reinstalling, the installer reports such records as not found on their recorded gateway instead of claiming they were recovered; run `$$nemoclaw destroy` to clear a stranded record, then `$$nemoclaw onboard` to rebuild it. diff --git a/docs/reference/host-files-and-state.mdx b/docs/reference/host-files-and-state.mdx index 138795d3a3b..fff074ddff9 100644 --- a/docs/reference/host-files-and-state.mdx +++ b/docs/reference/host-files-and-state.mdx @@ -28,11 +28,13 @@ Share redacted diagnostics only. ## Files +Unless a row says otherwise, full-uninstall removal behavior in this table applies outside portable cleanup. | Path | Purpose | Safe to delete | |---|---|---| | `~/.nemoclaw/config.json` | Host-level CLI configuration and defaults created by onboarding or config commands. | Only if you want NemoClaw to forget host defaults and rebuild them on the next setup. | | `~/.nemoclaw/credentials.json` | Legacy plaintext credential file from earlier releases. Onboarding stages allowlisted values, registers them with the OpenShell gateway, and securely deletes the file only after verifying their migration. Current releases do not create this file. | No; run `$$nemoclaw onboard` to complete migration and cleanup so you do not lose a credential that is not yet registered with the gateway. | | `~/.nemoclaw/sandboxes.json` | Current sandbox registry used by `$$nemoclaw list`, default sandbox selection, rebuild, and recovery commands. See [Malformed Registry File](#malformed-registry-file) when a command reports that this file is not valid JSON. | No. Deleting it makes the host forget existing sandboxes and can block state-preserving recovery. | +| `~/.nemoclaw/portable-uninstall-retirement.json` | Current-user-owned portable cleanup recovery authority, written at mode `0600` under the current-user-owned, mode-`0700` `~/.nemoclaw/` directory. It binds exact cleanup targets and remains after cleanup until a later completed onboarding durably supersedes it. Its hashes are dictionary-testable pseudonymous fingerprints, not secrets. | No while cleanup, retry, or supersession recovery may be needed. Do not remove it manually; later completed onboarding retires it. | | `~/.nemoclaw/onboard-session.json` | Resume marker for an onboarding attempt that failed before completion. | Yes, when you intentionally want to discard the failed session and start over. Prefer `$$nemoclaw onboard --fresh` when available. | | `~/.nemoclaw/usage-notice.json` | Records the third-party software notice version in `acceptedVersion` and the acceptance time in `acceptedAt`. Install, onboarding, and rebuild flows consult this file and prompt again when its recorded version differs from the current notice or the file is absent. | Yes; deleting it makes the next applicable install, onboarding, or rebuild flow prompt for acceptance again. | | `~/.nemoclaw/ollama-proxy-token` | Host-global auth token used by the host-side Ollama auth proxy. Every gateway port on the host uses this one file. | Yes, but rerun onboarding for every gateway port that uses Local Ollama afterward, because a new token invalidates the shared proxy for all of them. | @@ -123,7 +125,7 @@ Listing works on native Windows, but migration snapshot creation and deletion do ## Uninstall Behavior -`$$nemoclaw uninstall --yes` removes active NemoClaw runtime resources but preserves the user data needed for recovery by default. +Outside portable cleanup, `$$nemoclaw uninstall --yes` removes active NemoClaw runtime resources but preserves the user data needed for recovery by default. Preserved entries include `rebuild-backups/`, `backups/`, and `sandboxes.json`. Preserved `sandboxes.json` records are not automatically recoverable after reinstall, because uninstall removes the gateway registration, provider registrations, and Docker image they reference; uninstall warns about this at preserve time, and a later reinstall reports such records as not found on their recorded gateway with `$$nemoclaw destroy` / `$$nemoclaw onboard` remediation. Interactive uninstall prompts before removing preserved state. @@ -132,14 +134,16 @@ For non-interactive runs, pass `--destroy-user-data` only when you accept losing Uninstall scopes cleanup to one gateway port: it operates on the selected state root and leaves every other `~/.nemoclaw/gateways//` root and its gateway in place. It names the environments it left behind and prints the command that removes one of them. Pass `--all-gateway-ports`, or set `NEMOCLAW_UNINSTALL_ALL_GATEWAY_PORTS=1`, to remove every gateway port in one run. -The shared host state is eligible for removal only when `--keep-openshell` is absent, no port fails or survives, and the selected gateway is not externally supervised. +Outside portable cleanup, the shared host state is eligible for removal only when `--keep-openshell` is absent, no port fails or survives, and the selected gateway is not externally supervised. + +Portable cleanup instead retires only exact receipt-owned portable resources. It preserves model stores, Podman images, shared OpenShell providers and gateway registrations, the gateway service and process, and shared OpenShell binaries and configuration. It retains `~/.nemoclaw/portable-uninstall-retirement.json` even with `--destroy-user-data` so a retry or later completed onboarding can finish recovery. Before full uninstall deletes shared state, it removes only an exact authenticated host-local vLLM container and exact NemoClaw-owned llama.cpp container and network. Catalog-selected host-local vLLM also requires its exact owner-only runtime receipt; legacy authenticated containers without catalog profile labels retain their existing cleanup behavior. It refuses the remaining uninstall steps when Docker is unavailable or persisted ownership cannot be proved. Resolve the reported Docker or ownership error and rerun uninstall with the state still present. The shared Hugging Face cache used by managed vLLM and managed llama.cpp remains in place by default. -With `--delete-models`, full uninstall deletes every model in the local Ollama inventory and all non-credential data in the current user's shared Hugging Face cache. +Outside portable cleanup, `--delete-models` deletes every model in the local Ollama inventory and all non-credential data in the current user's shared Hugging Face cache. The flag can delete cached files that other applications installed or use. It preserves the Hugging Face `token` and `stored_tokens` authentication files. NemoClaw deletes the non-credential cache data only after managed model runtimes stop and only when no sibling gateway remains. diff --git a/src/commands/internal/uninstall/run-plan.ts b/src/commands/internal/uninstall/run-plan.ts index 5ee8b54ad42..0201d195320 100644 --- a/src/commands/internal/uninstall/run-plan.ts +++ b/src/commands/internal/uninstall/run-plan.ts @@ -6,7 +6,7 @@ import { allGatewayPortsRequested, runUninstallAllGatewayPorts, } from "../../../lib/actions/uninstall/all-gateway-ports"; -import { runUninstallPlan } from "../../../lib/actions/uninstall/run-plan"; +import { runUninstallPlanProduction } from "../../../lib/actions/uninstall/run-plan"; import { CLI_DISPLAY_NAME, CLI_NAME } from "../../../lib/cli/branding"; import { NemoClawCommand } from "../../../lib/cli/nemoclaw-oclif-command"; import { GATEWAY_PORT } from "../../../lib/core/ports"; @@ -53,11 +53,11 @@ export default class InternalUninstallRunPlanCommand extends NemoClawCommand { keepOpenShell: flags["keep-openshell"] ?? false, }; if (allGatewayPortsRequested(flags["all-gateway-ports"], process.env)) { - this.applyExitResult(runUninstallAllGatewayPorts(options)); + this.applyExitResult(await runUninstallAllGatewayPorts(options)); return; } this.applyExitResult( - runUninstallPlan(options, { + await runUninstallPlanProduction(options, { requireCompleteGatewayProcessCleanup: flags["all-gateway-ports-child"] ?? false, }), ); diff --git a/src/lib/actions/sandbox/rebuild-pipeline.ts b/src/lib/actions/sandbox/rebuild-pipeline.ts index 3f1c140de4c..dd9b8664ad0 100644 --- a/src/lib/actions/sandbox/rebuild-pipeline.ts +++ b/src/lib/actions/sandbox/rebuild-pipeline.ts @@ -1,6 +1,9 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import os from "node:os"; +import path from "node:path"; + import type { RebuildSandboxOptions } from "../../domain/lifecycle/options"; import { normalizeRebuildSandboxOptions } from "../../domain/lifecycle/options"; import { BRAVE_API_KEY_ENV, TAVILY_API_KEY_ENV } from "../../inference/web-search"; @@ -8,9 +11,10 @@ import { MESSAGING_SETUP_APPLIER_ENV_KEY } from "../../messaging/applier/types"; import { MESSAGING_CHANNEL_CONFIG_ENV_KEYS } from "../../messaging-channel-config"; import { hydrateCredentialEnv } from "../../onboard/credential-env"; import { DOCKER_GPU_PATCH_NETWORK_ENV } from "../../onboard/docker-gpu-patch"; +import { withPortableOnboardRetirementBoundary } from "../../onboard/portable-retirement-authority"; import { withMcpLifecycleLock } from "../../state/mcp-lifecycle-lock"; import * as onboardSession from "../../state/onboard-session"; -import { load as loadRegistry } from "../../state/registry/persistence"; +import { load as loadRegistry, REGISTRY_FILE } from "../../state/registry/persistence"; import { normalizeRebuildTargetPolicyPresets, runRebuildBackupPhase } from "./rebuild-backup-phase"; import { buildRefreshMutableOpenClawConfigHashCommand } from "./rebuild-config-hash"; import { DCODE_AGENT_NAME } from "./rebuild-dcode-target"; @@ -80,7 +84,15 @@ export async function rebuildSandbox( options: string[] | RebuildSandboxOptions = {}, opts: RebuildSandboxExecutionOptions = {}, ): Promise { - return withMcpLifecycleLock(sandboxName, async () => { + const homeDir = process.env.HOME || os.homedir(); + return withPortableOnboardRetirementBoundary( + { + homeDir, + registryFile: REGISTRY_FILE, + sessionFile: onboardSession.SESSION_FILE, + stateDir: path.dirname(onboardSession.SESSION_FILE), + }, + () => withMcpLifecycleLock(sandboxName, async () => { const scopedEnvKeys = [ BRAVE_API_KEY_ENV, TAVILY_API_KEY_ENV, @@ -102,7 +114,9 @@ export async function rebuildSandbox( ), ); } - }); + }), + { loadRegistry, withLifecycleLock: withMcpLifecycleLock }, + ); } async function rebuildSandboxUnlocked( diff --git a/src/lib/actions/uninstall/all-gateway-ports.test.ts b/src/lib/actions/uninstall/all-gateway-ports.test.ts index 1de807cde12..e0e9ee47301 100644 --- a/src/lib/actions/uninstall/all-gateway-ports.test.ts +++ b/src/lib/actions/uninstall/all-gateway-ports.test.ts @@ -30,7 +30,7 @@ afterEach(() => { function sweepDeps(overrides: AllGatewayPortsDeps = {}) { const error = vi.fn(); const runPortPass = vi.fn((_port: number) => 0); - const runSelectedPass = vi.fn((_options: UninstallRunOptions, _deps: UninstallRunDeps) => ({ + const runSelectedPass = vi.fn(async (_options: UninstallRunOptions, _deps: UninstallRunDeps) => ({ exitCode: 0, })); const deps: AllGatewayPortsDeps = { @@ -56,10 +56,10 @@ describe("uninstall across every gateway port (#7791)", () => { expect(allGatewayPortsRequested(flag, env as NodeJS.ProcessEnv)).toBe(expected); }); - it("uninstalls each other gateway port in ascending order and the current port last", () => { + it("uninstalls each other gateway port in ascending order and the current port last", async () => { const { deps, runPortPass, runSelectedPass } = sweepDeps(); - const result = runUninstallAllGatewayPorts(OPTIONS, deps); + const result = await runUninstallAllGatewayPorts(OPTIONS, deps); expect(result.exitCode).toBe(0); expect(result.ports).toEqual([9000, 18080, 8080]); @@ -67,32 +67,35 @@ describe("uninstall across every gateway port (#7791)", () => { expect(runSelectedPass).toHaveBeenCalledTimes(1); }); - it("forces confirmation-free child passes after one whole-host confirmation", () => { + it("forces confirmation-free child passes after one whole-host confirmation", async () => { const { deps, runSelectedPass } = sweepDeps({ readLine: () => "y" }); - runUninstallAllGatewayPorts({ ...OPTIONS, assumeYes: false }, deps); + await runUninstallAllGatewayPorts({ ...OPTIONS, assumeYes: false }, deps); expect(runSelectedPass.mock.calls[0]?.[0]).toMatchObject({ assumeYes: true, }); }); - it("does not run any pass when the whole-host confirmation is declined", () => { + it("does not run any pass when the whole-host confirmation is declined", async () => { const { deps, runPortPass, runSelectedPass } = sweepDeps({ readLine: () => "n", }); - const result = runUninstallAllGatewayPorts({ ...OPTIONS, assumeYes: false }, deps); + const result = await runUninstallAllGatewayPorts({ ...OPTIONS, assumeYes: false }, deps); expect(result.exitCode).toBe(0); expect(runPortPass).not.toHaveBeenCalled(); expect(runSelectedPass).not.toHaveBeenCalled(); }); - it("rejects a mismatched gateway check before any port pass", () => { + it("rejects a mismatched gateway check before any port pass", async () => { const { deps, error, runPortPass, runSelectedPass } = sweepDeps(); - const result = runUninstallAllGatewayPorts({ ...OPTIONS, gatewayName: "nemoclaw-9000" }, deps); + const result = await runUninstallAllGatewayPorts( + { ...OPTIONS, gatewayName: "nemoclaw-9000" }, + deps, + ); expect(result).toEqual({ exitCode: 1, ports: [] }); expect(runPortPass).not.toHaveBeenCalled(); @@ -100,13 +103,13 @@ describe("uninstall across every gateway port (#7791)", () => { expect(error).toHaveBeenCalledWith(expect.stringContaining("Refusing to uninstall gateway")); }); - it("reports a failed port pass and still finishes the remaining ports", () => { + it("reports a failed port pass and still finishes the remaining ports", async () => { const failingPortPass = vi.fn((port: number) => (port === 9000 ? 1 : 0)); const { deps, error, runSelectedPass } = sweepDeps({ runPortPass: failingPortPass, }); - const result = runUninstallAllGatewayPorts(OPTIONS, deps); + const result = await runUninstallAllGatewayPorts(OPTIONS, deps); expect(result.exitCode).toBe(1); expect(failingPortPass.mock.calls.map(([port]) => port)).toEqual([9000, 18080]); @@ -114,27 +117,27 @@ describe("uninstall across every gateway port (#7791)", () => { expect(error).toHaveBeenCalledWith(expect.stringContaining("9000")); }); - it("carries a failed port into the final pass so shared cleanup stays scoped (#7791)", () => { + it("carries a failed port into the final pass so shared cleanup stays scoped (#7791)", async () => { const { deps, runSelectedPass } = sweepDeps({ runPortPass: vi.fn((port: number) => (port === 9000 ? 1 : 0)), }); - runUninstallAllGatewayPorts(OPTIONS, deps); + await runUninstallAllGatewayPorts(OPTIONS, deps); expect(runSelectedPass.mock.calls[0]?.[1]).toMatchObject({ retainedGatewayPorts: [9000], }); }); - it("returns nonzero when the final pass still observes another gateway environment", () => { + it("returns nonzero when the final pass still observes another gateway environment", async () => { const { deps, error } = sweepDeps({ - runSelectedPass: vi.fn(() => ({ + runSelectedPass: vi.fn(async () => ({ exitCode: 0, otherGatewayEnvironmentsRemain: true, })), }); - const result = runUninstallAllGatewayPorts(OPTIONS, deps); + const result = await runUninstallAllGatewayPorts(OPTIONS, deps); expect(result.exitCode).toBe(1); expect(error).toHaveBeenCalledWith( @@ -142,10 +145,10 @@ describe("uninstall across every gateway port (#7791)", () => { ); }); - it("retains no port for the final pass when every other port uninstalled", () => { + it("retains no port for the final pass when every other port uninstalled", async () => { const { deps, runSelectedPass } = sweepDeps(); - runUninstallAllGatewayPorts(OPTIONS, deps); + await runUninstallAllGatewayPorts(OPTIONS, deps); expect(runSelectedPass.mock.calls[0]?.[1]).toMatchObject({ requireCompleteGatewayProcessCleanup: true, @@ -153,12 +156,12 @@ describe("uninstall across every gateway port (#7791)", () => { }); }); - it("runs a single scoped pass when no other gateway port exists", () => { + it("runs a single scoped pass when no other gateway port exists", async () => { const { deps, runPortPass, runSelectedPass } = sweepDeps({ listGatewayPorts: () => [8080], }); - const result = runUninstallAllGatewayPorts(OPTIONS, deps); + const result = await runUninstallAllGatewayPorts(OPTIONS, deps); expect(result.ports).toEqual([8080]); expect(runPortPass).not.toHaveBeenCalled(); @@ -174,20 +177,20 @@ describe("uninstall across every gateway port (#7791)", () => { const sharedStateDir = path.join(home, ".nemoclaw"); const selectedStateDir = path.join(sharedStateDir, "gateways", "9123"); const apiKeyPath = path.join(sharedStateDir, "dual-station-vllm-api-key"); - fs.mkdirSync(selectedStateDir, { recursive: true }); + fs.mkdirSync(selectedStateDir, { mode: 0o700, recursive: true }); fs.writeFileSync(path.join(selectedStateDir, "selected-only"), "remove me\n"); fs.writeFileSync(apiKeyPath, "ab".repeat(32), { mode: 0o600 }); try { vi.stubEnv("NEMOCLAW_GATEWAY_PORT", "9123"); vi.resetModules(); - const { runUninstallAllGatewayPorts: runNonDefaultSweep } = await import( - "./all-gateway-ports" - ); - const result = runNonDefaultSweep(OPTIONS, { + const { runUninstallAllGatewayPorts: runNonDefaultSweep } = + await import("./all-gateway-ports"); + const result = await runNonDefaultSweep(OPTIONS, { commandExists: () => true, env: { HOME: home, NEMOCLAW_GATEWAY_PORT: "9123" }, existsSync: fs.existsSync, + hasPortableRuntimeCleanup: () => false, home, isTty: false, kill: () => true, @@ -229,16 +232,16 @@ describe("uninstall across every gateway port (#7791)", () => { } }); - it("returns nonzero when an undiscovered environment remains after the only selected pass", () => { + it("returns nonzero when an undiscovered environment remains after the only selected pass", async () => { const { deps, error } = sweepDeps({ listGatewayPorts: () => [8080], - runSelectedPass: vi.fn(() => ({ + runSelectedPass: vi.fn(async () => ({ exitCode: 0, otherGatewayEnvironmentsRemain: true, })), }); - const result = runUninstallAllGatewayPorts(OPTIONS, deps); + const result = await runUninstallAllGatewayPorts(OPTIONS, deps); expect(result.exitCode).toBe(1); expect(error).toHaveBeenCalledWith( @@ -246,14 +249,14 @@ describe("uninstall across every gateway port (#7791)", () => { ); }); - it("fails without uninstalling anything when the gateway ports cannot be enumerated", () => { + it("fails without uninstalling anything when the gateway ports cannot be enumerated", async () => { const { deps, error, runPortPass, runSelectedPass } = sweepDeps({ listGatewayPorts: () => { throw new Error("gateways/ is a symbolic link"); }, }); - const result = runUninstallAllGatewayPorts(OPTIONS, deps); + const result = await runUninstallAllGatewayPorts(OPTIONS, deps); expect(result).toEqual({ exitCode: 1, ports: [] }); expect(runPortPass).not.toHaveBeenCalled(); @@ -261,6 +264,69 @@ describe("uninstall across every gateway port (#7791)", () => { expect(error).toHaveBeenCalledWith(expect.stringContaining("symbolic link")); }); + it("awaits the selected pass when no sibling gateway exists (#9189)", async () => { + let finish!: () => void; + const selected = vi.fn( + () => + new Promise<{ exitCode: number }>((resolve) => (finish = () => resolve({ exitCode: 0 }))), + ); + const { deps, runPortPass } = sweepDeps({ + listGatewayPorts: () => [8080], + runSelectedPass: selected, + }); + let completed = false; + const pending = runUninstallAllGatewayPorts(OPTIONS, deps).then((result) => { + completed = true; + return result; + }); + + await Promise.resolve(); + expect(selected).toHaveBeenCalledOnce(); + expect(runPortPass).not.toHaveBeenCalled(); + expect(completed).toBe(false); + finish(); + await expect(pending).resolves.toEqual({ exitCode: 0, ports: [8080] }); + }); + + it("finishes child passes before the awaited selected pass and retains failures (#9189)", async () => { + const order: string[] = []; + const { deps } = sweepDeps({ + runPortPass: (port) => { + order.push(`child:${String(port)}`); + return port === 9000 ? 1 : 0; + }, + runSelectedPass: async (_options, selectedDeps) => { + order.push(`selected:${selectedDeps.retainedGatewayPorts?.join(",")}`); + await Promise.resolve(); + order.push("selected:done"); + return { exitCode: 0 }; + }, + }); + + await expect(runUninstallAllGatewayPorts(OPTIONS, deps)).resolves.toMatchObject({ + exitCode: 1, + }); + expect(order).toEqual(["child:9000", "child:18080", "selected:9000", "selected:done"]); + }); + + it("maps a selected-pass rejection to exit 1 without a follow-on pass (#9189)", async () => { + const selected = vi.fn(async () => { + throw new Error("host fence release failed"); + }); + const { deps, error, runPortPass } = sweepDeps({ + listGatewayPorts: () => [8080], + runSelectedPass: selected, + }); + + await expect(runUninstallAllGatewayPorts(OPTIONS, deps)).resolves.toEqual({ + exitCode: 1, + ports: [8080], + }); + expect(selected).toHaveBeenCalledOnce(); + expect(runPortPass).not.toHaveBeenCalled(); + expect(error).toHaveBeenCalledWith(expect.stringContaining("host fence release failed")); + }); + it("binds each child pass to its own gateway port and drops the sweep request", () => { const env = { HOME: "/home/tester", diff --git a/src/lib/actions/uninstall/all-gateway-ports.ts b/src/lib/actions/uninstall/all-gateway-ports.ts index 9b293897b51..2d3fc35296c 100644 --- a/src/lib/actions/uninstall/all-gateway-ports.ts +++ b/src/lib/actions/uninstall/all-gateway-ports.ts @@ -29,7 +29,7 @@ import { readLineFromStdin } from "../../core/stdin"; import { resolveGatewayName } from "../../onboard/gateway-binding"; import { listGatewayStateRoots } from "../../state/gateway-registry"; import { - runUninstallPlan, + runUninstallPlanProduction, type UninstallRunDeps, type UninstallRunOptions, type UninstallRunOutcome, @@ -44,7 +44,7 @@ export interface AllGatewayPortsDeps extends UninstallRunDeps { runSelectedPass?: ( options: UninstallRunOptions, deps: UninstallRunDeps, - ) => Pick; + ) => Promise>; } export interface AllGatewayPortsOutcome { @@ -122,10 +122,10 @@ function confirmSweep( return false; } -export function runUninstallAllGatewayPorts( +export async function runUninstallAllGatewayPorts( options: UninstallRunOptions, deps: AllGatewayPortsDeps = {}, -): AllGatewayPortsOutcome { +): Promise { const env = { ...process.env, ...(deps.env ?? {}) }; const branding = getAgentBranding(deps.env?.NEMOCLAW_AGENT ?? process.env.NEMOCLAW_AGENT); const home = deps.home ?? deps.env?.HOME ?? process.env.HOME ?? os.homedir(); @@ -134,7 +134,7 @@ export function runUninstallAllGatewayPorts( const readLine = deps.readLine ?? (() => readLineFromStdin()); const listPorts = deps.listGatewayPorts ?? defaultListGatewayPorts; const runPortPass = deps.runPortPass ?? defaultRunPortPass; - const runSelectedPass = deps.runSelectedPass ?? runUninstallPlan; + const runSelectedPass = deps.runSelectedPass ?? runUninstallPlanProduction; const runDeps = { ...deps, env }; const expectedGatewayName = resolveGatewayName(GATEWAY_PORT); @@ -161,11 +161,19 @@ export function runUninstallAllGatewayPorts( .filter((port) => port !== GATEWAY_PORT) .sort((left, right) => left - right); if (otherPorts.length === 0) { - const selected = runSelectedPass(options, { - ...runDeps, - requireCompleteGatewayProcessCleanup: true, - retainedGatewayPorts: [], - }); + let selected: Pick; + try { + selected = await runSelectedPass(options, { + ...runDeps, + requireCompleteGatewayProcessCleanup: true, + retainedGatewayPorts: [], + }); + } catch (failure) { + error( + `Whole-host selected gateway uninstall failed: ${failure instanceof Error ? failure.message : String(failure)}`, + ); + return { exitCode: 1, ports: [GATEWAY_PORT] }; + } if (selected.otherGatewayEnvironmentsRemain) { error("Whole-host uninstall is incomplete because another gateway-port environment remains."); return { exitCode: 1, ports: [GATEWAY_PORT] }; @@ -191,14 +199,22 @@ export function runUninstallAllGatewayPorts( log( `Uninstalling gateway '${resolveGatewayName(GATEWAY_PORT)}' on port ${String(GATEWAY_PORT)}.`, ); - const selected = runSelectedPass( - { ...options, assumeYes: true }, - { - ...runDeps, - requireCompleteGatewayProcessCleanup: true, - retainedGatewayPorts, - }, - ); + let selected: Pick; + try { + selected = await runSelectedPass( + { ...options, assumeYes: true }, + { + ...runDeps, + requireCompleteGatewayProcessCleanup: true, + retainedGatewayPorts, + }, + ); + } catch (failure) { + error( + `Whole-host selected gateway uninstall failed: ${failure instanceof Error ? failure.message : String(failure)}`, + ); + return { exitCode: 1, ports: ordered }; + } if (selected.exitCode !== 0) exitCode = 1; if (selected.otherGatewayEnvironmentsRemain) { exitCode = 1; diff --git a/src/lib/actions/uninstall/plan.ts b/src/lib/actions/uninstall/plan.ts index fa344d02ae6..8c1705f15d0 100644 --- a/src/lib/actions/uninstall/plan.ts +++ b/src/lib/actions/uninstall/plan.ts @@ -4,7 +4,11 @@ import fs from "node:fs"; import os from "node:os"; -import { defaultUninstallPaths } from "../../domain/uninstall/paths"; +import { + defaultUninstallPaths, + NEMOCLAW_PROVIDERS, + type UninstallPaths, +} from "../../domain/uninstall/paths"; import { buildUninstallPlan, type UninstallPlan, @@ -12,6 +16,9 @@ import { } from "../../domain/uninstall/plan"; import { classifyNemoclawShim, type ShimClassification } from "../../domain/uninstall/shims"; +export { buildUninstallPlan, defaultUninstallPaths, NEMOCLAW_PROVIDERS }; +export type { UninstallPaths, UninstallPlan }; + export interface FileSystemDeps { closeSync?: typeof fs.closeSync; fstatSync?: typeof fs.fstatSync; diff --git a/src/lib/actions/uninstall/portable-runtime-cleanup.test.ts b/src/lib/actions/uninstall/portable-runtime-cleanup.test.ts new file mode 100644 index 00000000000..eef2ca76e88 --- /dev/null +++ b/src/lib/actions/uninstall/portable-runtime-cleanup.test.ts @@ -0,0 +1,1393 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawn, spawnSync, type ChildProcess } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterAll, afterEach, assert, beforeAll, describe, expect, it, vi } from "vitest"; + +import type { CheckpointPortableRuntimeAuthority } from "../../state/onboard-checkpoint-types"; +import { createSession } from "../../state/onboard-session"; +import { withMcpLifecycleLockSync } from "../../state/mcp-lifecycle-lock-acquisition"; +import { + hasPortableRetirementRecord, + inspectPortableRetirementRecovery, + preparePortableRetirement, + publishAndRetirePortableEvidence, +} from "../../state/portable-uninstall-retirement"; +import { + supersedePortableRetirementAfterCompletedOnboard, + withPortableOnboardRetirementBoundary, + type PortableOnboardRetirementBoundary, +} from "../../onboard/portable-retirement-authority"; +import { listPortableDemoSandboxLifecycleReceipts } from "../../onboard/experimental/portable-demo-lifecycle"; +import { portableDemoReceiptPath } from "../../onboard/experimental/portable-runtime-receipt-readiness"; +import { + withProcessBoundRegistryLockAt, + withRegistryLockAt, + type RegistryLockDeps, +} from "../../state/registry/lock"; +import { + hasPortableRuntimeCleanup, + runPortableRuntimeCleanupTransaction, + type PortableRuntimeCleanupDeps, + type PortableRuntimeCleanupInput, +} from "./portable-runtime-cleanup"; + +const UID = process.getuid?.() ?? 1001; +const ALPHA_ID = "a".repeat(64); +const BETA_ID = "b".repeat(64); +const REGISTRY_ID = "c".repeat(64); +const PROCESS_IDENTITY = "12345678-1234-1234-1234-123456789abc 123456"; + +interface ContainerRecord { + id: string; + name: string; + labels: Record; + running: boolean; +} + +const temporaryDirectories: string[] = []; + +const RETIREMENT_COMPETITOR_SCRIPT = String.raw` + import fs from "node:fs"; + const [lifecycleUrl, registryUrl, stateDir, registryFile, receiptFile, marker, sandboxName, control] = process.argv.slice(1); + const lifecycle = (await import(lifecycleUrl)).default; + const registry = (await import(registryUrl)).default; + const attempt = (owner) => { + const mutate = () => { + fs.writeFileSync(marker, "entered"); + fs.unlinkSync(receiptFile); + }; + try { + if (owner === "registry-only") { + registry.withRegistryLockAt(registryFile, mutate, { maxRetries: 2, wait: () => {} }); + } else { + lifecycle.withMcpLifecycleLockSync(owner, () => registry.withRegistryLockAt( + registryFile, + mutate, + { maxRetries: 2, wait: () => {} }, + ), { stateDir, pollIntervalMs: 1, timeoutMs: 10 }); + } + return 0; + } catch { + return 2; + } + }; + if (!control) process.exit(attempt(sandboxName)); + fs.writeFileSync(control + ".ready", "ready"); + while (!fs.existsSync(control + ".trigger")) await new Promise(resolve => setTimeout(resolve, 1)); + fs.writeFileSync(control + ".result", JSON.stringify([attempt(sandboxName), attempt("registry-only")])); + process.exit(0); +`; + +function fixture() { + const homeDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-portable-uninstall-")); + temporaryDirectories.push(homeDir); + const stateDir = path.join(homeDir, ".nemoclaw"); + const registryFile = path.join(stateDir, "sandboxes.json"); + const authority: CheckpointPortableRuntimeAuthority = { + schemaVersion: 1, + kind: "podman", + ownership: "current-user", + uid: UID, + homeDir, + configHome: path.join(homeDir, ".config"), + runtimeDir: path.join("/run/user", String(UID)), + socketPath: path.join("/run/user", String(UID), "podman", "podman.sock"), + }; + fs.mkdirSync(path.dirname(portableDemoReceiptPath("alpha", stateDir)), { + mode: 0o700, + recursive: true, + }); + const writeReceipt = (sandboxName: string, containerId: string, sandboxId: string) => { + fs.writeFileSync( + portableDemoReceiptPath(sandboxName, stateDir), + `${JSON.stringify( + { + schemaVersion: 4, + sandboxName, + sandboxId, + containerId, + dashboardPort: sandboxName === "alpha" ? 18789 : 18790, + registryGeneration: containerId, + runtimeAuthority: authority, + }, + null, + 2, + )}\n`, + { mode: 0o600 }, + ); + }; + writeReceipt("alpha", ALPHA_ID, "sandbox-alpha"); + fs.writeFileSync( + registryFile, + `${JSON.stringify({ + defaultSandbox: "alpha", + sandboxes: { + alpha: { + name: "alpha", + agent: "openclaw", + gatewayName: "nemoclaw", + gatewayPort: 8080, + openshellDriver: "docker", + lifecycleGeneration: ALPHA_ID, + }, + }, + })}\n`, + { mode: 0o600 }, + ); + const containersConf = path.join(homeDir, ".config/nemoclaw/portable/containers.conf"); + fs.mkdirSync(path.dirname(containersConf), { mode: 0o700, recursive: true }); + fs.writeFileSync(containersConf, "[engine]\nhelper_binaries_dir=[]\n", { mode: 0o600 }); + const containers = new Map(); + const addSandbox = ( + sandboxName: string, + sandboxId: string, + containerId: string, + labels: Record = {}, + ) => { + containers.set(containerId, { + id: containerId, + name: `openshell-default--${sandboxName}-${sandboxId}`, + labels: { + "openshell.managed": "true", + "openshell.ai/sandbox-id": sandboxId, + "openshell.ai/sandbox-name": sandboxName, + "openshell.ai/sandbox-namespace": "", + "openshell.ai/sandbox-workspace": "default", + ...labels, + }, + running: true, + }); + }; + addSandbox("alpha", "sandbox-alpha", ALPHA_ID); + containers.set(REGISTRY_ID, { + id: REGISTRY_ID, + name: "nemoclaw-portable-registry", + labels: { "com.nvidia.nemoclaw.portable": "1" }, + running: true, + }); + const podmanCalls: string[][] = []; + const podmanEnvironments: NodeJS.ProcessEnv[] = []; + const podmanHandlers = new Map< + string, + (args: readonly string[]) => { status: number; stdout?: string; stderr?: string } + >([ + [ + "ps", + (args) => { + const joined = args.join(" "); + const sandbox = /openshell\.ai\/sandbox-name=([^ ]+)/u.exec(joined)?.[1]; + const matchesPortableRegistry = joined.includes("com.nvidia.nemoclaw.portable=1"); + const matches = [...containers.values()].filter((container) => + matchesPortableRegistry + ? container.labels["com.nvidia.nemoclaw.portable"] === "1" + : sandbox !== undefined && container.labels["openshell.ai/sandbox-name"] === sandbox, + ); + return { status: 0, stdout: matches.map(({ id }) => id).join("\n") }; + }, + ], + [ + "inspect", + (args) => { + const target = String(args[1]); + const record = + containers.get(target) ?? [...containers.values()].find(({ name }) => name === target); + return record === undefined + ? { status: 1, stderr: `Error: no such container ${target}` } + : { + status: 0, + stdout: JSON.stringify([ + { + Id: record.id, + Name: record.name, + Config: { Labels: record.labels }, + State: { Running: record.running }, + }, + ]), + }; + }, + ], + [ + "rm", + (args) => { + containers.delete(String(args[2])); + return { status: 0 }; + }, + ], + ]); + const unexpectedPodmanCommand = (args: readonly string[]) => { + throw new Error(`Unexpected Podman command: ${args.join(" ")}`); + }; + const podman = vi.fn((rawArgs: readonly string[], env?: NodeJS.ProcessEnv) => { + podmanEnvironments.push({ ...(env ?? {}) }); + const args = rawArgs[0] === "--url" ? rawArgs.slice(2) : rawArgs; + podmanCalls.push([...args]); + return (podmanHandlers.get(String(args[0])) ?? unexpectedPodmanCommand)(args); + }); + const selectors = new Map([ + ["CONTAINERS_CONF", `$'${containersConf}'`], + ["NETAVARK_FW", "iptables"], + ["CONTAINER_HOST", "ssh://user-managed.example"], + ["CONTAINER_CONNECTION", "user-managed"], + ["CONTAINER_SSHKEY", "/home/test/.ssh/user-managed"], + ["UNRELATED", "keep"], + ]); + const systemctlCalls: string[][] = []; + const systemctlHandlers = new Map< + string, + (args: readonly string[]) => { status: number; stdout?: string } + >([ + [ + "show-environment", + () => ({ + status: 0, + stdout: [...selectors].map(([name, value]) => `${name}=${value}`).join("\n"), + }), + ], + [ + "unset-environment", + (args) => { + for (const name of args.slice(2)) selectors.delete(name); + return { status: 0 }; + }, + ], + ]); + const unexpectedSystemctlCommand = (args: readonly string[]) => { + throw new Error(`Unexpected systemctl command: ${args.join(" ")}`); + }; + const systemctl = vi.fn((args: readonly string[]) => { + systemctlCalls.push([...args]); + return (systemctlHandlers.get(String(args[1])) ?? unexpectedSystemctlCommand)(args); + }); + const input: PortableRuntimeCleanupInput = { + env: { + HOME: homeDir, + CONTAINER_HOST: "tcp://ambient-attacker.invalid", + CONTAINER_CONNECTION: "ambient-attacker", + CONTAINER_SSHKEY: "/tmp/ambient-attacker-key", + }, + gatewayName: "nemoclaw", + gatewayPort: 8080, + homeDir, + registryFile, + stateDir, + }; + const deps: PortableRuntimeCleanupDeps = { + hardenSocketDirectory: vi.fn(), + platform: "linux", + podman, + systemctl, + runtimeReadiness: { + uid: UID, + home: homeDir, + systemctl: () => ({ status: 0 }), + captureSocketAuthority: () => ({ + socketPath: authority.socketPath, + device: "1", + inode: "2", + mode: String(0o660), + ownerUid: String(UID), + directoryChain: [], + }), + assertSocketAuthority: vi.fn(), + podmanCapture: () => ({ + status: 0, + stdout: JSON.stringify({ Server: { Version: "5.6.1" } }), + stderr: "", + }), + }, + log: vi.fn(), + inspectRetirement: () => null, + prepareRetirement: preparePortableRetirement, + publishRetirement: vi.fn(), + withRegistryLock: (_registryFile, operation) => operation(), + }; + return { + addSandbox, + authority, + containers, + deps, + homeDir, + input, + podman, + podmanCalls, + podmanEnvironments, + registryFile, + selectors, + stateDir, + systemctl, + systemctlCalls, + writeReceipt, + }; +} + +function completeCleanup(input: PortableRuntimeCleanupInput, deps: PortableRuntimeCleanupDeps) { + return runPortableRuntimeCleanupTransaction(input, () => true, deps); +} + +function addRetirementConfig(test: ReturnType): string { + const target = path.join(test.homeDir, ".config/nemoclaw/portable/containers.conf"); + fs.mkdirSync(path.dirname(target), { mode: 0o700, recursive: true }); + fs.writeFileSync(target, "[engine]\nhelper_binaries_dir=[]\n", { mode: 0o600 }); + return target; +} + +function authorityDeps(test: ReturnType) { + return { + listReceipts: listPortableDemoSandboxLifecycleReceipts, + loadRegistry: () => JSON.parse(fs.readFileSync(test.registryFile, "utf8")), + withLifecycleLock: async (sandboxName: string, operation: () => Promise | T) => + await withMcpLifecycleLockSync(sandboxName, operation, { + stateDir: path.join(test.stateDir, "state"), + }), + }; +} + +function stageRetirementTarget( + test: ReturnType, + role: "config" | "receipt" | "registry", +): string { + addRetirementConfig(test); + const basename = { + config: "containers.conf", + receipt: path.basename(portableDemoReceiptPath("alpha", test.stateDir)), + registry: "sandboxes.json", + }[role]; + const unlink = fs.unlinkSync.bind(fs); + vi.spyOn(fs, "unlinkSync").mockImplementation((target) => { + String(target).includes(`.${basename}.portable-uninstall-`) && + assert.fail(`injected ${role} retirement crash`); + unlink(target); + }); + expect(() => + publishAndRetirePortableEvidence( + preparePortableRetirement(test.homeDir, [ + path.basename(portableDemoReceiptPath("alpha", test.stateDir)), + ]), + ), + ).toThrow(/injected/); + vi.restoreAllMocks(); + const artifact = inspectPortableRetirementRecovery(test.homeDir)!.artifacts.find( + (candidate) => candidate.root === role, + )!; + const directory = + role === "config" + ? path.join(test.homeDir, ".config/nemoclaw/portable") + : role === "receipt" + ? path.join(test.stateDir, "portable-demo-lifecycle") + : test.stateDir; + return path.join(directory, artifact.basename); +} + +function completedOnboardAuthority(test: ReturnType, portable: boolean) { + const sessionFile = path.join(test.stateDir, "onboard-session.json"); + const generation = "e".repeat(64); + const sandboxName = "later-sandbox"; + const sessionId = "completed-after-portable-uninstall"; + const dashboardPort = 18791; + const gateway = { + gatewayName: "nemoclaw", + gatewayPort: 8080, + mode: "nemoclaw-managed" as const, + source: "standalone" as const, + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }; + const session = createSession({ + agent: "openclaw", + sandboxName, + sessionId, + metadata: { gatewayName: gateway.gatewayName, fromDockerfile: null }, + }); + session.status = "complete"; + session.resumable = false; + session.machine = { + version: 1, + state: "complete", + stateEnteredAt: "2026-08-15T00:00:00.000Z", + revision: 1, + }; + session.checkpoint = { + schemaVersion: 4, + sessionId, + machineState: "complete", + updatedAt: "2026-08-15T00:00:00.000Z", + profile: { kind: "selected", value: portable ? "portable" : "default" }, + runtimeAuthority: portable ? { kind: "selected", value: test.authority } : { kind: "unset" }, + sandboxIdentity: { kind: "selected", value: { name: sandboxName, agent: "openclaw" } }, + webSearch: { kind: "unset" }, + messaging: { kind: "unset" }, + resourceProfile: { kind: "unset" }, + gatewayAuthority: { kind: "selected", value: gateway }, + effectGroups: {}, + bindings: { credentialEnvs: [], registeredProviders: [] }, + sandboxRecreate: null, + }; + fs.mkdirSync(test.stateDir, { recursive: true }); + fs.writeFileSync(sessionFile, `${JSON.stringify(session)}\n`, { mode: 0o600 }); + fs.writeFileSync( + test.registryFile, + `${JSON.stringify({ + defaultSandbox: sandboxName, + sandboxes: { + [sandboxName]: { + name: sandboxName, + agent: null, + dashboardPort, + gatewayName: gateway.gatewayName, + gatewayPort: gateway.gatewayPort, + lifecycleGeneration: generation, + openshellDriver: "docker", + }, + }, + })}\n`, + { mode: 0o600 }, + ); + const writePortableReceipt = () => { + const receipt = portableDemoReceiptPath(sandboxName, test.stateDir); + fs.mkdirSync(path.dirname(receipt), { mode: 0o700, recursive: true }); + fs.writeFileSync( + receipt, + `${JSON.stringify({ + schemaVersion: 4, + sandboxName, + sandboxId: "later-sandbox-id", + containerId: "f".repeat(64), + dashboardPort, + registryGeneration: generation, + runtimeAuthority: test.authority, + })}\n`, + { mode: 0o600 }, + ); + }; + portable && writePortableReceipt(); + portable && addRetirementConfig(test); + return { + boundary: { + homeDir: test.homeDir, + registryFile: test.registryFile, + sessionFile, + stateDir: test.stateDir, + } satisfies PortableOnboardRetirementBoundary, + deps: authorityDeps(test), + profile: portable ? ("portable" as const) : ("default" as const), + sandboxName, + }; +} + +function retiredOnboardAuthority(portable: boolean) { + const test = fixture(); + addRetirementConfig(test); + publishAndRetirePortableEvidence( + preparePortableRetirement(test.homeDir, [ + path.basename(portableDemoReceiptPath("alpha", test.stateDir)), + ]), + ); + return { authority: completedOnboardAuthority(test, portable), test }; +} + +type ReplacementPhase = "config" | "receipt" | "registry" | "pre-complete"; + +function resumableReplacementAuthority( + test: ReturnType, + portable: boolean, + phase: ReplacementPhase, +) { + const authority = completedOnboardAuthority(test, portable); + rewriteJson(authority.boundary.sessionFile, (session) => { + const state = phase === "pre-complete" ? "failed" : "init"; + session.status = phase === "pre-complete" ? "failed" : "in_progress"; + session.resumable = true; + session.machine.state = state; + session.checkpoint.machineState = state; + }); + portable && + phase === "config" && + (() => { + fs.rmSync(path.join(test.stateDir, "portable-demo-lifecycle"), { recursive: true }); + })(); + (phase === "config" || phase === "receipt") && fs.unlinkSync(test.registryFile); + return authority; +} + +async function supersedeCompleted( + authority: ReturnType, +): Promise { + await supersedePortableRetirementAfterCompletedOnboard( + authority.boundary, + authority.profile, + authority.deps, + ); +} + +function rewriteJson(target: string, mutate: (value: Record) => void): void { + const value = JSON.parse(fs.readFileSync(target, "utf8")) as Record; + mutate(value); + fs.writeFileSync(target, `${JSON.stringify(value)}\n`, { mode: 0o600 }); +} + +function afterReceiptDirectoryRead(directory: string, mutate: () => void): void { + const readdir = fs.readdirSync.bind(fs); + vi.spyOn(fs, "readdirSync").mockImplementation(((target: fs.PathLike, options?: any) => { + const entries = readdir(target, options); + new Map([[directory, mutate]]).get(String(target))?.(); + return entries; + }) as typeof fs.readdirSync); +} + +const receiptDirectoryMutations = { + "link-count": (directory: string) => + afterReceiptDirectoryRead(directory, () => + fs.mkdirSync(path.join(directory, "concurrent-generation")), + ), + replacement: (directory: string) => + afterReceiptDirectoryRead(directory, () => { + fs.renameSync(directory, `${directory}.moved`); + fs.mkdirSync(directory, { mode: 0o700 }); + }), + symlink: (directory: string) => { + const moved = `${directory}.moved`; + fs.renameSync(directory, moved); + fs.symlinkSync(moved, directory); + }, +} as const; +const receiptDirectoryMutationNames = ["link-count", "replacement", "symlink"] as const; + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + fs.rmSync(directory, { recursive: true, force: true }); + } + vi.restoreAllMocks(); +}); + +describe("portable runtime uninstall cleanup", () => { + it("admits only exact completed ordinary authority to generic uninstall (#9189)", () => { + const ordinary = fixture(); + fs.rmSync(path.join(ordinary.stateDir, "portable-demo-lifecycle"), { recursive: true }); + fs.rmSync(path.join(ordinary.homeDir, ".config/nemoclaw/portable"), { recursive: true }); + completedOnboardAuthority(ordinary, false); + expect(hasPortableRuntimeCleanup(ordinary.stateDir)).toBe(false); + }); + + it.each([false, true])( + "supersedes the retry record only after durable completed onboarding (portable=%s) (#9189)", + async (portable) => { + const { authority, test } = retiredOnboardAuthority(portable); + const record = path.join(test.stateDir, "portable-uninstall-retirement.json"); + + await supersedeCompleted(authority); + + expect(fs.existsSync(record)).toBe(false); + expect( + fs + .readdirSync(test.stateDir) + .filter((name) => name.includes("portable-uninstall-retirement")), + ).toEqual([]); + expect(fs.existsSync(authority.boundary.sessionFile)).toBe(true); + expect(fs.existsSync(authority.boundary.registryFile)).toBe(true); + }, + ); + + it("skips completed authority proof when no retirement record exists (#9189)", async () => { + const authority = completedOnboardAuthority(fixture(), false); + const loadRegistry = vi.fn(() => authority.deps.loadRegistry()); + await supersedePortableRetirementAfterCompletedOnboard(authority.boundary, "default", { + ...authority.deps, + loadRegistry, + }); + expect(loadRegistry).not.toHaveBeenCalled(); + }); + + it.each( + ( + [ + ["default", false], + ["portable", true], + ] as const + ).flatMap(([profile, portable]) => + (["config", "receipt", "registry", "pre-complete"] as const).map( + (phase) => [`${profile} after ${phase}`, portable, phase] as const, + ), + ), + )("keeps R through %s replacement state recovery (#9189)", async (_case, portable, phase) => { + const { test } = retiredOnboardAuthority(portable); + const authority = resumableReplacementAuthority(test, portable, phase); + const record = path.join(test.stateDir, "portable-uninstall-retirement.json"); + const resumed = await withPortableOnboardRetirementBoundary( + authority.boundary, + () => true, + authority.deps, + ); + expect(resumed).toBe(true); + expect(fs.existsSync(record)).toBe(true); + !portable && (phase === "config" || phase === "receipt") + ? expect(hasPortableRuntimeCleanup(test.stateDir)).toBe(true) + : expect(() => hasPortableRuntimeCleanup(test.stateDir)).toThrow(); + completedOnboardAuthority(test, portable); + await withPortableOnboardRetirementBoundary(authority.boundary, () => {}, authority.deps); + expect(fs.existsSync(record)).toBe(false); + }); + + it.each(["config", "receipt", "registry"] as const)( + "retires an authenticated staged %s target before onboarding writes (#9189)", + async (role) => { + const test = fixture(); + const staged = stageRetirementTarget(test, role); + const boundary = { + homeDir: test.homeDir, + registryFile: test.registryFile, + sessionFile: path.join(test.stateDir, "onboard-session.json"), + stateDir: test.stateDir, + }; + let newAuthority: ReturnType | null = null; + + await withPortableOnboardRetirementBoundary( + boundary, + () => { + expect(fs.existsSync(staged)).toBe(false); + newAuthority = completedOnboardAuthority(test, true); + }, + authorityDeps(test), + ); + + expect(newAuthority).not.toBeNull(); + expect(fs.existsSync(test.registryFile)).toBe(true); + expect( + fs.existsSync(path.join(test.homeDir, ".config/nemoclaw/portable/containers.conf")), + ).toBe(true); + expect(listPortableDemoSandboxLifecycleReceipts(test.stateDir)).toHaveLength(1); + }, + ); + + it.each(["R+S", "RC+S", "S", "SC"] as const)( + "continues completed-onboarding supersession from fixed %s state (#9189)", + async (fixedState) => { + const { authority, test } = retiredOnboardAuthority(true); + const record = path.join(test.stateDir, "portable-uninstall-retirement.json"); + const canonicalCleanup = path.join( + test.stateDir, + ".portable-uninstall-retirement.canonical.cleanup", + ); + const superseded = path.join(test.stateDir, ".portable-uninstall-retirement.superseded"); + const supersededCleanup = path.join( + test.stateDir, + ".portable-uninstall-retirement.superseded.cleanup", + ); + fs.linkSync(record, superseded); + fixedState === "RC+S" && fs.renameSync(record, canonicalCleanup); + (fixedState === "S" || fixedState === "SC") && fs.unlinkSync(record); + fixedState === "SC" && fs.renameSync(superseded, supersededCleanup); + + await supersedeCompleted(authority); + + expect([record, canonicalCleanup, superseded, supersededCleanup].some(fs.existsSync)).toBe( + false, + ); + }, + ); + + describe("completed-onboarding writer exclusion", () => { + let scope: ReturnType; + let writer: ChildProcess; + let control: string; + let receipt: string; + + beforeAll(async () => { + scope = retiredOnboardAuthority(true); + control = path.join(scope.test.homeDir, "resident-writer"); + receipt = portableDemoReceiptPath(scope.authority.sandboxName, scope.test.stateDir); + writer = spawn(process.execPath, [ + "--no-warnings", + "--import", + "tsx", + "--input-type=module", + "-e", + RETIREMENT_COMPETITOR_SCRIPT, + new URL("../../state/mcp-lifecycle-lock-acquisition.ts", import.meta.url).href, + new URL("../../state/registry/lock.ts", import.meta.url).href, + path.join(scope.test.stateDir, "state"), + scope.test.registryFile, + receipt, + path.join(scope.test.homeDir, "unused-entered"), + scope.authority.sandboxName, + control, + ]); + for (let attempt = 0; attempt < 1_000 && !fs.existsSync(`${control}.ready`); attempt++) + await new Promise((resolve) => setTimeout(resolve, 5)); + expect(fs.existsSync(`${control}.ready`)).toBe(true); + }); + + afterAll(() => { + writer?.kill("SIGTERM"); + }); + + it("blocks real lifecycle and registry writers through final supersession (#9189)", async () => { + const originalLoad = scope.authority.deps.loadRegistry; + const loadRegistry = () => { + fs.writeFileSync(`${control}.trigger`, "trigger"); + const deadline = Date.now() + 1_000; + while (!fs.existsSync(`${control}.result`) && Date.now() < deadline) + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 1); + expect(JSON.parse(fs.readFileSync(`${control}.result`, "utf8"))).toEqual([2, 2]); + return originalLoad(); + }; + await supersedePortableRetirementAfterCompletedOnboard( + scope.authority.boundary, + scope.authority.profile, + { ...scope.authority.deps, loadRegistry }, + ); + await new Promise((resolve) => + writer.exitCode === null ? writer.once("exit", () => resolve()) : resolve(), + ); + expect(writer.exitCode).toBe(0); + expect(fs.existsSync(path.join(scope.test.homeDir, "unused-entered"))).toBe(false); + expect(fs.existsSync(receipt)).toBe(true); + }); + }); + + it("preserves the retry record when completed onboarding authority is incomplete (#9189)", async () => { + const { authority, test: incomplete } = retiredOnboardAuthority(true); + fs.unlinkSync(portableDemoReceiptPath(authority.sandboxName, incomplete.stateDir)); + await expect(supersedeCompleted(authority)).rejects.toThrow(/ENOENT|missing/); + expect(hasPortableRetirementRecord(incomplete.homeDir)).toBe(true); + }); + + it("rejects an extra portable config-root entry before supersession (#9189)", async () => { + const scope = retiredOnboardAuthority(true); + fs.writeFileSync( + path.join(scope.test.homeDir, ".config/nemoclaw/portable/unexpected.conf"), + "unexpected\n", + { mode: 0o600 }, + ); + await expect(supersedeCompleted(scope.authority)).rejects.toThrow(/incomplete/); + expect(hasPortableRetirementRecord(scope.test.homeDir)).toBe(true); + }); + + it.each([ + ["an unknown profile", "session", (value: any) => (value.checkpoint.profile.value = "unknown")], + ["a profile mismatch", "session", (value: any) => (value.checkpoint.profile.value = "default")], + ["a schema-old checkpoint", "session", (value: any) => (value.checkpoint.schemaVersion = 3)], + ["a session identity drift", "session", (value: any) => (value.checkpoint.sessionId = "other")], + [ + "a sandbox identity drift", + "session", + (value: any) => (value.checkpoint.sandboxIdentity.value.name = "other"), + ], + [ + "a checkpoint gateway drift", + "session", + (value: any) => (value.checkpoint.gatewayAuthority.value.gatewayName = "other"), + ], + [ + "an incomplete registry row", + "registry", + (value: any) => delete value.sandboxes["later-sandbox"].openshellDriver, + ], + [ + "a registry gateway drift", + "registry", + (value: any) => (value.sandboxes["later-sandbox"].gatewayName = "other"), + ], + [ + "a schema-old receipt", + "receipt", + (value: any) => { + value.schemaVersion = 3; + delete value.runtimeAuthority; + }, + ], + ["a runtime authority drift", "receipt", (value: any) => (value.runtimeAuthority.uid += 1)], + [ + "a registry generation drift", + "receipt", + (value: any) => (value.registryGeneration = "other"), + ], + ])("rejects %s before supersession (#9189)", async (_label, target, mutate) => { + const scope = retiredOnboardAuthority(true); + const authorityFile = + target === "session" + ? scope.authority.boundary.sessionFile + : target === "registry" + ? scope.authority.boundary.registryFile + : portableDemoReceiptPath(scope.authority.sandboxName, scope.test.stateDir); + rewriteJson(authorityFile, mutate); + + await expect(supersedeCompleted(scope.authority)).rejects.toThrow(); + expect(hasPortableRetirementRecord(scope.test.homeDir)).toBe(true); + }); + + it.each([ + [ + "state", + (scope: ReturnType) => + path.join( + scope.test.stateDir, + `.sandboxes.json.portable-uninstall-${"f".repeat(64)}.cleanup`, + ), + ], + [ + "receipt", + (scope: ReturnType) => + portableDemoReceiptPath("unknown", scope.test.stateDir), + ], + [ + "receipt cleanup", + (scope: ReturnType) => + path.join( + scope.test.stateDir, + `portable-demo-lifecycle/.${"f".repeat(64)}.json.portable-uninstall-${"e".repeat(64)}.cleanup`, + ), + ], + [ + "receipt temporary", + (scope: ReturnType) => + path.join(scope.test.stateDir, "portable-demo-lifecycle/.unknown.tmp"), + ], + [ + "receipt retiring", + (scope: ReturnType) => + path.join(scope.test.stateDir, "portable-demo-lifecycle/.unknown.retiring"), + ], + [ + "configuration", + (scope: ReturnType) => + path.join(scope.test.homeDir, ".config/nemoclaw/portable/containers.conf"), + ], + [ + "configuration cleanup", + (scope: ReturnType) => + path.join( + scope.test.homeDir, + `.config/nemoclaw/portable/.containers.conf.portable-uninstall-${"e".repeat(64)}.cleanup`, + ), + ], + [ + "configuration temporary", + (scope: ReturnType) => + path.join(scope.test.homeDir, ".config/nemoclaw/portable/.unknown.tmp"), + ], + [ + "configuration retiring", + (scope: ReturnType) => + path.join(scope.test.homeDir, ".config/nemoclaw/portable/.unknown.retiring"), + ], + ])( + "rejects an ordinary completion with a portable %s artifact (#9189)", + async (_label, target) => { + const scope = retiredOnboardAuthority(false); + const artifact = target(scope); + fs.mkdirSync(path.dirname(artifact), { mode: 0o700, recursive: true }); + fs.writeFileSync(artifact, "{}\n", { mode: 0o600 }); + + await expect(supersedeCompleted(scope.authority)).rejects.toThrow(); + expect(hasPortableRetirementRecord(scope.test.homeDir)).toBe(true); + }, + ); + + it.each(receiptDirectoryMutationNames)( + "rejects receipt-directory %s drift (#9189)", + async (mutation) => { + const scope = retiredOnboardAuthority(true); + const receiptDirectory = path.join(scope.test.stateDir, "portable-demo-lifecycle"); + receiptDirectoryMutations[mutation](receiptDirectory); + + await expect(supersedeCompleted(scope.authority)).rejects.toThrow(/changed|Unsafe|ENOTDIR/); + expect(hasPortableRetirementRecord(scope.test.homeDir)).toBe(true); + }, + ); + + it("removes only exact receipt-owned containers and exact current selector projections (#9189)", () => { + const test = fixture(); + + expect(hasPortableRuntimeCleanup(test.stateDir)).toBe(true); + const cleanup = completeCleanup(test.input, test.deps); + + expect(cleanup).toEqual({ + registryRemoved: true, + sandboxContainersRemoved: 1, + selectorsRemoved: ["CONTAINERS_CONF", "NETAVARK_FW"], + }); + expect(test.containers.has(ALPHA_ID)).toBe(false); + expect(test.containers.size).toBe(0); + expect(test.selectors).toEqual( + new Map([ + ["CONTAINER_HOST", "ssh://user-managed.example"], + ["CONTAINER_CONNECTION", "user-managed"], + ["CONTAINER_SSHKEY", "/home/test/.ssh/user-managed"], + ["UNRELATED", "keep"], + ]), + ); + expect(fs.existsSync(portableDemoReceiptPath("alpha", test.stateDir))).toBe(true); + expect(test.podmanEnvironments).not.toEqual([]); + for (const env of test.podmanEnvironments) { + expect(env.CONTAINER_HOST).toBeUndefined(); + expect(env.CONTAINER_CONNECTION).toBeUndefined(); + expect(env.CONTAINER_SSHKEY).toBeUndefined(); + } + expect(fs.existsSync(`${test.registryFile}.lock`)).toBe(false); + }); + + it("preserves changed current-user manager selector values (#9189)", () => { + const test = fixture(); + test.selectors.set("CONTAINERS_CONF", "/home/test/user-containers.conf"); + test.selectors.set("NETAVARK_FW", "nftables"); + + expect(completeCleanup(test.input, test.deps)).toEqual({ + registryRemoved: true, + sandboxContainersRemoved: 1, + selectorsRemoved: [], + }); + expect(test.selectors.get("CONTAINERS_CONF")).toBe("/home/test/user-containers.conf"); + expect(test.selectors.get("NETAVARK_FW")).toBe("nftables"); + }); + + it("prevalidates every receipt before deleting the first container (#9189)", () => { + const test = fixture(); + test.writeReceipt("beta", BETA_ID, "sandbox-beta"); + test.addSandbox("beta", "sandbox-beta", BETA_ID, { "openshell.managed": "false" }); + const registry = JSON.parse(fs.readFileSync(test.registryFile, "utf8")) as { + sandboxes: Record; + }; + registry.sandboxes.beta = { + name: "beta", + agent: "openclaw", + gatewayName: "nemoclaw", + gatewayPort: 8080, + openshellDriver: "docker", + lifecycleGeneration: BETA_ID, + }; + fs.writeFileSync(test.registryFile, `${JSON.stringify(registry)}\n`); + + expect(() => completeCleanup(test.input, test.deps)).toThrow( + /OpenShell identity does not match sandbox 'beta'/, + ); + expect(test.containers.has(ALPHA_ID)).toBe(true); + expect(test.podmanCalls.some((args) => args[0] === "rm")).toBe(false); + }); + + it.each([ + [ + "missing", + (test: ReturnType) => { + fs.writeFileSync( + test.registryFile, + `${JSON.stringify({ defaultSandbox: null, sandboxes: {} })}\n`, + ); + }, + /has no current registry ownership/, + ], + [ + "extra", + (test: ReturnType) => { + const registry = JSON.parse(fs.readFileSync(test.registryFile, "utf8")) as { + sandboxes: Record; + }; + registry.sandboxes.beta = { + name: "beta", + agent: "openclaw", + gatewayName: "nemoclaw", + gatewayPort: 8080, + openshellDriver: "docker", + lifecycleGeneration: BETA_ID, + }; + fs.writeFileSync(test.registryFile, `${JSON.stringify(registry)}\n`); + }, + /not represented by the complete lifecycle receipt set/, + ], + [ + "mismatched gateway name", + (test: ReturnType) => { + const registry = JSON.parse(fs.readFileSync(test.registryFile, "utf8")) as { + sandboxes: { alpha: { gatewayName: string } }; + }; + registry.sandboxes.alpha.gatewayName = "other"; + fs.writeFileSync(test.registryFile, `${JSON.stringify(registry)}\n`); + }, + /unrecognized gatewayName/, + ], + [ + "mismatched gateway port", + (test: ReturnType) => { + const registry = JSON.parse(fs.readFileSync(test.registryFile, "utf8")) as { + sandboxes: { alpha: { gatewayPort: number } }; + }; + registry.sandboxes.alpha.gatewayPort = 9000; + fs.writeFileSync(test.registryFile, `${JSON.stringify(registry)}\n`); + }, + /conflicting gateway identity/, + ], + ])("rejects a %s registry row before any mutation (#9189)", (_case, prepare, expected) => { + const test = fixture(); + prepare(test); + + expect(() => completeCleanup(test.input, test.deps)).toThrow(expected); + expect(test.containers.has(ALPHA_ID)).toBe(true); + expect(test.containers.has(REGISTRY_ID)).toBe(true); + expect(test.selectors.has("CONTAINERS_CONF")).toBe(true); + expect(fs.existsSync(portableDemoReceiptPath("alpha", test.stateDir))).toBe(true); + expect(test.podmanCalls.some((args) => args[0] === "rm")).toBe(false); + }); + + it("preserves shared portable evidence when exact OpenShell retirement fails (#9189)", () => { + const test = fixture(); + const configMarker = path.join(test.homeDir, ".config", "nemoclaw", "keep-for-retry"); + fs.mkdirSync(path.dirname(configMarker), { recursive: true }); + fs.writeFileSync(configMarker, "retry\n"); + const retireOpenShell = vi.fn(() => false); + + expect(runPortableRuntimeCleanupTransaction(test.input, retireOpenShell, test.deps)).toBeNull(); + expect(retireOpenShell).toHaveBeenCalledWith(1, ["alpha"], "nemoclaw"); + expect(test.containers.has(ALPHA_ID)).toBe(false); + expect(test.containers.has(REGISTRY_ID)).toBe(true); + expect(test.selectors.has("CONTAINERS_CONF")).toBe(true); + expect(fs.existsSync(portableDemoReceiptPath("alpha", test.stateDir))).toBe(true); + expect(fs.existsSync(test.registryFile)).toBe(true); + expect(fs.existsSync(configMarker)).toBe(true); + expect(fs.existsSync(test.stateDir)).toBe(true); + }); + + it("holds sorted lifecycle locks before the registry lock and releases in reverse (#9189)", () => { + const test = fixture(); + const order: string[] = []; + test.writeReceipt("beta", BETA_ID, "sandbox-beta"); + test.addSandbox("beta", "sandbox-beta", BETA_ID); + const registry = JSON.parse(fs.readFileSync(test.registryFile, "utf8")) as { + sandboxes: Record; + }; + registry.sandboxes.beta = { + name: "beta", + agent: "openclaw", + gatewayName: "nemoclaw", + gatewayPort: 8080, + openshellDriver: "docker", + lifecycleGeneration: BETA_ID, + }; + fs.writeFileSync(test.registryFile, `${JSON.stringify(registry)}\n`); + + expect( + runPortableRuntimeCleanupTransaction(test.input, () => true, { + ...test.deps, + withLifecycleLock: (sandboxName, operation) => { + order.push(`acquire:${sandboxName}`); + try { + return operation(); + } finally { + order.push(`release:${sandboxName}`); + } + }, + withRegistryLock: (_registryFile, operation) => { + order.push("acquire:registry"); + try { + return operation(); + } finally { + order.push("release:registry"); + } + }, + }), + ).toMatchObject({ sandboxContainersRemoved: 2 }); + expect(order).toEqual([ + "acquire:alpha", + "acquire:beta", + "acquire:registry", + "release:registry", + "release:beta", + "release:alpha", + ]); + }); + + it("runs final evidence retirement while both production lock primitives are held (#9189)", () => { + const test = fixture(); + const lifecycleStateDir = path.join(test.stateDir, "state"); + const lockDeps: RegistryLockDeps = { + isProcessAlive: () => true, + readProcessIdentity: () => PROCESS_IDENTITY, + }; + let lifecycleHeld = false; + let registryHeld = false; + const retireEvidence = vi.fn(() => { + expect(lifecycleHeld).toBe(true); + expect(registryHeld).toBe(true); + return true; + }); + + expect( + runPortableRuntimeCleanupTransaction(test.input, () => true, { + ...test.deps, + publishRetirement: retireEvidence, + withLifecycleLock: (sandboxName, operation, stateDir) => + withMcpLifecycleLockSync( + sandboxName, + () => { + lifecycleHeld = true; + try { + return operation(); + } finally { + lifecycleHeld = false; + } + }, + { stateDir }, + ), + withRegistryLock: (registryFile, operation) => + withProcessBoundRegistryLockAt( + registryFile, + () => { + registryHeld = true; + try { + return operation(); + } finally { + registryHeld = false; + } + }, + lockDeps, + ), + }), + ).toMatchObject({ sandboxContainersRemoved: 1 }); + expect(retireEvidence).toHaveBeenCalledOnce(); + expect(lifecycleHeld).toBe(false); + expect(registryHeld).toBe(false); + expect(fs.existsSync(path.join(lifecycleStateDir, "mcp-lifecycle-locks"))).toBe(true); + }); + + it("preserves receipts and selectors when exact container removal cannot be verified (#9189)", () => { + const test = fixture(); + test.deps.podman = vi.fn((rawArgs: readonly string[], env?: NodeJS.ProcessEnv) => { + const args = rawArgs[0] === "--url" ? rawArgs.slice(2) : rawArgs; + return args[0] === "rm" + ? { status: 1, stderr: "permission denied" } + : test.podman(rawArgs, env); + }); + + expect(() => completeCleanup(test.input, test.deps)).toThrow(/still has/); + expect(test.systemctlCalls.some((args) => args[1] === "unset-environment")).toBe(false); + expect(fs.existsSync(portableDemoReceiptPath("alpha", test.stateDir))).toBe(true); + }); + + it("preserves retry evidence when managed registry removal fails (#9189)", () => { + const test = fixture(); + const failingDeps: PortableRuntimeCleanupDeps = { + ...test.deps, + podman: vi.fn((rawArgs: readonly string[], env?: NodeJS.ProcessEnv) => { + const args = rawArgs[0] === "--url" ? rawArgs.slice(2) : rawArgs; + return args[0] === "rm" && args[2] === REGISTRY_ID + ? { status: 1, stderr: "registry removal denied" } + : test.podman(rawArgs, env); + }), + }; + + expect(() => completeCleanup(test.input, failingDeps)).toThrow( + /Removing the managed portable registry container failed: registry removal denied/, + ); + expect(test.containers.has(REGISTRY_ID)).toBe(true); + expect(test.systemctlCalls.some((args) => args[1] === "unset-environment")).toBe(false); + expect(fs.existsSync(portableDemoReceiptPath("alpha", test.stateDir))).toBe(true); + + expect(completeCleanup(test.input, test.deps)).toEqual({ + registryRemoved: true, + sandboxContainersRemoved: 0, + selectorsRemoved: ["CONTAINERS_CONF", "NETAVARK_FW"], + }); + }); + + it("retries selector cleanup after the managed registry was already removed (#9189)", () => { + const test = fixture(); + const systemctl = test.deps.systemctl!; + const failingDeps: PortableRuntimeCleanupDeps = { + ...test.deps, + systemctl: vi.fn((args, env) => + args[1] === "unset-environment" + ? { status: 1, stderr: "permission denied" } + : systemctl(args, env), + ), + }; + + expect(() => completeCleanup(test.input, failingDeps)).toThrow( + /Clearing NemoClaw portable selectors.*permission denied/, + ); + expect(test.containers.has(REGISTRY_ID)).toBe(false); + expect(fs.existsSync(portableDemoReceiptPath("alpha", test.stateDir))).toBe(true); + + expect(completeCleanup(test.input, test.deps)).toEqual({ + registryRemoved: false, + sandboxContainersRemoved: 0, + selectorsRemoved: ["CONTAINERS_CONF", "NETAVARK_FW"], + }); + }); + + it("accepts an already-absent exact sandbox on retry but rejects a replaced identity (#9189)", () => { + const retry = fixture(); + retry.containers.delete(ALPHA_ID); + expect(completeCleanup(retry.input, retry.deps)).toEqual({ + registryRemoved: true, + sandboxContainersRemoved: 0, + selectorsRemoved: ["CONTAINERS_CONF", "NETAVARK_FW"], + }); + + const replaced = fixture(); + replaced.containers.delete(ALPHA_ID); + replaced.addSandbox("alpha", "sandbox-replacement", BETA_ID); + expect(() => completeCleanup(replaced.input, replaced.deps)).toThrow( + /replaced or ambiguous container/, + ); + }); + + it("rejects duplicate containers in the sandbox label index before removal (#9189)", () => { + const test = fixture(); + test.addSandbox("alpha", "sandbox-duplicate", BETA_ID); + + expect(() => completeCleanup(test.input, test.deps)).toThrow(/replaced or ambiguous container/); + expect(test.containers.has(ALPHA_ID)).toBe(true); + expect(test.containers.has(BETA_ID)).toBe(true); + expect(test.podmanCalls.some((args) => args[0] === "rm")).toBe(false); + }); + + it("fails closed on malformed receipts and mismatched lifecycle generations (#9189)", () => { + const malformed = fixture(); + fs.writeFileSync(portableDemoReceiptPath("alpha", malformed.stateDir), "{not-json\n"); + expect(() => hasPortableRuntimeCleanup(malformed.stateDir)).toThrow(/malformed/); + + const mismatch = fixture(); + const registry = JSON.parse(fs.readFileSync(mismatch.registryFile, "utf8")) as { + sandboxes: { alpha: { lifecycleGeneration: string } }; + }; + registry.sandboxes.alpha.lifecycleGeneration = "different-generation"; + fs.writeFileSync(mismatch.registryFile, `${JSON.stringify(registry)}\n`); + expect(() => completeCleanup(mismatch.input, mismatch.deps)).toThrow( + /current registry ownership/, + ); + expect(mismatch.containers.has(ALPHA_ID)).toBe(true); + }); + + it("blocks a destroy-shaped retirement until exact shared cleanup completes (#9189)", () => { + const test = fixture(); + const receiptFile = portableDemoReceiptPath("alpha", test.stateDir); + const configMarker = path.join(test.homeDir, ".config", "nemoclaw", "keep-for-retry"); + const lifecycleStateDir = path.join(test.stateDir, "state"); + const competitorMarker = path.join(test.homeDir, "competing-destroy-entered"); + const lifecycleUrl = new URL("../../state/mcp-lifecycle-lock-acquisition.ts", import.meta.url) + .href; + const registryUrl = new URL("../../state/registry/lock.ts", import.meta.url).href; + const lockDeps: RegistryLockDeps = { + isProcessAlive: () => true, + readProcessIdentity: () => PROCESS_IDENTITY, + }; + fs.mkdirSync(path.dirname(configMarker), { recursive: true }); + fs.writeFileSync(configMarker, "retry\n"); + const competingDestroy = () => + spawnSync( + process.execPath, + [ + "--no-warnings", + "--import", + "tsx", + "--input-type=module", + "-e", + RETIREMENT_COMPETITOR_SCRIPT, + lifecycleUrl, + registryUrl, + lifecycleStateDir, + test.registryFile, + receiptFile, + competitorMarker, + "alpha", + ], + { encoding: "utf8" }, + ); + + const cleanup = runPortableRuntimeCleanupTransaction( + test.input, + () => { + expect(competingDestroy().status).toBe(2); + return true; + }, + { + ...test.deps, + publishRetirement: () => { + expect(competingDestroy().status).toBe(2); + }, + withLifecycleLock: (sandboxName, operation, stateDir) => + withMcpLifecycleLockSync(sandboxName, operation, { stateDir }), + withRegistryLock: (registryFile, operation) => + withProcessBoundRegistryLockAt(registryFile, operation, lockDeps), + }, + ); + + expect(cleanup).toEqual({ + registryRemoved: true, + sandboxContainersRemoved: 1, + selectorsRemoved: ["CONTAINERS_CONF", "NETAVARK_FW"], + }); + expect(fs.existsSync(competitorMarker)).toBe(false); + expect(test.containers.has(ALPHA_ID)).toBe(false); + expect(test.containers.has(REGISTRY_ID)).toBe(false); + expect(test.selectors.has("CONTAINERS_CONF")).toBe(false); + expect(test.selectors.get("CONTAINER_HOST")).toBe("ssh://user-managed.example"); + expect(fs.existsSync(receiptFile)).toBe(true); + expect(fs.existsSync(test.registryFile)).toBe(true); + expect( + (JSON.parse(fs.readFileSync(test.registryFile, "utf8")) as { sandboxes: object }).sandboxes, + ).toHaveProperty("alpha"); + expect(fs.existsSync(configMarker)).toBe(true); + expect(fs.existsSync(`${test.registryFile}.lock`)).toBe(false); + }); + + it("fails before mutation when destroy retires ownership before lock acquisition (#9189)", () => { + const test = fixture(); + const receiptFile = portableDemoReceiptPath("alpha", test.stateDir); + const lifecycleLocks = path.join(test.stateDir, "test-pre-acquisition-locks"); + const configMarker = path.join(test.homeDir, ".config", "nemoclaw", "keep-for-retry"); + const lockDeps: RegistryLockDeps = { + isProcessAlive: () => true, + readProcessIdentity: () => PROCESS_IDENTITY, + }; + const withLifecycleLock = (sandboxName: string, operation: () => Value): Value => { + const lockPath = path.join(lifecycleLocks, sandboxName); + fs.mkdirSync(lockPath, { recursive: false }); + try { + return operation(); + } finally { + fs.rmdirSync(lockPath); + } + }; + const retireOwnership = () => + withLifecycleLock("alpha", () => + withRegistryLockAt( + test.registryFile, + () => { + fs.unlinkSync(receiptFile); + fs.writeFileSync( + test.registryFile, + `${JSON.stringify({ defaultSandbox: null, sandboxes: {} })}\n`, + ); + }, + lockDeps, + ), + ); + const acquireAfterRetirement = (sandboxName: string, operation: () => Value): Value => { + retireOwnership(); + return withLifecycleLock(sandboxName, operation); + }; + fs.mkdirSync(lifecycleLocks, { recursive: true }); + fs.mkdirSync(path.dirname(configMarker), { recursive: true }); + fs.writeFileSync(configMarker, "retry\n"); + + expect(() => + runPortableRuntimeCleanupTransaction(test.input, () => true, { + ...test.deps, + withLifecycleLock: acquireAfterRetirement, + withRegistryLock: (registryFile, operation) => + withProcessBoundRegistryLockAt(registryFile, operation, lockDeps), + }), + ).toThrow(/state changed while uninstall acquired its fences/); + expect(test.containers.has(ALPHA_ID)).toBe(true); + expect(test.containers.has(REGISTRY_ID)).toBe(true); + expect(test.selectors.has("CONTAINERS_CONF")).toBe(true); + expect(fs.existsSync(configMarker)).toBe(true); + expect(test.podmanCalls.some((args) => args[0] === "rm")).toBe(false); + }); +}); diff --git a/src/lib/actions/uninstall/portable-runtime-cleanup.ts b/src/lib/actions/uninstall/portable-runtime-cleanup.ts new file mode 100644 index 00000000000..e8012323531 --- /dev/null +++ b/src/lib/actions/uninstall/portable-runtime-cleanup.ts @@ -0,0 +1,557 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import path from "node:path"; +import { TextDecoder } from "node:util"; +import { isDeepStrictEqual } from "node:util"; + +import type { CheckpointPortableRuntimeAuthority } from "../../state/onboard-checkpoint-types"; +import { hasPortableUninstallAuthority } from "../../onboard/portable-retirement-authority"; +import { withMcpLifecycleLockSync } from "../../state/mcp-lifecycle-lock-acquisition"; +import { + inspectPortableRetirementRecovery, + PORTABLE_RETIREMENT_STATE_ENTRIES, + preparePortableRetirement, + publishAndRetirePortableEvidence, + resumePortableEvidenceRetirement, + withPortableHostFence, + type PreparedPortableRetirement, + type PortableRetirementRecovery, +} from "../../state/portable-uninstall-retirement"; + +export { PORTABLE_RETIREMENT_STATE_ENTRIES, withPortableHostFence }; +import { withProcessBoundRegistryLockAt } from "../../state/registry/lock"; +import { + readGatewayRegistryFile, + registryEntryGatewayPort, + type GatewayRegistryEntry, +} from "../../state/gateway-registry"; +import { + createPortablePodmanLifecycleTransport, + listPortableDemoSandboxLifecycleReceipts, + preparePortableDemoSandboxRemoval, + type PortableDemoLifecycleDeps, + type PortableDemoLifecycleReceiptRecord, + type PortablePodmanLifecycleCommandResult, + type PortablePodmanLifecycleTransport, +} from "../../onboard/experimental/portable-demo-lifecycle"; +import { portablePodmanCommandEnvironment } from "../../onboard/experimental/portable-runtime-readiness"; + +const REGISTRY_CONTAINER_NAME = "nemoclaw-portable-registry"; +const REGISTRY_LABEL_NAME = "com.nvidia.nemoclaw.portable"; +const REGISTRY_LABEL_VALUE = "1"; +const CONTAINER_ID_PATTERN = /^[a-f0-9]{64}$/u; +const MAX_SYSTEMD_ENVIRONMENT_BYTES = 1024 * 1024; +const PORTABLE_SELECTOR_NAMES = [ + "CONTAINERS_CONF", + "NETAVARK_FW", + "CONTAINER_HOST", + "CONTAINER_CONNECTION", + "CONTAINER_SSHKEY", +] as const; +const UTF8 = new TextDecoder("utf-8", { fatal: true }); + +interface PortableRegistryRemoval { + readonly present: boolean; + removeAndVerify(): void; +} + +export interface PortableRuntimeCleanupInput { + readonly env: NodeJS.ProcessEnv; + readonly gatewayName: string; + readonly gatewayPort: number; + readonly homeDir: string; + readonly registryFile: string; + readonly stateDir: string; +} + +export interface PortableRuntimeCleanupDeps extends PortableDemoLifecycleDeps { + readonly systemctl?: ( + args: readonly string[], + env: NodeJS.ProcessEnv, + ) => PortablePodmanLifecycleCommandResult; + readonly withLifecycleLock?: (sandboxName: string, operation: () => T, stateDir: string) => T; + readonly withRegistryLock?: (registryFile: string, operation: () => T) => T; + readonly inspectRetirement?: (homeDir: string) => PortableRetirementRecovery | null; + readonly prepareRetirement?: ( + homeDir: string, + receiptBasenames: readonly string[], + ) => PreparedPortableRetirement; + readonly publishRetirement?: (prepared: PreparedPortableRetirement) => void; + readonly resumeRetirement?: (homeDir: string) => void; +} + +export interface PortableRuntimeCleanupResult { + readonly registryRemoved: boolean; + readonly sandboxContainersRemoved: number; + readonly selectorsRemoved: readonly string[]; +} + +function commandDetail(result: PortablePodmanLifecycleCommandResult): string { + if (result.error) { + return (result.error as NodeJS.ErrnoException).code ?? result.error.message; + } + const stderr = String(result.stderr ?? "").trim(); + return stderr || `exit ${String(result.status)}`; +} + +function requireCommand(result: PortablePodmanLifecycleCommandResult, description: string): void { + if (result.status === 0 && !result.error) return; + throw new Error(`${description} failed: ${commandDetail(result)}`); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function isMissingContainer(result: PortablePodmanLifecycleCommandResult): boolean { + if (result.status === 0 && !result.error) return false; + const detail = `${String(result.stderr ?? "")}\n${String(result.stdout ?? "")}`; + return /\b(?:no such (?:object|container)|no container with (?:name|id)|container .* not found)\b/iu.test( + detail, + ); +} + +function withPortableFences( + input: PortableRuntimeCleanupInput, + sandboxNames: readonly string[], + deps: PortableRuntimeCleanupDeps, + operation: () => T, +): T { + const lifecycleStateDir = path.join(input.stateDir, "state"); + const withLifecycleLock = + deps.withLifecycleLock ?? + ((sandboxName: string, inner: () => Value, stateDir: string) => + withMcpLifecycleLockSync(sandboxName, inner, { stateDir })); + const withRegistryLock = deps.withRegistryLock ?? withProcessBoundRegistryLockAt; + const acquireNext = (index: number): T => { + const sandboxName = sandboxNames[index]; + return sandboxName + ? withLifecycleLock(sandboxName, () => acquireNext(index + 1), lifecycleStateDir) + : withRegistryLock(input.registryFile, operation); + }; + return acquireNext(0); +} + +function commonRuntimeAuthority( + receipts: readonly PortableDemoLifecycleReceiptRecord[], +): CheckpointPortableRuntimeAuthority { + const authority = receipts[0]?.runtimeAuthority; + if (!authority) throw new Error("Portable uninstall requires at least one lifecycle receipt"); + for (const receipt of receipts.slice(1)) { + if (!isDeepStrictEqual(receipt.runtimeAuthority, authority)) { + throw new Error("Portable lifecycle receipts disagree on their Podman runtime authority"); + } + } + return authority; +} + +function requireReceiptRegistryOwnership( + receipt: PortableDemoLifecycleReceiptRecord, + entry: GatewayRegistryEntry | undefined, + gatewayPort: number, + gatewayName: string, +): void { + if (!entry) { + throw new Error( + `Portable lifecycle receipt for sandbox '${receipt.sandboxName}' has no current registry ownership`, + ); + } + if ( + registryEntryGatewayPort(entry) !== gatewayPort || + entry.gatewayPort !== gatewayPort || + entry.gatewayName !== gatewayName || + entry.agent !== "openclaw" || + entry.openshellDriver !== "docker" || + entry.lifecycleGeneration !== receipt.registryGeneration + ) { + throw new Error( + `Portable lifecycle receipt for sandbox '${receipt.sandboxName}' does not match its current registry ownership`, + ); + } +} + +function requireCompleteReceiptRegistryOwnership( + receipts: readonly PortableDemoLifecycleReceiptRecord[], + registry: ReturnType, + gatewayPort: number, + gatewayName: string, +): string { + if (!registry) throw new Error("Portable lifecycle receipts have no current sandbox registry"); + const receiptNames = receipts.map((receipt) => receipt.sandboxName).sort(); + for (const receipt of receipts) { + requireReceiptRegistryOwnership( + receipt, + registry.sandboxes[receipt.sandboxName], + gatewayPort, + gatewayName, + ); + } + const registryNames = Object.keys(registry.sandboxes).sort(); + if (!isDeepStrictEqual(registryNames, receiptNames)) { + throw new Error( + "Portable sandbox registry ownership is not represented by the complete lifecycle receipt set", + ); + } + return gatewayName; +} + +function currentReceipts(stateDir: string): PortableDemoLifecycleReceiptRecord[] { + return listPortableDemoSandboxLifecycleReceipts(stateDir).sort((left, right) => + left.sandboxName.localeCompare(right.sandboxName), + ); +} + +/** Detect portable uninstall from strict durable receipts, never ambient selectors or names. */ +export function hasPortableRuntimeCleanup(stateDir: string): boolean { + const homeDir = path.dirname(stateDir); + const registryFile = path.join(stateDir, "sandboxes.json"); + return hasPortableUninstallAuthority( + { + homeDir, + registryFile, + sessionFile: path.join(stateDir, "onboard-session.json"), + stateDir, + }, + { + loadRegistry: () => { + const registry = readGatewayRegistryFile(homeDir, registryFile); + if (!registry) throw new Error("Completed onboarding registry is missing"); + return registry; + }, + }, + ); +} + +export function portableRetirementPreservationEntries(stateDir: string): { + config: string[]; + stateRoot: string[]; +} { + const artifacts = inspectPortableRetirementRecovery(path.dirname(stateDir))?.artifacts ?? []; + return { + config: artifacts.filter(({ root }) => root === "config").map(({ basename }) => basename), + stateRoot: artifacts.filter(({ root }) => root === "registry").map(({ basename }) => basename), + }; +} + +function recordedRegistrySandboxNames(registryBytes: Buffer): string[] { + let registry: unknown; + try { + registry = JSON.parse(UTF8.decode(registryBytes)); + } catch { + throw new Error("Recorded portable registry authority is malformed"); + } + if (!isRecord(registry) || !isRecord(registry.sandboxes)) { + throw new Error("Recorded portable registry authority is invalid"); + } + const names = Object.entries(registry.sandboxes).map(([name, value]) => { + if (!isRecord(value) || value.name !== name || name.length < 1 || name.length > 256) { + throw new Error("Recorded portable registry sandbox identity is invalid"); + } + return name; + }); + return names.sort(); +} + +/** Remove receipt-owned portable resources under lifecycle and registry locks through retirement. */ +export function runPortableRuntimeCleanupTransaction( + input: PortableRuntimeCleanupInput, + continueAfterSandboxRemoval: ( + removed: number, + sandboxNames: readonly string[], + gatewayName: string, + ) => boolean, + deps: PortableRuntimeCleanupDeps = {}, +): PortableRuntimeCleanupResult | null { + const inspectRetirement = deps.inspectRetirement ?? inspectPortableRetirementRecovery; + const recovery = inspectRetirement(input.homeDir); + if (recovery) { + const sandboxNames = recovery.registryBytes + ? recordedRegistrySandboxNames(recovery.registryBytes) + : []; + return withPortableFences(input, sandboxNames, deps, () => { + (deps.resumeRetirement ?? resumePortableEvidenceRetirement)(input.homeDir); + return { registryRemoved: false, sandboxContainersRemoved: 0, selectorsRemoved: [] }; + }); + } + const receipts = currentReceipts(input.stateDir); + const registry = readGatewayRegistryFile(input.homeDir, input.registryFile); + if (receipts.length === 0) { + throw new Error("Portable lifecycle receipts disappeared before uninstall acquired its fences"); + } + return withPortableFences( + input, + receipts.map((receipt) => receipt.sandboxName), + deps, + () => { + const current = currentReceipts(input.stateDir); + const currentRegistry = readGatewayRegistryFile(input.homeDir, input.registryFile); + if (!isDeepStrictEqual(current, receipts) || !isDeepStrictEqual(currentRegistry, registry)) { + throw new Error( + "Portable lifecycle or registry state changed while uninstall acquired its fences", + ); + } + const authority = commonRuntimeAuthority(receipts); + const transport = createPortablePodmanLifecycleTransport(authority, { + ...deps, + env: input.env, + stateDir: input.stateDir, + }); + const gatewayName = requireCompleteReceiptRegistryOwnership( + receipts, + registry, + input.gatewayPort, + input.gatewayName, + ); + const receiptBasenames = receipts.map( + (receipt) => `${createHash("sha256").update(receipt.sandboxName).digest("hex")}.json`, + ); + const retirement = (deps.prepareRetirement ?? preparePortableRetirement)( + input.homeDir, + receiptBasenames, + ); + const prepared = receipts.map((receipt) => + preparePortableDemoSandboxRemoval(receipt, transport, input.stateDir), + ); + const portableRegistry = preparePortableRegistryRemoval(transport); + inspectPortableUserManagerEnvironment(authority, input.env, deps); + for (const target of prepared) target.removeAndVerify(); + const sandboxContainersRemoved = prepared.filter((target) => target.present).length; + if ( + !continueAfterSandboxRemoval( + sandboxContainersRemoved, + receipts.map((receipt) => receipt.sandboxName), + gatewayName, + ) + ) + return null; + if ( + !isDeepStrictEqual(currentReceipts(input.stateDir), receipts) || + !isDeepStrictEqual(readGatewayRegistryFile(input.homeDir, input.registryFile), registry) + ) { + throw new Error( + "Portable lifecycle or registry state changed during exact uninstall cleanup", + ); + } + for (const target of prepared) target.verifyAbsent(); + portableRegistry.removeAndVerify(); + const selectorsRemoved = clearPortableUserManagerSelectors(authority, input.env, deps); + if ( + !isDeepStrictEqual(currentReceipts(input.stateDir), receipts) || + !isDeepStrictEqual(readGatewayRegistryFile(input.homeDir, input.registryFile), registry) + ) { + throw new Error("Portable lifecycle or registry state changed before evidence retirement"); + } + (deps.publishRetirement ?? publishAndRetirePortableEvidence)(retirement); + return { + registryRemoved: portableRegistry.present, + sandboxContainersRemoved, + selectorsRemoved, + }; + }, + ); +} + +function parseContainerIds( + result: PortablePodmanLifecycleCommandResult, + description: string, +): string[] { + requireCommand(result, description); + const ids = String(result.stdout ?? "") + .split(/\r?\n/u) + .map((line) => line.trim()) + .filter(Boolean); + if (ids.some((id) => !CONTAINER_ID_PATTERN.test(id))) { + throw new Error(`${description} returned an invalid container ID`); + } + return ids; +} + +function registryLabelContainerIds(transport: PortablePodmanLifecycleTransport): string[] { + return parseContainerIds( + transport.podman([ + "ps", + "-a", + "--no-trunc", + "--filter", + `label=${REGISTRY_LABEL_NAME}=${REGISTRY_LABEL_VALUE}`, + "--format", + "{{.ID}}", + ]), + "Finding the managed portable registry container", + ); +} + +function inspectRegistryContainer( + transport: PortablePodmanLifecycleTransport, + result = transport.podman(["inspect", REGISTRY_CONTAINER_NAME]), +): string | null { + if (isMissingContainer(result)) return null; + requireCommand(result, "Inspecting the managed portable registry container"); + let parsed: unknown; + try { + parsed = JSON.parse(String(result.stdout ?? "")); + } catch { + throw new Error("Inspecting the managed portable registry container returned invalid JSON"); + } + if (!Array.isArray(parsed) || parsed.length !== 1 || !isRecord(parsed[0])) { + throw new Error( + "Inspecting the managed portable registry container returned an invalid record", + ); + } + const record = parsed[0]; + const config = isRecord(record.Config) ? record.Config : null; + const labels = config && isRecord(config.Labels) ? config.Labels : null; + const state = isRecord(record.State) ? record.State : null; + if ( + typeof record.Id !== "string" || + !CONTAINER_ID_PATTERN.test(record.Id) || + record.Name !== REGISTRY_CONTAINER_NAME || + labels?.[REGISTRY_LABEL_NAME] !== REGISTRY_LABEL_VALUE || + typeof state?.Running !== "boolean" + ) { + throw new Error("The portable registry container does not match NemoClaw ownership"); + } + return record.Id; +} + +function preparePortableRegistryRemoval( + transport: PortablePodmanLifecycleTransport, +): PortableRegistryRemoval { + transport.assertRuntimeAuthority(); + const labelIds = registryLabelContainerIds(transport); + const containerId = inspectRegistryContainer(transport); + if (containerId === null) { + if (labelIds.length !== 0) { + throw new Error( + "Portable registry ownership is ambiguous because a labeled replacement exists", + ); + } + return { present: false, removeAndVerify: () => transport.assertRuntimeAuthority() }; + } + if (labelIds.length !== 1 || labelIds[0] !== containerId) { + throw new Error("Portable registry ownership is ambiguous"); + } + return { + present: true, + removeAndVerify: () => { + transport.assertRuntimeAuthority(); + const currentId = inspectRegistryContainer(transport); + if (currentId !== containerId) { + throw new Error("The portable registry container changed after prevalidation"); + } + requireCommand( + transport.podman(["rm", "--force", containerId]), + "Removing the managed portable registry container", + ); + const exact = transport.podman(["inspect", containerId]); + if (!isMissingContainer(exact)) { + if (exact.status !== 0 || exact.error) { + requireCommand(exact, "Verifying portable registry removal"); + } + throw new Error("The managed portable registry container still exists after removal"); + } + if ( + inspectRegistryContainer(transport) !== null || + registryLabelContainerIds(transport).length + ) { + throw new Error("A managed portable registry container remains after removal"); + } + transport.assertRuntimeAuthority(); + }, + }; +} + +function defaultSystemctl( + args: readonly string[], + env: NodeJS.ProcessEnv, +): PortablePodmanLifecycleCommandResult { + const result = spawnSync("systemctl", [...args], { + encoding: "utf8", + env, + stdio: ["ignore", "pipe", "pipe"], + timeout: 30_000, + }); + return { + status: result.status, + stdout: String(result.stdout ?? ""), + stderr: String(result.stderr ?? ""), + ...(result.error ? { error: result.error } : {}), + }; +} + +function parseUserManagerEnvironment(output: string): Map { + if (Buffer.byteLength(output, "utf8") > MAX_SYSTEMD_ENVIRONMENT_BYTES || output.includes("\0")) { + throw new Error("The current-user systemd manager environment is too large or invalid"); + } + const values = new Map(); + for (const line of output.split(/\r?\n/u)) { + if (!line) continue; + const match = /^([A-Za-z_][A-Za-z0-9_]*)=(.*)$/u.exec(line); + if (!match) throw new Error("The current-user systemd manager environment is malformed"); + const name = match[1]!; + if (values.has(name)) { + throw new Error(`The current-user systemd manager environment repeats '${name}'`); + } + values.set(name, match[2]!); + } + return values; +} + +function inspectPortableUserManagerEnvironment( + authority: CheckpointPortableRuntimeAuthority, + env: NodeJS.ProcessEnv, + deps: PortableRuntimeCleanupDeps, +): { + readonly commandEnv: NodeJS.ProcessEnv; + readonly systemctl: NonNullable; + readonly values: ReadonlyMap; +} { + const systemctl = deps.systemctl ?? defaultSystemctl; + const commandEnv = portablePodmanCommandEnvironment(authority, env); + const show = systemctl(["--user", "show-environment"], commandEnv); + requireCommand(show, "Inspecting the current-user systemd manager environment"); + const current = parseUserManagerEnvironment(String(show.stdout ?? "")); + return { + commandEnv, + systemctl, + values: new Map(PORTABLE_SELECTOR_NAMES.map((name) => [name, current.get(name)])), + }; +} + +function clearPortableUserManagerSelectors( + authority: CheckpointPortableRuntimeAuthority, + env: NodeJS.ProcessEnv, + deps: PortableRuntimeCleanupDeps, +): string[] { + const { commandEnv, systemctl, values } = inspectPortableUserManagerEnvironment( + authority, + env, + deps, + ); + const expected = new Map([ + ["CONTAINERS_CONF", path.join(authority.configHome, "nemoclaw", "portable", "containers.conf")], + ["NETAVARK_FW", "iptables"], + ]); + const unset = [...expected.entries()] + .filter(([name, value]) => + [value, `$'${value.replaceAll("\\", "\\\\").replaceAll("'", "\\'")}'`].includes( + values.get(name) ?? "", + ), + ) + .map(([name]) => name); + if (unset.length === 0) return []; + requireCommand( + systemctl(["--user", "unset-environment", ...unset], commandEnv), + "Clearing NemoClaw portable selectors from the current-user systemd manager", + ); + const verified = systemctl(["--user", "show-environment"], commandEnv); + requireCommand(verified, "Verifying the current-user systemd manager environment"); + const remaining = parseUserManagerEnvironment(String(verified.stdout ?? "")); + if (unset.some((name) => remaining.has(name))) { + throw new Error("A NemoClaw portable selector remains in the current-user systemd manager"); + } + return unset; +} diff --git a/src/lib/actions/uninstall/run-plan-dual-station.test.ts b/src/lib/actions/uninstall/run-plan-dual-station.test.ts index dc017acea18..981ca59c2d7 100644 --- a/src/lib/actions/uninstall/run-plan-dual-station.test.ts +++ b/src/lib/actions/uninstall/run-plan-dual-station.test.ts @@ -32,6 +32,7 @@ function ok(stdout = ""): RunResult { function runUninstallPlan(options: UninstallRunOptions, deps: UninstallRunDeps) { return runUninstallPlanBase(options, { + hasPortableRuntimeCleanup: () => false, resolveGatewayTeardownAuthority: ({ gatewayName, gatewayPort }) => ({ gatewayName, gatewayPort, @@ -289,7 +290,7 @@ describe("managed distributed vLLM runtime uninstall", () => { expect(runDocker).not.toHaveBeenCalled(); expect(rmSync).not.toHaveBeenCalled(); expect(errors.join("\n")).toContain( - "Managed distributed vLLM SSH binding exists without its ownership receipt", + "A managed distributed vLLM SSH binding exists without its ownership receipt", ); expect(fs.existsSync(discoveryBindingPath)).toBe(true); } finally { @@ -524,7 +525,7 @@ describe("managed distributed vLLM runtime uninstall", () => { expect(runDocker).not.toHaveBeenCalled(); expect(rmSync).not.toHaveBeenCalled(); expect(errors.join("\n")).toContain( - "Managed distributed vLLM SSH binding exists without its ownership receipt", + "A managed distributed vLLM SSH binding exists without its ownership receipt", ); expect(fs.existsSync(bindingPath)).toBe(true); } finally { diff --git a/src/lib/actions/uninstall/run-plan-foreign-user-gateway.test.ts b/src/lib/actions/uninstall/run-plan-foreign-user-gateway.test.ts index 2f85bb19d18..e152c6580dc 100644 --- a/src/lib/actions/uninstall/run-plan-foreign-user-gateway.test.ts +++ b/src/lib/actions/uninstall/run-plan-foreign-user-gateway.test.ts @@ -49,6 +49,7 @@ function uninstallWithHostGatewayOwnedBy(uid: number): { env: { HOME: tmpHome, NO_COLOR: "1" }, error: (message) => errors.push(message), existsSync: () => false, + hasPortableRuntimeCleanup: () => false, isTty: false, kill: () => false, log: vi.fn(), diff --git a/src/lib/actions/uninstall/run-plan-gateway-segregation-selected-port.test.ts b/src/lib/actions/uninstall/run-plan-gateway-segregation-selected-port.test.ts index 8b835237a10..fa00ccc4695 100644 --- a/src/lib/actions/uninstall/run-plan-gateway-segregation-selected-port.test.ts +++ b/src/lib/actions/uninstall/run-plan-gateway-segregation-selected-port.test.ts @@ -107,6 +107,7 @@ describe("uninstall selected gateway-port segregation (#3053)", () => { commandExists: (command) => command === "openshell", env: { HOME: tmpHome, NEMOCLAW_NON_INTERACTIVE: "1" } as NodeJS.ProcessEnv, existsSync: (target) => target.startsWith(tmpHome) && fs.existsSync(target), + hasPortableRuntimeCleanup: () => false, isTty: false, log: (line) => logs.push(line), rmSync: fs.rmSync, diff --git a/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts b/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts index fd584e49421..276c4d223ac 100644 --- a/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts +++ b/src/lib/actions/uninstall/run-plan-gateway-segregation.test.ts @@ -953,6 +953,7 @@ describe("uninstall gateway-port segregation (#3053)", () => { commandExists: (command) => command === "openshell", env: { HOME: tmpHome, NEMOCLAW_NON_INTERACTIVE: "1" } as NodeJS.ProcessEnv, existsSync: (target) => target.startsWith(tmpHome) && fs.existsSync(target), + hasPortableRuntimeCleanup: () => false, isTty: false, log: (line) => logs.push(line), rmSync: fs.rmSync, @@ -1002,6 +1003,7 @@ describe("uninstall gateway-port segregation (#3053)", () => { commandExists: (command) => command === "openshell", env: { HOME: tmpHome, NEMOCLAW_NON_INTERACTIVE: "1" } as NodeJS.ProcessEnv, existsSync: (target) => target.startsWith(tmpHome) && fs.existsSync(target), + hasPortableRuntimeCleanup: () => false, isTty: false, log: (line) => logs.push(line), rmSync: fs.rmSync, diff --git a/src/lib/actions/uninstall/run-plan-gateway-service.test.ts b/src/lib/actions/uninstall/run-plan-gateway-service.test.ts index e55f6ef28c2..c5756950cc9 100644 --- a/src/lib/actions/uninstall/run-plan-gateway-service.test.ts +++ b/src/lib/actions/uninstall/run-plan-gateway-service.test.ts @@ -129,6 +129,7 @@ function uninstall( withProvenManagedGatewayProcess({ env: test.env, existsSync: (target) => String(target).startsWith(test.root) && fs.existsSync(target), + hasPortableRuntimeCleanup: () => false, isPortFree: () => true, isTty: false, platform: "linux", diff --git a/src/lib/actions/uninstall/run-plan-other-gateway-report.test.ts b/src/lib/actions/uninstall/run-plan-other-gateway-report.test.ts index 31ab58a481c..0c7b67df707 100644 --- a/src/lib/actions/uninstall/run-plan-other-gateway-report.test.ts +++ b/src/lib/actions/uninstall/run-plan-other-gateway-report.test.ts @@ -33,13 +33,7 @@ function ok(stdout = ""): RunResult { } function writeScopedGatewayState(home: string, port = 8080): void { - const stateDir = path.join( - home, - ".local", - "state", - "nemoclaw", - resolveGatewayStateDirName(port), - ); + const stateDir = path.join(home, ".local", "state", "nemoclaw", resolveGatewayStateDirName(port)); const jwtBundle = ensureDockerDriverGatewayJwtBundle(stateDir); fs.writeFileSync( path.join(stateDir, "openshell-gateway.toml"), @@ -134,21 +128,24 @@ describe("uninstall reporting for other gateway-port environments (#7791)", () = it.each([ ["a live sibling gateway with a shared registry row", SIBLING_REGISTRY], ["a live sibling gateway with no registry row", null], - ] as const)("warns that %s remains after the selected-port uninstall (#8797)", (_scenario, registry) => { - const { logs, warnings } = uninstallOutputFor(registry, ["nemoclaw", "nemoclaw-9000"]); - - expect(warnings).toContainEqual( - expect.stringContaining("⚠ Other NemoClaw gateway-port environments remain on this host"), - ); - expect(warnings).toContainEqual(" · gateway 'nemoclaw-9000' on port 9000"); - expect(warnings).toContainEqual( - expect.stringContaining("NEMOCLAW_GATEWAY_PORT=9000 nemoclaw uninstall"), - ); - expect(warnings).toContainEqual(expect.stringContaining("uninstall --all-gateway-ports")); - expect(logs).not.toContainEqual( - expect.stringContaining("gateway-port environments remain on this host"), - ); - }); + ] as const)( + "warns that %s remains after the selected-port uninstall (#8797)", + (_scenario, registry) => { + const { logs, warnings } = uninstallOutputFor(registry, ["nemoclaw", "nemoclaw-9000"]); + + expect(warnings).toContainEqual( + expect.stringContaining("⚠ Other NemoClaw gateway-port environments remain on this host"), + ); + expect(warnings).toContainEqual(" · gateway 'nemoclaw-9000' on port 9000"); + expect(warnings).toContainEqual( + expect.stringContaining("NEMOCLAW_GATEWAY_PORT=9000 nemoclaw uninstall"), + ); + expect(warnings).toContainEqual(expect.stringContaining("uninstall --all-gateway-ports")); + expect(logs).not.toContainEqual( + expect.stringContaining("gateway-port environments remain on this host"), + ); + }, + ); it("colors the retained gateway warning only for interactive stderr without NO_COLOR (#8797)", () => { vi.stubEnv("NO_COLOR", undefined); @@ -226,6 +223,7 @@ describe("uninstall reporting for other gateway-port environments (#7791)", () = env: { HOME: "/tmp/nemoclaw-uninstall-test-scan" } as NodeJS.ProcessEnv, error: (line) => errors.push(line), existsSync: () => false, + hasPortableRuntimeCleanup: () => false, isTty: false, kill: vi.fn(() => true), log: vi.fn(), diff --git a/src/lib/actions/uninstall/run-plan-portable-runtime.test.ts b/src/lib/actions/uninstall/run-plan-portable-runtime.test.ts new file mode 100644 index 00000000000..62d769cb443 --- /dev/null +++ b/src/lib/actions/uninstall/run-plan-portable-runtime.test.ts @@ -0,0 +1,1223 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, assert, describe, expect, it, vi } from "vitest"; +import { + withProvenManagedGatewayProcess, + writeManagedGatewayRuntimeProof, +} from "../../../../test/support/uninstall-managed-gateway-test-support"; + +import { + type RunResult, + runUninstallPlan as runUninstallPlanBase, + runUninstallPlanProduction, + type UninstallRunDeps, + type UninstallRunOptions, +} from "./run-plan"; +import { + hasPortableRuntimeCleanup, + runPortableRuntimeCleanupTransaction, + type PortableRuntimeCleanupInput, +} from "./portable-runtime-cleanup"; +import { + preparePortableRetirement, + publishAndRetirePortableEvidence, +} from "../../state/portable-uninstall-retirement"; +import { portableDemoReceiptPath } from "../../onboard/experimental/portable-runtime-receipt-readiness"; +import { + buildDockerDriverGatewayConfigToml, + gatewayIdForStateDir, +} from "../../onboard/docker-driver-gateway-config"; +import { ensureDockerDriverGatewayJwtBundle } from "../../onboard/docker-driver-gateway-jwt-bundle"; +import { defaultUninstallPaths } from "./plan"; + +function ok(stdout = ""): RunResult { + return { status: 0, stdout, stderr: "" }; +} + +function notFound(): RunResult { + return { status: 1, stdout: "", stderr: "" }; +} + +function sandboxAbsent(name: string): RunResult { + return { status: 1, stdout: "", stderr: `sandbox ${name} not found` }; +} + +function runUninstallPlan(options: UninstallRunOptions, deps: UninstallRunDeps) { + return runUninstallPlanBase(options, { + resolveGatewayTeardownAuthority: ({ gatewayName, gatewayPort }) => ({ + gatewayName, + gatewayPort, + mode: "nemoclaw-managed", + source: gatewayPort === 8080 ? "packaged-service" : "standalone", + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }), + ...deps, + }); +} + +function okWithKnownGatewayList(command: string, args: readonly string[]): RunResult { + return command === "openshell" && args[0] === "gateway" && args[1] === "list" + ? ok(JSON.stringify([{ name: "nemoclaw" }])) + : ok(); +} + +function sharedOpenShellTeardownWasCalled( + calls: readonly (readonly [string, readonly string[]])[], +): boolean { + return calls.some( + ([command, args]) => + command === "openshell" && + ((args[0] === "provider" && args[1] === "delete") || + (args[0] === "gateway" && ["destroy", "remove"].includes(String(args[1])))), + ); +} + +function modeledSandboxStatus( + sandboxName: string, + registeredSandboxes: ReadonlySet, + calls: readonly (readonly [string, readonly string[]])[], + sharedOpenShellFilesAvailable: boolean, +): RunResult { + return new Map([ + [true, ok(`${sandboxName} usable`)], + [false, { status: 1, stdout: "", stderr: `${sandboxName} unavailable` }], + ]).get( + registeredSandboxes.has(sandboxName) && + !sharedOpenShellTeardownWasCalled(calls) && + sharedOpenShellFilesAvailable, + )!; +} + +const temporaryDirectories: string[] = []; + +function sharedOpenShellFixture(prefix: string) { + const homeDir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + temporaryDirectories.push(homeDir); + const paths = defaultUninstallPaths({ home: homeDir }); + const localInstallPaths = paths.openshellInstallPaths.filter((target) => + target.startsWith(homeDir), + ); + for (const target of localInstallPaths) fs.mkdirSync(path.dirname(target), { recursive: true }); + fs.mkdirSync(paths.gatewayLocalStateDir, { recursive: true }); + fs.mkdirSync(paths.openshellConfigDir, { recursive: true }); + return { + homeDir, + sharedPaths: new Set([ + paths.gatewayLocalStateDir, + paths.openshellConfigDir, + ...localInstallPaths, + ]), + }; +} + +function writeAdmissionReceipt(homeDir: string, stateDir: string): string { + const target = portableDemoReceiptPath("alpha", stateDir); + const uid = process.getuid?.() ?? 1001; + fs.mkdirSync(path.dirname(target), { mode: 0o700, recursive: true }); + fs.writeFileSync( + target, + `${JSON.stringify({ + schemaVersion: 4, + sandboxName: "alpha", + sandboxId: "sandbox-alpha", + containerId: "a".repeat(64), + dashboardPort: 18789, + registryGeneration: "a".repeat(64), + runtimeAuthority: { + schemaVersion: 1, + kind: "podman", + ownership: "current-user", + uid, + homeDir, + configHome: path.join(homeDir, ".config"), + runtimeDir: `/run/user/${uid}`, + socketPath: `/run/user/${uid}/podman/podman.sock`, + }, + })}\n`, + { mode: 0o600 }, + ); + return target; +} + +function admissionFailureScope(prefix: string) { + const homeDir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + temporaryDirectories.push(homeDir); + const stateDir = path.join(homeDir, ".nemoclaw"); + const registry = path.join(stateDir, "sandboxes.json"); + fs.mkdirSync(stateDir, { mode: 0o700 }); + fs.writeFileSync(registry, '{"defaultSandbox":null,"sandboxes":{}}\n', { mode: 0o600 }); + return { + homeDir, + stateDir, + registry, + run: vi.fn(okWithKnownGatewayList), + runDocker: vi.fn(() => ok()), + runModelCleanup: vi.fn(() => ok()), + rmSync: vi.fn(), + kill: vi.fn(() => true), + runPortableCleanup: vi.fn(), + }; +} + +function admissionFailureDeps(scope: ReturnType): UninstallRunDeps { + return { + commandExists: () => false, + env: { HOME: scope.homeDir }, + hasPortableRuntimeCleanup, + isTty: false, + kill: scope.kill, + log: vi.fn(), + rmSync: scope.rmSync, + run: scope.run, + runDocker: scope.runDocker, + runDualStationRuntimeCleanup: scope.runModelCleanup, + runHuggingFaceCacheDataCleanup: scope.runModelCleanup, + runLocalModelRuntimeCleanup: scope.runModelCleanup, + runManagedLlamaCppRuntimeCleanup: scope.runModelCleanup, + runPortableRuntimeCleanupTransaction: scope.runPortableCleanup, + withPortableHostFence: async (_home, operation) => await operation(), + }; +} + +function directoryEvidence(target: string, mode: number, entries = 0): string { + fs.mkdirSync(target, { mode, recursive: true }); + for (let index = 0; index < entries; index++) + fs.writeFileSync(path.join(target, `extra-${index}`), "x"); + return target; +} + +function symlinkEvidence(target: string, source: string): string { + fs.mkdirSync(path.dirname(target), { mode: 0o700, recursive: true }); + fs.symlinkSync(source, target); + return target; +} + +function deferredEvidence(target: string, create: (target: string) => void) { + return { evidence: target, arm: () => create(target) }; +} + +type EvidenceMutation = (home: string, state: string) => string; +type DeferredMutation = (home: string, state: string) => ReturnType; + +afterEach(() => { + vi.restoreAllMocks(); + for (const directory of temporaryDirectories.splice(0)) { + fs.rmSync(directory, { force: true, recursive: true }); + } +}); + +describe("portable runtime cleanup in the uninstall run plan", () => { + it.each<[string, EvidenceMutation]>([ + ["receipt without configuration", (home, state) => writeAdmissionReceipt(home, state)], + [ + "forged configuration cleanup", + (home) => { + const target = path.join( + home, + ".config/nemoclaw/portable", + `.containers.conf.portable-uninstall-${"e".repeat(64)}.cleanup`, + ); + fs.mkdirSync(path.dirname(target), { mode: 0o700, recursive: true }); + fs.writeFileSync(target, "unknown", { mode: 0o600 }); + return target; + }, + ], + [ + "retirement record with replacement authority", + (home, state) => { + const receipt = writeAdmissionReceipt(home, state); + const registry = path.join(state, "sandboxes.json"); + const config = path.join(home, ".config/nemoclaw/portable/containers.conf"); + fs.mkdirSync(path.dirname(config), { mode: 0o700, recursive: true }); + fs.writeFileSync(config, "[engine]\n", { mode: 0o600 }); + fs.writeFileSync(registry, '{"sandboxes":{"alpha":{"name":"alpha"}}}\n', { + mode: 0o600, + }); + publishAndRetirePortableEvidence(preparePortableRetirement(home, [path.basename(receipt)])); + fs.writeFileSync(registry, "{}\n", { mode: 0o600 }); + return path.join(state, "portable-uninstall-retirement.json"); + }, + ], + ["unsafe state root", (_home, state) => (fs.chmodSync(state, 0o777), state)], + [ + "unsafe receipt root", + (_home, state) => directoryEvidence(path.join(state, "portable-demo-lifecycle"), 0o755), + ], + [ + "unsafe configuration root", + (home) => directoryEvidence(path.join(home, ".config/nemoclaw/portable"), 0o755), + ], + [ + "symlinked receipt root", + (_home, state) => symlinkEvidence(path.join(state, "portable-demo-lifecycle"), state), + ], + [ + "symlinked configuration root", + (home, state) => symlinkEvidence(path.join(home, ".config/nemoclaw/portable"), state), + ], + [ + "excess receipt entries", + (_home, state) => + directoryEvidence(path.join(state, "portable-demo-lifecycle"), 0o700, 1_025), + ], + [ + "excess configuration entries", + (home) => directoryEvidence(path.join(home, ".config/nemoclaw/portable"), 0o700, 1_025), + ], + ])("rejects %s before generic effects (#9189)", (_case, mutate) => { + const scope = admissionFailureScope("nemoclaw-portable-admission-"); + const evidence = mutate(scope.homeDir, scope.stateDir); + const expectedRegistry = fs.readFileSync(scope.registry, "utf8"); + const result = runUninstallPlan( + { assumeYes: true, deleteModels: true, destroyUserData: true, keepOpenShell: false }, + admissionFailureDeps(scope), + ); + expect(result.exitCode).toBe(1); + expect( + [ + scope.run, + scope.runDocker, + scope.runModelCleanup, + scope.rmSync, + scope.kill, + scope.runPortableCleanup, + ].every((effect) => effect.mock.calls.length === 0), + ).toBe(true); + expect(fs.readFileSync(scope.registry, "utf8")).toBe(expectedRegistry); + expect(fs.existsSync(evidence)).toBe(true); + }); + + it.each<[string, string, DeferredMutation]>([ + [ + "HOME gateway-limit", + "Managed llama.cpp cleanup could not safely inventory gateway-scoped ownership state.", + (_home, state) => + deferredEvidence(path.join(state, "gateways"), (evidence) => { + fs.mkdirSync(evidence, { recursive: true }); + for (let index = 0; index <= 1_024; index++) + fs.writeFileSync(path.join(evidence, `x${index}`), ""); + }), + ], + [ + "HOME gateway-file", + "Managed llama.cpp cleanup could not safely inventory gateway-scoped ownership state.", + (_home, state) => + deferredEvidence(path.join(state, "gateways", "8090"), (evidence) => { + fs.mkdirSync(path.dirname(evidence), { recursive: true }); + fs.writeFileSync(evidence, "unsafe"); + }), + ], + [ + "HOME receipt-inventory", + "Could not inspect managed distributed vLLM rollback state.", + (home, state) => { + let armed = false; + const readdir = fs.readdirSync.bind(fs); + vi.spyOn(fs, "readdirSync").mockImplementation(((target: fs.PathLike, options?: any) => + armed && String(target) === state + ? assert.fail(`${home}/receipt`) + : readdir(target, options)) as typeof fs.readdirSync); + return { evidence: state, arm: () => (armed = true) }; + }, + ], + [ + "HOME orphan-binding", + "A managed distributed vLLM SSH binding exists without its ownership receipt.", + (_home, state) => { + const evidence = path.join(state, "dual-station-vllm-runtime.json.ssh-binding"); + fs.mkdirSync(evidence); + return { evidence, arm: () => undefined }; + }, + ], + ])("rejects %s before generic effects (#9189)", async (_case, category, prepare) => { + const scope = admissionFailureScope("nemoclaw-secret-home-sentinel-"); + const { evidence, arm } = prepare(scope.homeDir, scope.stateDir); + const stderr = vi.spyOn(console, "error").mockImplementation(() => undefined); + const result = await runUninstallPlanProduction( + { assumeYes: true, deleteModels: true, destroyUserData: true, keepOpenShell: false }, + { + ...admissionFailureDeps(scope), + commandExists: (command) => command === "openshell", + hasPortableRuntimeCleanup: () => (arm(), false), + }, + ); + const output = stderr.mock.calls.flat().join("\n"); + expect(result.exitCode).toBe(1); + expect(output).toContain(category); + expect(output).not.toContain(scope.homeDir); + expect(output).not.toContain("secret-home-sentinel"); + expect(scope.run).toHaveBeenCalled(); + expect( + scope.run.mock.calls.every( + ([cmd, args]) => cmd === "openshell" && args.join(" ") === "gateway list -o json", + ), + ).toBe(true); + expect(scope.runDocker).not.toHaveBeenCalled(); + expect(scope.runModelCleanup).not.toHaveBeenCalled(); + expect(scope.rmSync).not.toHaveBeenCalled(); + expect(scope.kill).not.toHaveBeenCalled(); + expect(scope.runPortableCleanup).not.toHaveBeenCalled(); + expect(fs.existsSync(evidence)).toBe(true); + }); + + it("uses exact receipt names without Docker or an all-sandbox mutation (#9189)", () => { + const order: string[] = []; + const logs: string[] = []; + const registeredSandboxes = new Set(["alpha", "unrelated"]); + const { homeDir, sharedPaths: sharedOpenShellPaths } = sharedOpenShellFixture( + "nemoclaw-portable-success-", + ); + const removed: string[] = []; + const runHandlers = new Map RunResult>([ + ["pgrep", notFound], + ["lsof", notFound], + [ + "openshell sandbox delete -g nemoclaw alpha", + () => { + order.push("exact-openshell"); + registeredSandboxes.delete("alpha"); + return ok(); + }, + ], + ["openshell sandbox get -g nemoclaw alpha", () => sandboxAbsent("alpha")], + ["openshell status -g nemoclaw", () => ok("Status: Connected\nGateway: nemoclaw\n")], + ]); + const run = vi.fn((command: string, args: string[]) => + ( + runHandlers.get(`${command} ${args.join(" ")}`) ?? + runHandlers.get(command) ?? + (() => okWithKnownGatewayList(command, args)) + )(), + ); + const runPortableCleanup = vi.fn( + ( + _input: PortableRuntimeCleanupInput, + continueAfterSandboxRemoval: ( + removed: number, + sandboxNames: readonly string[], + gatewayName: string, + ) => boolean, + ) => { + order.push("exact-sandbox"); + expect(continueAfterSandboxRemoval(1, ["alpha"], "nemoclaw")).toBe(true); + order.push("exact-shared"); + return { + registryRemoved: true, + sandboxContainersRemoved: 1, + selectorsRemoved: ["CONTAINERS_CONF", "NETAVARK_FW"], + }; + }, + ); + const runDocker = vi.fn(() => ok("")); + const kill = vi.fn(() => true); + + const result = runUninstallPlan( + { assumeYes: true, deleteModels: true, keepOpenShell: false }, + { + commandExists: (command) => ["openshell", "pgrep", "lsof", "docker"].includes(command), + env: { HOME: homeDir } as NodeJS.ProcessEnv, + existsSync: (target) => sharedOpenShellPaths.has(String(target)), + hasPortableRuntimeCleanup: () => true, + isTty: false, + kill, + log: (line) => logs.push(line), + rmSync: vi.fn((target: fs.PathLike) => removed.push(String(target))), + run, + runDocker, + runPortableRuntimeCleanupTransaction: runPortableCleanup, + }, + ); + + expect(result.exitCode).toBe(0); + expect(order).toEqual(["exact-sandbox", "exact-openshell", "exact-shared"]); + expect(runPortableCleanup).toHaveBeenCalledOnce(); + expect(runDocker).not.toHaveBeenCalled(); + expect(kill).not.toHaveBeenCalled(); + expect(removed).toEqual([]); + expect(run.mock.calls.every(([command]) => command === "openshell")).toBe(true); + expect(run).toHaveBeenCalledWith( + "openshell", + ["sandbox", "delete", "-g", "nemoclaw", "alpha"], + expect.anything(), + ); + expect( + run.mock.calls.some( + ([command, args]) => command === "openshell" && args.join(" ") === "sandbox delete --all", + ), + ).toBe(false); + expect( + modeledSandboxStatus( + "unrelated", + registeredSandboxes, + run.mock.calls, + [...sharedOpenShellPaths].every((target) => !removed.includes(target)), + ), + ).toEqual(ok("unrelated usable")); + expect(logs).toContain("Kept Podman images and containers outside receipt-owned cleanup."); + expect(logs).toContain( + "Kept shared OpenShell provider and gateway registrations for unrelated sandboxes.", + ); + expect(logs).toContain("Removed the managed portable registry container."); + expect(logs.join("\n")).toContain("dictionary-testable pseudonymous fingerprints"); + expect(logs.join("\n")).toContain("another process running as this user can change it"); + }); + + it("keeps explicit receipt gateway scope when ambient selection drifts (#9189)", () => { + const registeredSandboxes = new Set([ + "nemoclaw/alpha", + "nemoclaw/beta", + "other/alpha", + "other/beta", + "other/unrelated", + ]); + let ambientGateway = "other"; + const { homeDir, sharedPaths: sharedOpenShellPaths } = sharedOpenShellFixture( + "nemoclaw-portable-gateway-scope-", + ); + const removed: string[] = []; + let statusCalls = 0; + const runHandlers = new Map RunResult>([ + [ + "openshell status -g nemoclaw", + () => { + ambientGateway = statusCalls++ % 2 === 0 ? "drifted-after-proof" : ambientGateway; + return ok("Status: Connected\nGateway: nemoclaw\n"); + }, + ], + [ + "openshell sandbox delete -g nemoclaw alpha", + () => { + registeredSandboxes.delete("nemoclaw/alpha"); + ambientGateway = "other"; + return ok(); + }, + ], + [ + "openshell sandbox delete -g nemoclaw beta", + () => { + registeredSandboxes.delete("nemoclaw/beta"); + ambientGateway = "other"; + return ok(); + }, + ], + ["openshell sandbox get -g nemoclaw alpha", () => sandboxAbsent("alpha")], + ["openshell sandbox get -g nemoclaw beta", () => sandboxAbsent("beta")], + ["pgrep", notFound], + ["lsof", notFound], + ]); + const run = vi.fn((command: string, args: string[]) => + ( + runHandlers.get(`${command} ${args.join(" ")}`) ?? + runHandlers.get(command) ?? + (() => okWithKnownGatewayList(command, args)) + )(), + ); + const runPortableCleanup = vi.fn( + ( + _input: PortableRuntimeCleanupInput, + continueAfterSandboxRemoval: ( + removedCount: number, + sandboxNames: readonly string[], + gatewayName: string, + ) => boolean, + ) => { + expect(continueAfterSandboxRemoval(2, ["alpha", "beta"], "nemoclaw")).toBe(true); + return { + registryRemoved: true, + sandboxContainersRemoved: 2, + selectorsRemoved: ["CONTAINERS_CONF"], + }; + }, + ); + + const result = runUninstallPlan( + { assumeYes: true, deleteModels: false, keepOpenShell: false }, + { + commandExists: (command) => ["openshell", "pgrep", "lsof"].includes(command), + env: { HOME: homeDir }, + existsSync: (target) => sharedOpenShellPaths.has(String(target)), + hasPortableRuntimeCleanup: () => true, + isTty: false, + log: vi.fn(), + rmSync: vi.fn((target: fs.PathLike) => removed.push(String(target))), + run, + runDocker: () => ok(""), + runPortableRuntimeCleanupTransaction: runPortableCleanup, + }, + ); + + expect(result.exitCode).toBe(0); + expect(ambientGateway).toBe("other"); + expect( + run.mock.calls + .filter(([command, args]) => command === "openshell" && args[0] === "status") + .map(([, args]) => args), + ).toEqual([ + ["status", "-g", "nemoclaw"], + ["status", "-g", "nemoclaw"], + ["status", "-g", "nemoclaw"], + ["status", "-g", "nemoclaw"], + ]); + expect( + run.mock.calls + .filter(([command, args]) => command === "openshell" && args[0] === "sandbox") + .map(([, args]) => args), + ).toEqual([ + ["sandbox", "delete", "-g", "nemoclaw", "alpha"], + ["sandbox", "get", "-g", "nemoclaw", "alpha"], + ["sandbox", "delete", "-g", "nemoclaw", "beta"], + ["sandbox", "get", "-g", "nemoclaw", "beta"], + ]); + expect( + run.mock.calls.some( + ([command, args]) => + command === "openshell" && args[0] === "gateway" && args[1] === "select", + ), + ).toBe(false); + expect(registeredSandboxes).toEqual(new Set(["other/alpha", "other/beta", "other/unrelated"])); + expect( + modeledSandboxStatus( + "other/alpha", + registeredSandboxes, + run.mock.calls, + [...sharedOpenShellPaths].every((target) => !removed.includes(target)), + ), + ).toEqual(ok("other/alpha usable")); + }); + + it("accepts exact structured sandbox absence only after proving gateway reachability (#9189)", () => { + const registeredSandboxes = new Set(["unrelated"]); + const { homeDir, sharedPaths: sharedOpenShellPaths } = sharedOpenShellFixture( + "nemoclaw-portable-absent-", + ); + const removed: string[] = []; + const error = "code:'Some requested entity was not found',message:'sandbox not found'"; + const runHandlers = new Map RunResult>([ + ["openshell sandbox delete -g nemoclaw alpha", () => sandboxAbsent("alpha")], + ["openshell sandbox get -g nemoclaw alpha", () => ({ status: 1, stdout: "", stderr: error })], + ["openshell status -g nemoclaw", () => ok("Status: Connected\nGateway: nemoclaw\n")], + ["pgrep", notFound], + ["lsof", notFound], + ]); + const run = vi.fn((command: string, args: string[]) => + ( + runHandlers.get(`${command} ${args.join(" ")}`) ?? + runHandlers.get(command) ?? + (() => okWithKnownGatewayList(command, args)) + )(), + ); + const runPortableCleanup = vi.fn( + ( + _input: PortableRuntimeCleanupInput, + continueAfterSandboxRemoval: ( + removed: number, + sandboxNames: readonly string[], + gatewayName: string, + ) => boolean, + ) => { + expect(continueAfterSandboxRemoval(1, ["alpha"], "nemoclaw")).toBe(true); + return { + registryRemoved: true, + sandboxContainersRemoved: 1, + selectorsRemoved: ["CONTAINERS_CONF"], + }; + }, + ); + + const result = runUninstallPlan( + { assumeYes: true, deleteModels: false, keepOpenShell: false }, + { + commandExists: (command) => ["openshell", "pgrep", "lsof"].includes(command), + env: { HOME: homeDir }, + existsSync: (target) => sharedOpenShellPaths.has(String(target)), + hasPortableRuntimeCleanup: () => true, + isTty: false, + log: vi.fn(), + rmSync: vi.fn((target: fs.PathLike) => removed.push(String(target))), + run, + runDocker: () => ok(""), + runPortableRuntimeCleanupTransaction: runPortableCleanup, + }, + ); + + expect(result.exitCode).toBe(0); + expect(run).toHaveBeenCalledWith("openshell", ["status", "-g", "nemoclaw"], expect.anything()); + expect( + modeledSandboxStatus( + "unrelated", + registeredSandboxes, + run.mock.calls, + [...sharedOpenShellPaths].every((target) => !removed.includes(target)), + ), + ).toEqual(ok("unrelated usable")); + }); + + it.each([ + ["eventual exact absence", 3, 0, 2], + ["exit zero while the sandbox remains", Number.POSITIVE_INFINITY, 1, 4], + ])( + "%s after a scoped delete uses bounded exact-name verification (#9189)", + (_case, absentAttempt, expectedExit, expectedSleeps) => { + let getCalls = 0; + const sleep = vi.fn(); + const runHandlers = new Map RunResult>([ + ["openshell status -g nemoclaw", () => ok("Status: Connected\nGateway: nemoclaw\n")], + ["openshell sandbox delete -g nemoclaw alpha", () => ok()], + [ + "openshell sandbox get -g nemoclaw alpha", + () => + ++getCalls >= absentAttempt ? sandboxAbsent("alpha") : ok("sandbox alpha present"), + ], + ["pgrep", notFound], + ["lsof", notFound], + ]); + const run = vi.fn((command: string, args: string[]) => + ( + runHandlers.get(`${command} ${args.join(" ")}`) ?? + runHandlers.get(command) ?? + (() => okWithKnownGatewayList(command, args)) + )(), + ); + const runPortableCleanup = vi.fn( + ( + _input: PortableRuntimeCleanupInput, + continueAfterSandboxRemoval: ( + removed: number, + sandboxNames: readonly string[], + gatewayName: string, + ) => boolean, + ) => + continueAfterSandboxRemoval(1, ["alpha"], "nemoclaw") + ? { registryRemoved: true, sandboxContainersRemoved: 1, selectorsRemoved: [] } + : null, + ); + + const result = runUninstallPlan( + { assumeYes: true, deleteModels: false, destroyUserData: true, keepOpenShell: false }, + { + commandExists: (command) => ["openshell", "pgrep", "lsof"].includes(command), + env: { HOME: "/tmp/nemoclaw-bounded-absence-9189" }, + existsSync: () => false, + hasPortableRuntimeCleanup: () => true, + isTty: false, + log: vi.fn(), + rmSync: vi.fn(), + run, + runDocker: vi.fn(() => ok()), + runPortableRuntimeCleanupTransaction: runPortableCleanup, + sleep, + }, + ); + + expect(result.exitCode).toBe(expectedExit); + expect(getCalls).toBe(expectedExit === 0 ? absentAttempt : 5); + expect(sleep).toHaveBeenCalledTimes(expectedSleeps); + expect(runPortableCleanup).toHaveBeenCalledOnce(); + }, + ); + + it.each([ + [ + "gateway missing", + 'Error: status: NotFound, message: "gateway nemoclaw not found"', + ok("Status: Connected\nGateway: nemoclaw\n"), + "OpenShell sandbox 'alpha' could not be removed", + ], + [ + "provider missing", + 'Error: status: NotFound, message: "provider nvidia-nim not found"', + ok("Status: Connected\nGateway: nemoclaw\n"), + "OpenShell sandbox 'alpha' could not be removed", + ], + [ + "transport failure", + "Error: transport failure: connection refused", + ok("Status: Connected\nGateway: nemoclaw\n"), + "OpenShell sandbox 'alpha' could not be removed", + ], + [ + "generic NotFound", + "Error: NotFound: requested entity is missing", + ok("Status: Connected\nGateway: nemoclaw\n"), + "OpenShell sandbox 'alpha' could not be removed", + ], + [ + "explicit gateway scoping rejected", + "Error: unknown option '-g'", + ok("Status: Connected\nGateway: nemoclaw\n"), + "OpenShell sandbox 'alpha' could not be removed", + ], + [ + "unreachable gateway after exact named absence", + "Error: sandbox alpha not found", + { status: 1, stdout: "", stderr: "connection refused" }, + "Portable OpenShell cleanup requires connected gateway 'nemoclaw'", + ], + ])( + "rejects %s and preserves all retry evidence (#9189)", + (_caseName, deleteError, statusResult, expectedError) => { + const removed: string[] = []; + const errors: string[] = []; + const registeredSandboxes = new Set(["alpha", "unrelated"]); + const portableEvidence = { + config: true, + gatewayRegistration: true, + providerRegistration: true, + receipt: true, + registryContainer: true, + registryRow: true, + selectors: true, + state: true, + }; + const { homeDir, sharedPaths: sharedOpenShellPaths } = sharedOpenShellFixture( + "nemoclaw-portable-failure-", + ); + const runHandlers = new Map RunResult>([ + [ + "openshell sandbox delete -g nemoclaw alpha", + () => ({ status: 1, stdout: "", stderr: deleteError }), + ], + ["openshell status -g nemoclaw", () => statusResult], + ["pgrep", notFound], + ["lsof", notFound], + ]); + const run = vi.fn((command: string, args: string[]) => + ( + runHandlers.get(`${command} ${args.join(" ")}`) ?? + runHandlers.get(command) ?? + (() => okWithKnownGatewayList(command, args)) + )(), + ); + const runPortableCleanup = vi.fn( + ( + _input: PortableRuntimeCleanupInput, + continueAfterSandboxRemoval: ( + removed: number, + sandboxNames: readonly string[], + gatewayName: string, + ) => boolean, + ) => { + const continued = continueAfterSandboxRemoval(1, ["alpha"], "nemoclaw"); + const finishSharedCleanup = () => { + Object.assign(portableEvidence, { + config: false, + gatewayRegistration: false, + providerRegistration: false, + receipt: false, + registryContainer: false, + registryRow: false, + selectors: false, + state: false, + }); + return { + registryRemoved: true, + sandboxContainersRemoved: 1, + selectorsRemoved: ["CONTAINERS_CONF"], + }; + }; + return new Map ReturnType | null>([ + [false, () => null], + [true, finishSharedCleanup], + ]).get(continued)!(); + }, + ); + + const result = runUninstallPlan( + { + assumeYes: true, + deleteModels: false, + destroyUserData: true, + keepOpenShell: false, + }, + { + commandExists: (command) => ["openshell", "pgrep", "lsof"].includes(command), + env: { HOME: homeDir }, + error: (line) => errors.push(line), + existsSync: (target) => sharedOpenShellPaths.has(String(target)), + hasPortableRuntimeCleanup: () => true, + isTty: false, + log: vi.fn(), + rmSync: vi.fn((target: fs.PathLike) => removed.push(String(target))), + run, + runDocker: () => ok(""), + runPortableRuntimeCleanupTransaction: runPortableCleanup, + }, + ); + + expect(result.exitCode).toBe(1); + expect(errors.join("\n")).toContain(expectedError); + expect(removed).toEqual([]); + expect(portableEvidence).toEqual({ + config: true, + gatewayRegistration: true, + providerRegistration: true, + receipt: true, + registryContainer: true, + registryRow: true, + selectors: true, + state: true, + }); + expect( + run.mock.calls.some( + ([command, args]) => command === "openshell" && args.join(" ") === "sandbox delete --all", + ), + ).toBe(false); + expect( + modeledSandboxStatus( + "unrelated", + registeredSandboxes, + run.mock.calls, + [...sharedOpenShellPaths].every((target) => !removed.includes(target)), + ), + ).toEqual(ok("unrelated usable")); + expect(sharedOpenShellTeardownWasCalled(run.mock.calls)).toBe(false); + }, + ); + + it("preserves retry evidence after an exact cleanup failure with destroy data (#9189)", () => { + const removed: string[] = []; + const errors: string[] = []; + const run = vi.fn((command: string, args: string[]) => + ["pgrep", "lsof"].includes(command) ? notFound() : okWithKnownGatewayList(command, args), + ); + const result = runUninstallPlan( + { + assumeYes: true, + deleteModels: false, + destroyUserData: true, + keepOpenShell: false, + }, + { + commandExists: (command) => ["openshell", "pgrep", "lsof"].includes(command), + env: { HOME: "/tmp/nemoclaw-uninstall-portable-failure-9189" } as NodeJS.ProcessEnv, + error: (line) => errors.push(line), + existsSync: () => false, + hasPortableRuntimeCleanup: () => true, + isTty: false, + log: vi.fn(), + runPortableRuntimeCleanupTransaction: () => { + throw new Error("recorded container remains"); + }, + rmSync: vi.fn((target: fs.PathLike) => removed.push(String(target))), + run, + runDocker: () => ok(""), + }, + ); + + expect(result.exitCode).toBe(1); + expect(errors.join("\n")).toContain("recorded container remains"); + expect(removed).toEqual([]); + expect(run.mock.calls.some(([command]) => command === "npm")).toBe(false); + }); + + it.each(["config", "registry"] as const)( + "keeps repeated %s-stage retirement uninstalls out of generic cleanup (#9189)", + (crashTarget) => { + const homeDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-portable-repeat-")); + temporaryDirectories.push(homeDir); + const stateDir = path.join(homeDir, ".nemoclaw"); + const receipt = path.join(stateDir, "portable-demo-lifecycle", `${"a".repeat(64)}.json`); + const registry = path.join(stateDir, "sandboxes.json"); + const config = path.join(homeDir, ".config/nemoclaw/portable/containers.conf"); + fs.mkdirSync(path.dirname(receipt), { mode: 0o700, recursive: true }); + fs.mkdirSync(path.dirname(config), { mode: 0o700, recursive: true }); + fs.writeFileSync(receipt, "{}\n", { mode: 0o600 }); + fs.writeFileSync(registry, '{"sandboxes":{"alpha":{"name":"alpha"}}}\n', { mode: 0o600 }); + fs.writeFileSync(config, "[engine]\n", { mode: 0o600 }); + const unlink = fs.unlinkSync.bind(fs); + vi.spyOn(fs, "unlinkSync").mockImplementation((target) => { + String(target).includes( + `.${crashTarget === "config" ? "containers.conf" : "sandboxes.json"}.portable-uninstall-`, + ) && assert.fail("injected registry retirement crash"); + unlink(target); + }); + expect(() => + publishAndRetirePortableEvidence( + preparePortableRetirement(homeDir, [path.basename(receipt)]), + ), + ).toThrow(/injected/); + vi.restoreAllMocks(); + const stageRoot = crashTarget === "config" ? path.dirname(config) : stateDir; + const stage = path.join( + stageRoot, + fs.readdirSync(stageRoot).find((name) => name.includes(".portable-uninstall-"))!, + ); + const runDocker = vi.fn(() => ok()); + const runModelCleanup = vi.fn(() => ok()); + const kill = vi.fn(() => true); + const remove = vi.fn(fs.rmSync); + const run = vi.fn((command: string, args: string[]) => + command === "openshell" ? okWithKnownGatewayList(command, args) : notFound(), + ); + let stagedObserved = false; + const deps: UninstallRunDeps = { + commandExists: (command) => command === "openshell", + env: { HOME: homeDir }, + existsSync: (target) => String(target).startsWith(homeDir) && fs.existsSync(target), + isTty: false, + kill, + log: vi.fn(), + rmSync: remove, + run, + runDocker, + runHuggingFaceCacheDataCleanup: runModelCleanup, + runLocalModelRuntimeCleanup: runModelCleanup, + runPortableRuntimeCleanupTransaction: (input, continueAfterSandboxRemoval) => + runPortableRuntimeCleanupTransaction(input, continueAfterSandboxRemoval, { + withRegistryLock: (_registryFile, operation) => { + !stagedObserved && expect(fs.existsSync(stage)).toBe(true); + stagedObserved = true; + return operation(); + }, + }), + }; + + expect( + runUninstallPlan( + { assumeYes: true, deleteModels: true, destroyUserData: true, keepOpenShell: false }, + deps, + ).exitCode, + ).toBe(0); + expect( + runUninstallPlan( + { assumeYes: true, deleteModels: true, destroyUserData: true, keepOpenShell: false }, + deps, + ).exitCode, + ).toBe(0); + expect(runDocker).not.toHaveBeenCalled(); + expect(runModelCleanup).not.toHaveBeenCalled(); + expect(kill).not.toHaveBeenCalled(); + expect(stagedObserved).toBe(true); + expect(fs.existsSync(stage)).toBe(false); + expect(remove.mock.calls.map(([target]) => String(target))).not.toContain(stage); + expect(run.mock.calls.every(([command]) => command === "openshell")).toBe(true); + expect(fs.existsSync(path.join(stateDir, "portable-uninstall-retirement.json"))).toBe(true); + }, + ); + + it("stops state deletion when portable state changes after sandbox removal (#9189)", () => { + const errors: string[] = []; + const removed: string[] = []; + const runPortableCleanup = vi.fn( + ( + _input: PortableRuntimeCleanupInput, + continueAfterSandboxRemoval: ( + removed: number, + sandboxNames: readonly string[], + gatewayName: string, + ) => boolean, + ) => { + continueAfterSandboxRemoval(1, ["alpha"], "nemoclaw"); + throw new Error( + "Portable lifecycle or registry state changed during exact uninstall cleanup", + ); + }, + ); + + const result = runUninstallPlan( + { + assumeYes: true, + deleteModels: false, + destroyUserData: true, + keepOpenShell: false, + }, + { + commandExists: (command) => ["openshell", "pgrep", "lsof"].includes(command), + env: { HOME: "/tmp/nemoclaw-uninstall-portable-interleaving-9189" } as NodeJS.ProcessEnv, + error: (line) => errors.push(line), + existsSync: () => false, + hasPortableRuntimeCleanup: () => true, + isTty: false, + log: vi.fn(), + rmSync: vi.fn((target: fs.PathLike) => removed.push(String(target))), + run: (command, args) => + ["pgrep", "lsof"].includes(command) ? notFound() : okWithKnownGatewayList(command, args), + runDocker: () => ok(""), + runPortableRuntimeCleanupTransaction: runPortableCleanup, + }, + ); + + expect(result.exitCode).toBe(1); + expect(runPortableCleanup).toHaveBeenCalledOnce(); + expect(errors.join("\n")).toContain("lifecycle or registry state changed"); + expect(removed).toEqual([]); + }); + + it("keeps detected portable receipts during a sibling-gateway scoped pass (#9189)", () => { + const homeDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-portable-sibling-")); + const stateDir = path.join(homeDir, ".nemoclaw"); + const uninstallPaths = defaultUninstallPaths({ home: homeDir }); + const gatewayStateDir = uninstallPaths.selectedGatewayLocalStateDir; + const jwt = ensureDockerDriverGatewayJwtBundle(gatewayStateDir); + fs.mkdirSync(gatewayStateDir, { mode: 0o700, recursive: true }); + fs.writeFileSync( + path.join(gatewayStateDir, "openshell-gateway.toml"), + buildDockerDriverGatewayConfigToml( + { + OPENSHELL_GRPC_ENDPOINT: "https://127.0.0.1:8080", + OPENSHELL_LOCAL_TLS_DIR: path.join(gatewayStateDir, "tls"), + OPENSHELL_DOCKER_NETWORK_NAME: "openshell-docker", + OPENSHELL_DOCKER_SUPERVISOR_IMAGE: "supervisor:test", + }, + "/usr/bin/openshell-sandbox", + jwt, + gatewayIdForStateDir(gatewayStateDir), + ), + { mode: 0o600 }, + ); + writeManagedGatewayRuntimeProof(gatewayStateDir, 8080); + const receiptFile = portableDemoReceiptPath("alpha", stateDir); + const containerId = "a".repeat(64); + fs.mkdirSync(path.dirname(receiptFile), { mode: 0o700, recursive: true }); + fs.writeFileSync( + receiptFile, + `${JSON.stringify({ + schemaVersion: 4, + sandboxName: "alpha", + sandboxId: "sandbox-alpha", + containerId, + dashboardPort: 18789, + registryGeneration: containerId, + runtimeAuthority: { + schemaVersion: 1, + kind: "podman", + ownership: "current-user", + uid: process.getuid?.() ?? 1001, + homeDir, + configHome: path.join(homeDir, ".config"), + runtimeDir: path.join("/run/user", String(process.getuid?.() ?? 1001)), + socketPath: path.join( + "/run/user", + String(process.getuid?.() ?? 1001), + "podman/podman.sock", + ), + }, + })}\n`, + { mode: 0o600 }, + ); + fs.writeFileSync( + path.join(stateDir, "sandboxes.json"), + `${JSON.stringify({ + defaultSandbox: "alpha", + sandboxes: { + alpha: { + name: "alpha", + agent: "openclaw", + gatewayName: "nemoclaw", + gatewayPort: 8080, + openshellDriver: "docker", + lifecycleGeneration: containerId, + }, + beta: { + name: "beta", + agent: "openclaw", + gatewayName: "nemoclaw-9000", + gatewayPort: 9000, + openshellDriver: "docker", + lifecycleGeneration: "b".repeat(64), + }, + }, + })}\n`, + { mode: 0o600 }, + ); + const config = path.join(homeDir, ".config/nemoclaw/portable/containers.conf"); + fs.mkdirSync(path.dirname(config), { mode: 0o700, recursive: true }); + fs.writeFileSync(config, "[engine]\n", { mode: 0o600 }); + const detectPortable = vi.fn(hasPortableRuntimeCleanup); + const runPortableCleanup = vi.fn(() => ({ + registryRemoved: true, + sandboxContainersRemoved: 1, + selectorsRemoved: [], + })); + try { + expect(hasPortableRuntimeCleanup(stateDir)).toBe(true); + const result = runUninstallPlan( + { assumeYes: true, deleteModels: false, keepOpenShell: false }, + withProvenManagedGatewayProcess({ + commandExists: (command) => ["openshell", "pgrep", "lsof"].includes(command), + env: { HOME: homeDir } as NodeJS.ProcessEnv, + existsSync: fs.existsSync, + hasPortableRuntimeCleanup: detectPortable, + isPortFree: () => true, + isTty: false, + log: vi.fn(), + rmSync: fs.rmSync, + run: (command, args) => + ["pgrep", "lsof"].includes(command) + ? notFound() + : command === "openshell" && args.join(" ") === "gateway list -o json" + ? ok(JSON.stringify([{ name: "nemoclaw" }, { name: "nemoclaw-9000" }])) + : ok(), + runDocker: () => ok(""), + runPortableRuntimeCleanupTransaction: runPortableCleanup, + }), + ); + + expect(result).toMatchObject({ exitCode: 0, otherGatewayEnvironmentsRemain: true }); + expect(detectPortable).not.toHaveBeenCalled(); + expect(runPortableCleanup).not.toHaveBeenCalled(); + expect(fs.existsSync(receiptFile)).toBe(true); + expect(fs.existsSync(path.join(stateDir, "sandboxes.json"))).toBe(true); + } finally { + fs.rmSync(homeDir, { force: true, recursive: true }); + } + }); + + it("leaves keep-openshell and external-supervisor flows unchanged (#9189)", () => { + const hasPortable = vi.fn(() => true); + const runPortableCleanup = vi.fn(() => ({ + registryRemoved: true, + sandboxContainersRemoved: 1, + selectorsRemoved: [], + })); + const baseDeps: UninstallRunDeps = { + commandExists: (command) => command === "openshell", + env: { HOME: "/tmp/nemoclaw-uninstall-portable-unchanged-9189" } as NodeJS.ProcessEnv, + existsSync: () => false, + hasPortableRuntimeCleanup: hasPortable, + isTty: false, + log: vi.fn(), + rmSync: vi.fn(), + run: vi.fn(okWithKnownGatewayList), + runDocker: () => ok(""), + runPortableRuntimeCleanupTransaction: runPortableCleanup, + }; + + expect( + runUninstallPlan({ assumeYes: true, deleteModels: false, keepOpenShell: true }, baseDeps) + .exitCode, + ).toBe(0); + expect( + runUninstallPlan( + { assumeYes: true, deleteModels: false, keepOpenShell: false }, + { + ...baseDeps, + resolveGatewayTeardownAuthority: ({ gatewayName, gatewayPort }) => ({ + gatewayName, + gatewayPort, + mode: "externally-supervised", + source: "declared", + endpoint: "https://127.0.0.1:8080", + stateDir: "/srv/external-openshell", + supervisor: { + kind: "systemd-system", + serviceName: "external-openshell.service", + execPath: "/usr/local/bin/openshell-gateway", + }, + requiredCapabilities: [], + }), + }, + ).exitCode, + ).toBe(0); + expect(hasPortable).not.toHaveBeenCalled(); + expect(runPortableCleanup).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/actions/uninstall/run-plan.test.ts b/src/lib/actions/uninstall/run-plan.test.ts index 0b38a5b735a..71b5280d965 100644 --- a/src/lib/actions/uninstall/run-plan.test.ts +++ b/src/lib/actions/uninstall/run-plan.test.ts @@ -161,6 +161,7 @@ describe("uninstall run plan", () => { command !== "docker" && command !== "lsof" && command !== "pgrep", env: { HOME: tmpHome } as NodeJS.ProcessEnv, existsSync: (target) => existing.has(target), + hasPortableRuntimeCleanup: () => false, isTty: false, log: (line) => logs.push(line), rmSync: vi.fn((target: fs.PathLike) => { @@ -206,6 +207,7 @@ describe("uninstall run plan", () => { command !== "docker" && command !== "lsof" && command !== "pgrep", env: { HOME: tmpHome } as NodeJS.ProcessEnv, existsSync: (target) => target === hermesShim || target === deepagentsShim, + hasPortableRuntimeCleanup: () => false, isTty: false, log: () => {}, rmSync: vi.fn((target: fs.PathLike) => { @@ -251,6 +253,7 @@ describe("uninstall run plan", () => { command !== "docker" && command !== "lsof" && command !== "pgrep", env: { HOME: tmpHome } as NodeJS.ProcessEnv, existsSync: (target) => target === hermesShim || target === deepagentsShim, + hasPortableRuntimeCleanup: () => false, isTty: false, log: () => {}, rmSync: vi.fn((target: fs.PathLike) => { @@ -1453,6 +1456,7 @@ describe("uninstall run plan", () => { commandExists: () => true, env: { HOME: "/tmp/nemoclaw-uninstall-test-3516" } as NodeJS.ProcessEnv, existsSync: () => false, + hasPortableRuntimeCleanup: () => false, isTty: false, kill: (pid) => { killed.push(pid); diff --git a/src/lib/actions/uninstall/run-plan.ts b/src/lib/actions/uninstall/run-plan.ts index 9360146fcc1..868f236abcb 100644 --- a/src/lib/actions/uninstall/run-plan.ts +++ b/src/lib/actions/uninstall/run-plan.ts @@ -22,12 +22,6 @@ import { sandboxDeleteAbsentMessage, sandboxDeleteFailureMessage, } from "../../domain/uninstall/messaging"; -import { - defaultUninstallPaths, - NEMOCLAW_PROVIDERS, - type UninstallPaths, -} from "../../domain/uninstall/paths"; -import { buildUninstallPlan, type UninstallPlan } from "../../domain/uninstall/plan"; import { cleanupManagedLlamaCppRuntimeForSandbox, HOST_LOCAL_VLLM_CONTAINER_NAME, @@ -81,7 +75,24 @@ import { stopHttpsPinRuntimeAdapter, stopOpenRouterRuntimeAdapter, } from "./openrouter-runtime-adapter-cleanup"; -import { classifyShimPath, type FileSystemDeps } from "./plan"; +import { + buildUninstallPlan, + classifyShimPath, + defaultUninstallPaths, + NEMOCLAW_PROVIDERS, + type FileSystemDeps, + type UninstallPaths, + type UninstallPlan, +} from "./plan"; +import { + hasPortableRuntimeCleanup, + PORTABLE_RETIREMENT_STATE_ENTRIES, + portableRetirementPreservationEntries, + runPortableRuntimeCleanupTransaction, + type PortableRuntimeCleanupInput, + type PortableRuntimeCleanupResult, + withPortableHostFence, +} from "./portable-runtime-cleanup"; export interface RunResult { status: number | null; @@ -123,8 +134,19 @@ export interface UninstallRunDeps { runHuggingFaceCacheDataCleanup?: (options?: SpawnSyncOptions) => RunResult; runLocalModelRuntimeCleanup?: (options?: SpawnSyncOptions) => RunResult; runManagedLlamaCppRuntimeCleanup?: (sandboxName: string, gatewayPort: number) => RunResult; + sleep?: (milliseconds: number) => void; + hasPortableRuntimeCleanup?: (stateDir: string) => boolean; + runPortableRuntimeCleanupTransaction?: ( + input: PortableRuntimeCleanupInput, + continueAfterSandboxRemoval: ( + removed: number, + sandboxNames: readonly string[], + gatewayName: string, + ) => boolean, + ) => PortableRuntimeCleanupResult | null; stderrHasColors?: boolean; stderrIsTty?: boolean; + withPortableHostFence?: typeof withPortableHostFence; } export interface UninstallRunOutcome { @@ -296,6 +318,12 @@ const OLLAMA_AUTH_PROXY_STATE_ENTRIES: readonly string[] = [ "ollama-auth-proxy.pid", "ollama-auth-proxy.status", ]; +const PORTABLE_DEFERRED_STEP_MESSAGES: Readonly> = { + "Docker resources": "Kept Podman images and containers outside receipt-owned cleanup.", + "Model stores": "Kept model stores and every Podman image during portable cleanup.", + "OpenShell resources": "Deferring exact portable resource cleanup until earlier work succeeds.", + "Stopping services": "Kept shared helper and model services during portable cleanup.", +}; // These entries can exist in the shared root without representing a running // default-port environment. Any other shared-root entry is treated @@ -311,6 +339,7 @@ const SHARED_HOST_STATE_ENTRIES = new Set([ MANAGED_VLLM_API_KEY_FILE, ...HTTPS_PIN_RUNTIME_ADAPTER_STATE_ENTRIES, ...OLLAMA_AUTH_PROXY_STATE_ENTRIES, + "portable-demo-lifecycle", ]); function isSharedHostStateEntry(entry: string): boolean { @@ -348,6 +377,7 @@ function scopedStatePreservationEntries( DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE, `${DUAL_STATION_VLLM_RUNTIME_RECEIPT_FILE}.ssh-binding`, MANAGED_VLLM_API_KEY_FILE, + "portable-demo-lifecycle", ]; } @@ -461,6 +491,16 @@ interface UninstallRuntime { runHuggingFaceCacheDataCleanup: (options?: SpawnSyncOptions) => RunResult; runLocalModelRuntimeCleanup: (options?: SpawnSyncOptions) => RunResult; runManagedLlamaCppRuntimeCleanup: (sandboxName: string, gatewayPort: number) => RunResult; + sleep: (milliseconds: number) => void; + hasPortableRuntimeCleanup: (stateDir: string) => boolean; + runPortableRuntimeCleanupTransaction: ( + input: PortableRuntimeCleanupInput, + continueAfterSandboxRemoval: ( + removed: number, + sandboxNames: readonly string[], + gatewayName: string, + ) => boolean, + ) => PortableRuntimeCleanupResult | null; stderrHasColors: boolean; stderrIsTty: boolean; warn: (message: string) => void; @@ -569,6 +609,19 @@ function buildRuntime(deps: UninstallRunDeps): UninstallRuntime { stderr: result.reason, }; }), + sleep: deps.sleep ?? sleepMs, + hasPortableRuntimeCleanup: deps.hasPortableRuntimeCleanup ?? hasPortableRuntimeCleanup, + runPortableRuntimeCleanupTransaction: + deps.runPortableRuntimeCleanupTransaction ?? + ((input, continueAfterSandboxRemoval) => + runPortableRuntimeCleanupTransaction(input, continueAfterSandboxRemoval, { + env: input.env, + platform: deps.platform, + podman: (args, env) => + (deps.run ?? defaultRun)("podman", [...args], { env: env ?? input.env }), + systemctl: (args, env) => (deps.run ?? defaultRun)("systemctl", [...args], { env }), + log: deps.log, + })), stderrHasColors: deps.stderrHasColors ?? (typeof process.stderr.hasColors === "function" && process.stderr.hasColors()), @@ -723,6 +776,59 @@ function deleteSelectedGatewaySandbox( return false; } +function portableGatewayIsReachable(runtime: UninstallRuntime, gatewayName: string): boolean { + const result = runtime.run("openshell", ["status", "-g", gatewayName], { env: runtime.env }); + if (result.status !== 0) return false; + const output = `${result.stdout}\n${result.stderr}`.replace(/\x1b\[[0-9;]*m/gu, ""); + const activeGateway = /^\s*Gateway:\s+(.+?)\s*$/mu.exec(output)?.[1]?.trim(); + return /^\s*Status:\s*Connected\b/imu.test(output) && activeGateway === gatewayName; +} + +function isExplicitPortableSandboxAbsence(result: RunResult, sandboxName: string): boolean { + if (result.status === 0) return false; + const clean = `${result.stdout}\n${result.stderr}`.replace(/\x1b\[[0-9;]*m|\r/gu, "").trim(); + const escapedName = sandboxName.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); + const namedSandbox = `(?:['\"]${escapedName}['\"]|${escapedName})`; + return ( + /^(?:error:\s*)?(?:×\s*)?code:\s*["']Some requested entity was not found["']\s*,\s*message:\s*["']sandbox not found["']$/iu.test( + clean, + ) || + new RegExp( + `^(?:error:\\s*)?sandbox\\s+${namedSandbox}\\s+(?:(?:is\\s+)?not\\s+(?:found|present)|does\\s+not\\s+exist)[.!]?$`, + "iu", + ).test(clean) || + new RegExp(`^(?:error:\\s*)?no\\s+such\\s+sandbox\\s+${namedSandbox}[.!]?$`, "iu").test(clean) + ); +} + +function deletePortableOpenShellSandbox( + runtime: UninstallRuntime, + sandboxName: string, + gatewayName: string, +): boolean { + const result = runtime.run("openshell", ["sandbox", "delete", "-g", gatewayName, sandboxName], { + env: runtime.env, + }); + if (result.status !== 0 && !isExplicitPortableSandboxAbsence(result, sandboxName)) { + runtime.warn(sandboxDeleteFailureMessage(sandboxName)); + return false; + } + if (result.status === 0) runtime.log(`Deleted OpenShell sandbox '${sandboxName}'`); + else { + runtime.warn(sandboxDeleteAbsentMessage(sandboxName)); + } + for (let attempt = 0; attempt < 5; attempt++) { + if (!portableGatewayIsReachable(runtime, gatewayName)) return false; + const verified = runtime.run("openshell", ["sandbox", "get", "-g", gatewayName, sandboxName], { + env: runtime.env, + }); + if (isExplicitPortableSandboxAbsence(verified, sandboxName)) return true; + if (attempt < 4) runtime.sleep(200); + } + runtime.warn(`OpenShell sandbox '${sandboxName}' did not reach verified absence.`); + return false; +} + const GATEWAY_ALREADY_ABSENT = /gateway[^\n]*(?:does not exist|not found)|No (?:active )?gateway|No gateway metadata found/i; const GATEWAY_REMOVE_UNSUPPORTED = @@ -1497,6 +1603,31 @@ function canRemoveScopedOpenShellResources( return false; } +function removePortableOpenShellResources( + runtime: UninstallRuntime, + sandboxNames: readonly string[], + gatewayName: string, +): boolean { + if (!runtime.commandExists("openshell")) { + runtime.error(OPENSHELL_COMMAND_MISSING_ERROR); + return false; + } + for (const sandboxName of sandboxNames) { + if (!portableGatewayIsReachable(runtime, gatewayName)) { + runtime.warn( + `Portable OpenShell cleanup requires connected gateway '${gatewayName}'; preserving its state for retry.`, + ); + return false; + } + if (!deletePortableOpenShellSandbox(runtime, sandboxName, gatewayName)) { + runtime.warn("Portable OpenShell cleanup was incomplete; preserving its state for retry."); + return false; + } + } + runtime.log("Kept shared OpenShell provider and gateway registrations for unrelated sandboxes."); + return true; +} + function removeAliases(paths: UninstallPaths, runtime: UninstallRuntime): void { for (const profile of paths.shellProfilePaths) { if (!runtime.existsSync(profile)) continue; @@ -1623,7 +1754,7 @@ function removeManagedDistributedVllmRuntime( }); } catch (error) { runtime.error( - `Could not inspect managed distributed vLLM rollback state: ${formatError(error)}`, + "Could not inspect managed distributed vLLM rollback state. NemoClaw refused uninstall before making changes.", ); return false; } @@ -1651,7 +1782,7 @@ function removeManagedDistributedVllmRuntime( const orphanBinding = bindingPaths.find((bindingPath) => !expectedBindingPaths.has(bindingPath)); if (orphanBinding) { runtime.error( - `Managed distributed vLLM SSH binding exists without its ownership receipt at ${orphanBinding}. NemoClaw refused uninstall before making changes. Recover or remove that state explicitly, then retry.`, + "A managed distributed vLLM SSH binding exists without its ownership receipt. NemoClaw refused uninstall before making changes. Recover or remove that state explicitly, then retry.", ); return false; } @@ -1759,7 +1890,7 @@ function managedLlamaCppCleanupTargets( return targets; } catch (error) { runtime.error( - `Managed llama.cpp cleanup could not safely inventory gateway-scoped ownership state: ${formatError(error)}. NemoClaw did not start the remaining uninstall steps.`, + "Managed llama.cpp cleanup could not safely inventory gateway-scoped ownership state. NemoClaw did not start the remaining uninstall steps.", ); return null; } @@ -2376,6 +2507,93 @@ function resolvePreserveSet( return PRESERVED_USER_DATA_ENTRIES; } +function executeOpenShellResourceCleanup( + paths: UninstallPaths, + options: UninstallRunOptions, + runtime: UninstallRuntime, + scopedToSelectedGateway: boolean, + sandboxNames: readonly string[], + teardownAuthority: GatewayOwner, + portableRuntimeCleanup: boolean, +): boolean { + const externallySupervised = isExternallySupervised(teardownAuthority); + const portableCleanupInput: PortableRuntimeCleanupInput = { + env: runtime.env, + gatewayName: options.gatewayName || resolveGatewayName(GATEWAY_PORT), + gatewayPort: GATEWAY_PORT, + homeDir: runtime.env.HOME || os.homedir(), + registryFile: path.join(paths.nemoclawStateDir, "sandboxes.json"), + stateDir: path.dirname(paths.managedSwapMarkerPath), + }; + if (portableRuntimeCleanup) { + try { + const cleanup = runtime.runPortableRuntimeCleanupTransaction( + portableCleanupInput, + (removed, receiptSandboxNames, receiptGatewayName) => { + runtime.log(`Removed ${String(removed)} receipt-owned portable sandbox container(s).`); + if (!removePortableOpenShellResources(runtime, receiptSandboxNames, receiptGatewayName)) { + return false; + } + return true; + }, + ); + if (cleanup === null) return false; + if (cleanup.registryRemoved) runtime.log("Removed the managed portable registry container."); + if (cleanup.selectorsRemoved.length > 0) { + runtime.log( + `Cleared NemoClaw portable selectors from the current-user systemd manager: ${cleanup.selectorsRemoved.join(", ")}.`, + ); + } + } catch (error) { + runtime.error(`Portable runtime cleanup failed: ${formatError(error)}`); + return false; + } + } else if ( + !removeOpenShellResources( + paths, + options, + runtime, + scopedToSelectedGateway, + sandboxNames, + teardownAuthority, + ) + ) { + return false; + } + if ( + scopedToSelectedGateway && + !finishScopedOpenShellCleanup(paths, options, runtime, sandboxNames, externallySupervised) + ) { + return false; + } + if (scopedToSelectedGateway && !options.keepOpenShell && !externallySupervised) { + if ( + !removeManagedDefaultGatewayUserService( + runtime, + options, + externallySupervised, + paths.selectedGatewayLocalStateDir, + true, + ) + ) { + return false; + } + stopHostGatewayProcessesForUninstall(runtime, { + gatewayBin: runtime.env.NEMOCLAW_OPENSHELL_GATEWAY_BIN, + logNoProcesses: true, + openShellGatewayName: options.gatewayName || resolveGatewayName(GATEWAY_PORT), + openShellGatewayPort: GATEWAY_PORT, + preserveRuntimeFilesOnNonMatching: true, + scopedGatewayStop: true, + stateDir: paths.selectedGatewayLocalStateDir, + usePgrepFallback: false, + }); + } else if (scopedToSelectedGateway && externallySupervised) { + runtime.log("Kept the externally supervised OpenShell gateway process running."); + } + return true; +} + function executePlan( plan: UninstallPlan, paths: UninstallPaths, @@ -2386,6 +2604,8 @@ function executePlan( sharedRegistryMustBePreserved: boolean, sandboxNames: readonly string[], teardownAuthority: GatewayOwner, + portableRuntimeCleanup: boolean, + portableRetirementEntries: ReturnType, ): { ok: boolean } { const externallySupervised = isExternallySupervised(teardownAuthority); if ( @@ -2401,10 +2621,48 @@ function executePlan( } let ok = true; const branding = runtimeBranding(runtime); + const preserveSharedOpenShell = + options.keepOpenShell || externallySupervised || portableRuntimeCleanup; + const portableStateEntries = portableRuntimeCleanup + ? [ + "portable-demo-lifecycle", + "sandboxes.json", + "sandboxes.json.lock", + "state", + ...PORTABLE_RETIREMENT_STATE_ENTRIES, + ...portableRetirementEntries.stateRoot, + ] + : []; + const serviceKeepMessage = portableRuntimeCleanup + ? "Keeping shared OpenShell gateway service, configuration, and processes for unrelated sandboxes." + : "Keeping OpenShell gateway service, configuration, and processes as requested."; + const sharedOpenShellReason = portableRuntimeCleanup + ? "portable" + : externallySupervised + ? "external" + : "requested"; + const binaryKeepMessage = { + external: "Keeping OpenShell binaries used by the externally supervised gateway.", + portable: "Keeping shared OpenShell binaries for unrelated sandboxes.", + requested: "Keeping OpenShell binaries as requested.", + }[sharedOpenShellReason]; + const configKeepMessage = { + external: "Keeping OpenShell gateway configuration used by the externally supervised gateway.", + portable: "Keeping shared OpenShell configuration for unrelated sandboxes.", + requested: "Keeping OpenShell gateway configuration as requested.", + }[sharedOpenShellReason]; for (const [index, step] of plan.steps.entries()) { runtime.log(`[${index + 1}/${plan.steps.length}] ${planStepDisplayName(step.name, branding)}`); + const portableStepMessage = PORTABLE_DEFERRED_STEP_MESSAGES[step.name]; + if (portableRuntimeCleanup && portableStepMessage) { + runtime.log(portableStepMessage); + continue; + } if (step.name === "Stopping services") { - if (!removeManagedModelRuntimes(paths, runtime, scopedToSelectedGateway)) { + if ( + !portableRuntimeCleanup && + !removeManagedModelRuntimes(paths, runtime, scopedToSelectedGateway) + ) { return { ok: false }; } // #8220: a gateway-scoped uninstall still needs the selected OpenShell @@ -2412,6 +2670,7 @@ function executePlan( // removes that unit after the sandbox delete succeeds. if ( !scopedToSelectedGateway && + !portableRuntimeCleanup && !removeManagedDefaultGatewayUserService(runtime, options, externallySupervised) ) { ok = false; @@ -2419,10 +2678,8 @@ function executePlan( if (!scopedToSelectedGateway) { stopHelperServices(paths, runtime); removeGlob(paths.helperServiceGlob, runtime); - if (options.keepOpenShell) { - runtime.log( - "Keeping OpenShell gateway service, configuration, and processes as requested.", - ); + if (options.keepOpenShell || portableRuntimeCleanup) { + runtime.log(serviceKeepMessage); } else { stopMatchingPids( `openshell.*forward.*${runtime.env.NEMOCLAW_DASHBOARD_PORT || "18789"}`, @@ -2473,48 +2730,18 @@ function executePlan( stopModelRouter(paths, runtime, !scopedToSelectedGateway); } else if (step.name === "OpenShell resources") { if ( - !removeOpenShellResources( + !executeOpenShellResourceCleanup( paths, options, runtime, scopedToSelectedGateway, sandboxNames, teardownAuthority, + false, ) ) { return { ok: false }; } - if ( - scopedToSelectedGateway && - !finishScopedOpenShellCleanup(paths, options, runtime, sandboxNames, externallySupervised) - ) { - return { ok: false }; - } - if (scopedToSelectedGateway && !options.keepOpenShell && !externallySupervised) { - if ( - !removeManagedDefaultGatewayUserService( - runtime, - options, - externallySupervised, - paths.selectedGatewayLocalStateDir, - true, - ) - ) { - return { ok: false }; - } - stopHostGatewayProcessesForUninstall(runtime, { - gatewayBin: runtime.env.NEMOCLAW_OPENSHELL_GATEWAY_BIN, - logNoProcesses: true, - openShellGatewayName: options.gatewayName || resolveGatewayName(GATEWAY_PORT), - openShellGatewayPort: GATEWAY_PORT, - preserveRuntimeFilesOnNonMatching: true, - scopedGatewayStop: true, - stateDir: paths.selectedGatewayLocalStateDir, - usePgrepFallback: false, - }); - } else if (scopedToSelectedGateway && externallySupervised) { - runtime.log("Kept the externally supervised OpenShell gateway process running."); - } } else if (step.name === "NemoClaw CLI") { if (scopedToSelectedGateway) { runtime.log("Sibling gateways remain; kept the shared NemoClaw CLI and shell shims."); @@ -2542,17 +2769,15 @@ function executePlan( if (action.kind === "delete-docker-volume") removeDockerVolume(action.name, runtime); } } else if (step.name === "Model stores") { - if (!removeHostModelStores(paths, options, runtime, scopedToSelectedGateway)) ok = false; + if (!removeHostModelStores(paths, options, runtime, scopedToSelectedGateway)) { + ok = false; + } } else if (step.name === "State and binaries") { removeManagedSwap(paths, runtime, scopedToSelectedGateway); if (!scopedToSelectedGateway) { for (const pattern of paths.runtimeTempGlobs) removeGlob(pattern, runtime); - if (options.keepOpenShell || externallySupervised) { - runtime.log( - externallySupervised - ? "Keeping OpenShell binaries used by the externally supervised gateway." - : "Keeping OpenShell binaries as requested.", - ); + if (preserveSharedOpenShell) { + runtime.log(binaryKeepMessage); } else if (GATEWAY_PORT !== DEFAULT_GATEWAY_PORT) { runtime.log("Keeping OpenShell binaries used by the default gateway service."); } else { @@ -2584,6 +2809,7 @@ function executePlan( paths.nemoclawStateDir, [ ...preserveUnderStateDir, + ...portableStateEntries, ...(selectedIsDefault ? [GATEWAYS_SUBDIR, path.basename(paths.managedSwapMarkerPath)] : []), @@ -2611,14 +2837,8 @@ function executePlan( } runtime.log("Sibling gateways remain; kept shared OpenShell and NemoClaw config."); } else { - if (!options.keepOpenShell && !externallySupervised) - removePath(paths.gatewayLocalStateDir, runtime); - if (options.keepOpenShell || externallySupervised) - runtime.log( - externallySupervised - ? "Keeping OpenShell gateway configuration used by the externally supervised gateway." - : "Keeping OpenShell gateway configuration as requested.", - ); + if (!preserveSharedOpenShell) removePath(paths.gatewayLocalStateDir, runtime); + if (preserveSharedOpenShell) runtime.log(configKeepMessage); else if (GATEWAY_PORT === DEFAULT_GATEWAY_PORT) { const envCleanup = removeNemoclawOpenShellGatewayEnv(paths, runtime); if (!envCleanup.ok) ok = false; @@ -2628,10 +2848,53 @@ function executePlan( removePath(paths.openshellConfigDir, runtime); } } else runtime.log("Keeping OpenShell configuration used by the default gateway service."); - removePath(paths.nemoclawConfigDir, runtime); + if (portableRuntimeCleanup) { + const portableConfigDir = path.join(paths.nemoclawConfigDir, "portable"); + const portableConfigEntries = ["containers.conf", ...portableRetirementEntries.config]; + if ( + portableConfigEntries.some((entry) => + runtime.existsSync(path.join(portableConfigDir, entry)), + ) && + !removePathExcept(portableConfigDir, portableConfigEntries, runtime) + ) + ok = false; + if (!removePathExcept(paths.nemoclawConfigDir, ["portable"], runtime)) ok = false; + } else { + removePath(paths.nemoclawConfigDir, runtime); + } } } } + return completePortablePlan( + ok, + portableRuntimeCleanup, + paths, + options, + runtime, + scopedToSelectedGateway, + sandboxNames, + teardownAuthority, + ); +} + +function completePortablePlan( + ok: boolean, + portable: boolean, + paths: UninstallPaths, + options: UninstallRunOptions, + runtime: UninstallRuntime, + scoped: boolean, + sandboxNames: readonly string[], + authority: GatewayOwner, +): { ok: boolean } { + if (!ok || !portable) return { ok }; + if ( + !executeOpenShellResourceCleanup(paths, options, runtime, scoped, sandboxNames, authority, true) + ) + return { ok: false }; + runtime.log( + "Kept ~/.nemoclaw/portable-uninstall-retirement.json until a later completed onboarding; it contains dictionary-testable pseudonymous fingerprints, not raw sandbox names or configuration, and another process running as this user can change it.", + ); return { ok }; } @@ -2640,6 +2903,7 @@ class IncompleteHostGatewayCleanupError extends Error {} function stopHostGatewayProcessesForUninstall( runtime: UninstallRuntime, options: StopHostGatewayOptions, + requireComplete = false, ): void { const result = stopHostGatewayProcesses( { @@ -2660,7 +2924,7 @@ function stopHostGatewayProcessesForUninstall( ); throw new IncompleteHostGatewayCleanupError(); } - if (!runtime.requireCompleteGatewayProcessCleanup) return; + if (!requireComplete && !runtime.requireCompleteGatewayProcessCleanup) return; if (result.failed.length === 0 && result.orphanScanComplete !== false) return; runtime.error("Cannot continue uninstall because host gateway process cleanup did not complete."); throw new IncompleteHostGatewayCleanupError(); @@ -2723,6 +2987,7 @@ export function runUninstallPlan( ); return { exitCode: 1, plan }; } + const externallySupervised = isExternallySupervised(teardownAuthority); let gatewayInspection = inspectOtherGatewayEnvironments(paths, runtime); let { otherGatewayEnvironmentsRemain: scopedToSelectedGateway } = gatewayInspection; let sandboxNames: string[] = []; @@ -2756,7 +3021,23 @@ export function runUninstallPlan( reportOtherGatewayEnvironments(boundaryInspection, runtime); } } - if (!runtime.commandExists("openshell")) { + let portableRuntimeCleanup = false; + let portableRetirementEntries = { config: [] as string[], stateRoot: [] as string[] }; + if (!scopedToSelectedGateway && !resolvedOptions.keepOpenShell && !externallySupervised) { + try { + portableRuntimeCleanup = runtime.hasPortableRuntimeCleanup( + path.dirname(paths.managedSwapMarkerPath), + ); + if (portableRuntimeCleanup) + portableRetirementEntries = portableRetirementPreservationEntries( + path.dirname(paths.managedSwapMarkerPath), + ); + } catch (error) { + runtime.error(`Portable lifecycle state is unsafe: ${formatError(error)}`); + return { exitCode: 1, plan }; + } + } + if (!portableRuntimeCleanup && !runtime.commandExists("openshell")) { runtime.error(OPENSHELL_COMMAND_MISSING_ERROR); return { exitCode: 1, plan }; } @@ -2773,6 +3054,8 @@ export function runUninstallPlan( gatewayInspection.sharedRegistryMustBePreserved, sandboxNames, teardownAuthority, + portableRuntimeCleanup, + portableRetirementEntries, )); } catch (error) { if (!(error instanceof IncompleteHostGatewayCleanupError)) throw error; @@ -2786,3 +3069,22 @@ export function runUninstallPlan( } return { exitCode: ok ? 0 : 1, otherGatewayEnvironmentsRemain: scopedToSelectedGateway, plan }; } + +/** Production entry: hold the host-wide portable authority fence through the sync plan. */ +export async function runUninstallPlanProduction( + options: UninstallRunOptions, + deps: UninstallRunDeps = {}, +): Promise { + const env = { ...process.env, ...(deps.env ?? {}) }; + const home = env.HOME || os.homedir(); + try { + return await (deps.withPortableHostFence ?? withPortableHostFence)(home, () => + runUninstallPlan(options, { ...deps, env }), + ); + } catch (error) { + (deps.error ?? ((message: string) => console.error(message)))( + `Uninstall could not acquire or release portable host authority: ${formatError(error)}`, + ); + return { exitCode: 1, plan: buildRunPlan(options, { ...deps, env }).plan }; + } +} diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index d266f5d3ab1..2976265a0ce 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -342,6 +342,7 @@ const { resolveSandboxImageTagFromCreateOutput } = require("./domain/sandbox/image-tag") as typeof import("./domain/sandbox/image-tag"); const nim: typeof import("./inference/nim") = require("./inference/nim"); const onboardSession: typeof import("./state/onboard-session") = require("./state/onboard-session"); +const portableRetirementAuthority: typeof import("./onboard/portable-retirement-authority") = require("./onboard/portable-retirement-authority"); const { registerIncompleteOnboardExitHandlerForSession, }: typeof import("./onboard/onboard-exit-handler") = require("./onboard/onboard-exit-handler"); @@ -3078,35 +3079,21 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { initialHint: opts.baseImageResolutionHint, initialPreResolvedMetadata: opts.preResolvedBaseImageMetadata, }); - const ownsOnboardLock = opts.onboardLockAlreadyHeld !== true; - const lockResult = ownsOnboardLock - ? onboardSession.acquireOnboardLock( - `nemoclaw onboard${resume ? " --resume" : ""}${fresh ? " --fresh" : ""}${isNonInteractive() ? " --non-interactive" : ""}${requestedFromDockerfile ? ` --from ${requestedFromDockerfile}` : ""}`, - ) - : { acquired: true as const }; - if (!lockResult.acquired) { - console.error(` Another ${cliDisplayName()} onboarding run is already in progress.`); - if (lockResult.holderPid) { - console.error(` Lock holder PID: ${lockResult.holderPid}`); - } - if (lockResult.holderStartedAt) { - console.error(` Started: ${lockResult.holderStartedAt}`); - } - console.error(" Wait for it to finish, or remove the stale lock if the previous run crashed:"); - console.error(` rm -f "${lockResult.lockFile}"`); - process.exit(1); - } - let lockReleased = false; - const releaseOnboardLock = () => { - if (lockReleased || !ownsOnboardLock) return; - lockReleased = true; - onboardSession.releaseOnboardLock(); - }; - if (ownsOnboardLock) process.once("exit", releaseOnboardLock); + const portableRetirementEntry = portableRetirementAuthority.beginPortableOnboardRetirementEntry({ + alreadyHeld: opts.onboardLockAlreadyHeld === true, + command: `nemoclaw onboard${resume ? " --resume" : ""}${fresh ? " --fresh" : ""}${isNonInteractive() ? " --non-interactive" : ""}${requestedFromDockerfile ? ` --from ${requestedFromDockerfile}` : ""}`, + displayName: cliDisplayName(), + homeDir: process.env.HOME || os.homedir(), + loadRegistry: registry.load, + registryFile: registry.REGISTRY_FILE, + sessionFile: onboardSession.SESSION_FILE, + withLifecycleLock: sandboxMutationLock.withMcpLifecycleLock, + }); let portableEnvScope: | import("./onboard/session-bootstrap").PortableOnboardEnvironmentScope | null = null; + const restorePortableEnvScope = () => portableEnvScope?.restore(); // Secure removal remains gated on successful migration of every staged legacy credential. let stagedLegacyKeys: string[] = []; @@ -3116,6 +3103,7 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { }; let completed = false, returnedNormally = false; try { + await portableRetirementEntry.run(async () => { const lockedRuntime = await resumeRuntime.prepare(opts, resume, isNonInteractive(), onboardSession.loadSession); portableEnvScope = lockedRuntime.environmentScope; entryDecisions.clearGatewayEnvironmentWithoutBinding(authoritativeGateway, process.env); @@ -3708,12 +3696,16 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { }, }); completed = finalFlowResult.session.machine.state === "complete"; + if (completed && finalFlowResult.session.sandboxName) { + await portableRetirementEntry.supersede(lockedRuntime.checkpointProfile); + } process.exitCode = completed ? 0 : 1; + }); } finally { try { await hermesApiPortReservationScope.release(); - portableEnvScope?.restore(); - releaseOnboardLock(); + restorePortableEnvScope(); + portableRetirementEntry.release(); onboardRuntimeBoundary.clear(); onboardTracing.finishOnboardTrace(onboardTrace, completed); GATEWAY_NAME = previousGatewayBinding.name; @@ -3723,7 +3715,7 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { else process.env.OPENSHELL_LOCAL_TLS_DIR = previousOpenshellLocalTlsDir; resetGatewayOwnerBinding(); } finally { - portableEnvScope?.restore(); + restorePortableEnvScope(); hostMountScope.restore(); } } diff --git a/src/lib/onboard/experimental/portable-demo-lifecycle.ts b/src/lib/onboard/experimental/portable-demo-lifecycle.ts index bdc2e4bd264..dd889cb63ce 100644 --- a/src/lib/onboard/experimental/portable-demo-lifecycle.ts +++ b/src/lib/onboard/experimental/portable-demo-lifecycle.ts @@ -5,6 +5,7 @@ import { spawn, spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { isDeepStrictEqual } from "node:util"; import type { ContainerEngineCommandCapture } from "../../adapters/container-engine"; import { openRegularFileNoFollow } from "../../adapters/fs/regular-file"; @@ -43,10 +44,12 @@ import { import { defaultPortableDemoStateDir, inspectPortableRuntimeReceiptReadiness, + portableDemoReceiptDirectory, portableDemoReceiptPath, } from "./portable-runtime-receipt-readiness"; const MAX_RECEIPT_BYTES = 4096; +const MAX_RECEIPT_DIRECTORY_ENTRIES = 1024; const COMMAND_TIMEOUT_MS = 30_000; const PROBE_TIMEOUT_MS = 5_000; const EXEC_READY_TIMEOUT_MS = 90_000; @@ -65,13 +68,15 @@ const STARTUP_PROCESS_PATTERN = "^(/usr/local/bin/nemoclaw-start|(bash|/bin/bash|/usr/bin/bash) /usr/local/bin/nemoclaw-start)( |$)"; const SLEEP_BUFFER = new Int32Array(new SharedArrayBuffer(4)); -type CommandResult = { +export type PortablePodmanLifecycleCommandResult = { status: number | null; stdout?: string | Buffer | null; stderr?: string | Buffer | null; error?: Error; }; +type CommandResult = PortablePodmanLifecycleCommandResult; + interface PortableDemoLifecycleReceipt { schemaVersion: 1 | 2 | 3 | 4; sandboxName: string; @@ -82,6 +87,28 @@ interface PortableDemoLifecycleReceipt { runtimeAuthority?: CheckpointPortableRuntimeAuthority; } +export interface PortableDemoLifecycleReceiptRecord { + readonly sandboxName: string; + readonly sandboxId: string; + readonly containerId: string; + readonly dashboardPort: number; + readonly registryGeneration: string; + readonly runtimeAuthority: CheckpointPortableRuntimeAuthority; +} + +export interface PortablePodmanLifecycleTransport { + readonly assertRuntimeAuthority: () => void; + readonly dockerHost: string; + readonly podman: (args: readonly string[]) => PortablePodmanLifecycleCommandResult; +} + +export interface PreparedPortableDemoSandboxRemoval { + readonly present: boolean; + readonly receipt: PortableDemoLifecycleReceiptRecord; + removeAndVerify(): void; + verifyAbsent(): void; +} + interface PodmanContainerInspection { containerId: string; sandboxId: string; @@ -337,6 +364,97 @@ function loadReceipt(sandboxName: string, stateDir: string): PortableDemoLifecyc } } +function currentReceipt(receipt: PortableDemoLifecycleReceipt): PortableDemoLifecycleReceiptRecord { + if ( + receipt.schemaVersion !== CURRENT_RECEIPT_SCHEMA_VERSION || + !receipt.registryGeneration || + !receipt.runtimeAuthority + ) { + throw new Error( + `Portable demo lifecycle receipt for sandbox '${receipt.sandboxName}' predates recorded runtime authority`, + ); + } + return { + sandboxName: receipt.sandboxName, + sandboxId: receipt.sandboxId, + containerId: receipt.containerId, + dashboardPort: receipt.dashboardPort, + registryGeneration: receipt.registryGeneration, + runtimeAuthority: receipt.runtimeAuthority, + }; +} + +/** Enumerate every strict current portable lifecycle receipt without following links. */ +export function listPortableDemoSandboxLifecycleReceipts( + stateDir = defaultStateDir(process.env), +): PortableDemoLifecycleReceiptRecord[] { + const directory = portableDemoReceiptDirectory(stateDir); + let opened: fs.Dir; + try { + const stat = fs.lstatSync(directory); + if (stat.isSymbolicLink() || !stat.isDirectory()) { + throw new Error( + `Portable demo lifecycle receipt path '${directory}' is not a real directory`, + ); + } + opened = fs.opendirSync(directory); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; + throw error; + } + + const receipts: PortableDemoLifecycleReceiptRecord[] = []; + const names = new Set(); + let inspected = 0; + try { + let entry: fs.Dirent | null; + while ((entry = opened.readSync()) !== null) { + inspected += 1; + if (inspected > MAX_RECEIPT_DIRECTORY_ENTRIES) { + throw new Error( + `Portable demo lifecycle receipt directory exceeds ${String(MAX_RECEIPT_DIRECTORY_ENTRIES)} entries`, + ); + } + if (entry.isSymbolicLink() || !entry.isFile() || !/^[a-f0-9]{64}\.json$/u.test(entry.name)) { + throw new Error( + `Portable demo lifecycle receipt directory contains an unsafe entry '${entry.name}'`, + ); + } + const filePath = path.join(directory, entry.name); + let file; + try { + file = openRegularFileNoFollow(filePath); + const raw: unknown = JSON.parse(file.readUtf8(MAX_RECEIPT_BYTES)); + const sandboxName = isRecord(raw) ? raw.sandboxName : null; + if (typeof sandboxName !== "string") { + throw new Error("Portable demo lifecycle receipt is malformed"); + } + const receipt = currentReceipt(parseReceipt(raw, sandboxName)); + if ( + receiptPath(receipt.sandboxName, stateDir) !== filePath || + names.has(receipt.sandboxName) + ) { + throw new Error( + `Portable demo lifecycle receipt identity is ambiguous for sandbox '${receipt.sandboxName}'`, + ); + } + names.add(receipt.sandboxName); + receipts.push(receipt); + } catch (error) { + if (error instanceof SyntaxError) { + throw new Error(`Portable demo lifecycle receipt '${filePath}' is malformed`); + } + throw error; + } finally { + file?.close(); + } + } + } finally { + opened.closeSync(); + } + return receipts.sort((left, right) => left.sandboxName.localeCompare(right.sandboxName)); +} + function removeReceipt(sandboxName: string, stateDir: string): void { try { fs.unlinkSync(receiptPath(sandboxName, stateDir)); @@ -473,15 +591,36 @@ function qualifiedPodmanAuthority( commandEnv: NodeJS.ProcessEnv, deps: PortableDemoLifecycleDeps, ) { - const readiness = inspectReceiptRuntimeReadiness(receipt, commandEnv, deps); - if (!readiness.ok) throw portablePodmanReadinessError(readiness); - if (!receipt.runtimeAuthority) { - throw new Error("Portable Podman readiness did not retain its recorded authority"); + if (receipt.schemaVersion !== CURRENT_RECEIPT_SCHEMA_VERSION || !receipt.runtimeAuthority) { + throw new Error( + "The lifecycle receipt predates recorded portable Podman authority; rerun onboarding.", + ); } - const podmanEnv = portablePodmanCommandEnvironment(receipt.runtimeAuthority, commandEnv); + return createPortablePodmanLifecycleTransport(receipt.runtimeAuthority, { + ...deps, + env: commandEnv, + }); +} + +/** Bind portable lifecycle commands to one revalidated current-user Podman socket. */ +export function createPortablePodmanLifecycleTransport( + runtimeAuthority: CheckpointPortableRuntimeAuthority, + deps: PortableDemoLifecycleDeps = {}, +): PortablePodmanLifecycleTransport { + const commandEnv = deps.env ?? process.env; + const podmanEnv = portablePodmanCommandEnvironment(runtimeAuthority, commandEnv); const capture = deps.podman ? podmanCapture(deps.podman, podmanEnv) : defaultPodmanCapture(podmanEnv); + const readiness = inspectPortablePodmanReadiness(runtimeAuthority, { + platform: deps.platform, + env: commandEnv, + socketAuthorityDeps: deps.podmanSocketAuthorityDeps, + hardenSocketDirectory: deps.hardenSocketDirectory ?? hardenPodmanSocketDirectory, + podmanCapture: capture, + ...deps.runtimeReadiness, + }); + if (!readiness.ok) throw portablePodmanReadinessError(readiness); (deps.log ?? console.log)( ` Portable Podman readiness: ${readiness.timing.mode}; activation ${String(readiness.timing.activationMs)} ms; API ${String(readiness.timing.apiMs)} ms; total ${String(readiness.timing.totalMs)} ms.`, ); @@ -495,12 +634,133 @@ function qualifiedPodmanAuthority( }); return { assertRuntimeAuthority: () => - assertPodmanSocketAuthority(socketAuthority, deps.podmanSocketAuthorityDeps), + (deps.runtimeReadiness?.assertSocketAuthority ?? assertPodmanSocketAuthority)( + socketAuthority, + deps.podmanSocketAuthorityDeps, + ), dockerHost: readiness.dockerHost, podman: (args: readonly string[]) => provider.capture(args, COMMAND_TIMEOUT_MS), }; } +function matchingPortableSandboxContainerIds( + sandboxName: string, + podman: PortablePodmanLifecycleTransport["podman"], +): string[] { + const result = podman([ + "ps", + "-a", + "--no-trunc", + "--filter", + `label=${PODMAN_MANAGED_LABEL}=true`, + "--filter", + `label=${PODMAN_SANDBOX_NAME_LABEL}=${sandboxName}`, + "--filter", + `label=${PODMAN_SANDBOX_WORKSPACE_LABEL}=${PODMAN_SANDBOX_WORKSPACE}`, + "--format", + "{{.ID}}", + ]); + requireCommand(result, `Finding portable sandbox '${sandboxName}'`); + const ids = String(result.stdout ?? "") + .split(/\r?\n/u) + .map((line) => line.trim()) + .filter(Boolean); + if (ids.some((id) => !CONTAINER_ID_PATTERN.test(id))) { + throw new Error(`Finding portable sandbox '${sandboxName}' returned an invalid container ID`); + } + return ids; +} + +/** Prevalidate one exact receipt-owned sandbox before a full uninstall mutates Podman. */ +export function preparePortableDemoSandboxRemoval( + receiptRecord: PortableDemoLifecycleReceiptRecord, + transport: PortablePodmanLifecycleTransport, + stateDir = defaultStateDir(process.env), +): PreparedPortableDemoSandboxRemoval { + const loaded = loadReceipt(receiptRecord.sandboxName, stateDir); + if (!loaded || !isDeepStrictEqual(currentReceipt(loaded), receiptRecord)) { + throw new Error( + `Portable demo lifecycle receipt changed for sandbox '${receiptRecord.sandboxName}'`, + ); + } + requireCurrentRegistryGeneration(loaded, receiptRecord.registryGeneration); + transport.assertRuntimeAuthority(); + const matches = matchingPortableSandboxContainerIds(receiptRecord.sandboxName, transport.podman); + if (matches.length > 1 || (matches.length === 1 && matches[0] !== receiptRecord.containerId)) { + throw new Error( + `Portable demo lifecycle found a replaced or ambiguous container for sandbox '${receiptRecord.sandboxName}'`, + ); + } + const initial = transport.podman(["inspect", receiptRecord.containerId]); + let present = true; + if (isMissingPodmanContainer(initial)) { + present = false; + if (matches.length !== 0) { + throw new Error( + `Portable demo lifecycle found a replacement container for sandbox '${receiptRecord.sandboxName}'`, + ); + } + } else { + const inspection = inspectPodmanContainer( + receiptRecord.containerId, + receiptRecord.sandboxName, + transport.podman, + initial, + ); + requireReceiptOwnedInspection(loaded, inspection); + if (matches.length !== 1) { + throw new Error( + `Portable demo lifecycle could not prove the label index for sandbox '${receiptRecord.sandboxName}'`, + ); + } + } + + const assertReceiptAndAuthority = (): void => { + const current = loadReceipt(receiptRecord.sandboxName, stateDir); + if (!current || !isDeepStrictEqual(currentReceipt(current), receiptRecord)) { + throw new Error( + `Portable demo lifecycle receipt changed for sandbox '${receiptRecord.sandboxName}'`, + ); + } + transport.assertRuntimeAuthority(); + }; + const verifyAbsent = (): void => { + assertReceiptAndAuthority(); + const inspected = transport.podman(["inspect", receiptRecord.containerId]); + if (!isMissingPodmanContainer(inspected)) { + if (inspected.status !== 0 || inspected.error) { + requireCommand( + inspected, + `Verifying portable sandbox '${receiptRecord.sandboxName}' removal`, + ); + } + throw new Error( + `Portable sandbox '${receiptRecord.sandboxName}' still has its recorded Podman container`, + ); + } + const remaining = matchingPortableSandboxContainerIds( + receiptRecord.sandboxName, + transport.podman, + ); + if (remaining.length !== 0) { + throw new Error( + `Portable demo lifecycle found a replacement container for sandbox '${receiptRecord.sandboxName}'`, + ); + } + transport.assertRuntimeAuthority(); + }; + return { + present, + receipt: receiptRecord, + removeAndVerify: () => { + assertReceiptAndAuthority(); + if (present) transport.podman(["rm", "--force", receiptRecord.containerId]); + verifyAbsent(); + }, + verifyAbsent, + }; +} + function requireReceiptOwnedInspection( receipt: PortableDemoLifecycleReceipt, inspection: PodmanContainerInspection, @@ -517,35 +777,6 @@ function requireReceiptOwnedInspection( } } -function inspectReceiptRuntimeReadiness( - receipt: PortableDemoLifecycleReceipt, - commandEnv: NodeJS.ProcessEnv, - deps: PortableDemoLifecycleDeps, -): PortablePodmanReadinessResult { - if (receipt.schemaVersion !== CURRENT_RECEIPT_SCHEMA_VERSION || !receipt.runtimeAuthority) { - return { - ok: false, - stage: "socket authority", - detail: - "The lifecycle receipt predates recorded portable Podman authority; rerun onboarding.", - recovery: "portable-onboarding", - timing: { mode: "warm", activationMs: 0, apiMs: 0, totalMs: 0 }, - }; - } - const podmanEnv = portablePodmanCommandEnvironment(receipt.runtimeAuthority, commandEnv); - const capture = deps.podman - ? podmanCapture(deps.podman, podmanEnv) - : defaultPodmanCapture(podmanEnv); - return inspectPortablePodmanReadiness(receipt.runtimeAuthority, { - platform: deps.platform, - env: commandEnv, - socketAuthorityDeps: deps.podmanSocketAuthorityDeps, - hardenSocketDirectory: deps.hardenSocketDirectory ?? hardenPodmanSocketDirectory, - podmanCapture: capture, - ...deps.runtimeReadiness, - }); -} - /** Inspect the receipt-owned portable runtime, or return null for an ordinary sandbox. */ export function inspectPortableDemoRuntimeReadiness( sandboxName: string, diff --git a/src/lib/onboard/experimental/portable-runtime-receipt-readiness.ts b/src/lib/onboard/experimental/portable-runtime-receipt-readiness.ts index 5aaf13f859e..51dd9fe96ed 100644 --- a/src/lib/onboard/experimental/portable-runtime-receipt-readiness.ts +++ b/src/lib/onboard/experimental/portable-runtime-receipt-readiness.ts @@ -50,6 +50,10 @@ export function portableDemoReceiptPath(sandboxName: string, stateDir: string): return path.join(stateDir, RECEIPT_DIRECTORY, fileName); } +export function portableDemoReceiptDirectory(stateDir: string): string { + return path.join(stateDir, RECEIPT_DIRECTORY); +} + export function defaultPortableDemoStateDir(env: NodeJS.ProcessEnv): string { if ( env.VITEST === "true" && diff --git a/src/lib/onboard/portable-resume-lock-boundary.test.ts b/src/lib/onboard/portable-resume-lock-boundary.test.ts index 0c22d2b2d17..7555e5dca87 100644 --- a/src/lib/onboard/portable-resume-lock-boundary.test.ts +++ b/src/lib/onboard/portable-resume-lock-boundary.test.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { spawn } from "node:child_process"; +import { spawn, spawnSync } from "node:child_process"; import { once } from "node:events"; import fs from "node:fs"; import os from "node:os"; @@ -17,6 +17,7 @@ let tempHome: string; let configWriteMarker: string; let socketActivationMarker: string; let preparationObservedLock = false; +let preparationObservedHostFence = false; let activeLockFile = ""; let boundaryModules: Awaited>; const preparePortableHost = vi.fn((): never => { @@ -41,11 +42,16 @@ beforeAll(async () => { }, 30_000); beforeEach(() => { + for (const directory of [".nemoclaw", ".nemoclaw/gateways/19093"]) { + fs.mkdirSync(path.join(tempHome, directory), { mode: 0o700, recursive: true }); + fs.chmodSync(path.join(tempHome, directory), 0o700); + } configWriteMarker = path.join(tempHome, "portable-config-written"); socketActivationMarker = path.join(tempHome, "podman-socket-activated"); fs.rmSync(configWriteMarker, { force: true }); fs.rmSync(socketActivationMarker, { force: true }); preparationObservedLock = false; + preparationObservedHostFence = false; preparePortableHost.mockClear(); process.env = { ...originalEnv, @@ -74,9 +80,83 @@ async function loadBoundaryModules() { }; const checkpointMigration = await import("../state/onboard-checkpoint-migrate"); const resumeIntent = await import("./resume/portable-resume-intent"); - return { command, onboardModule, session, checkpointMigration, resumeIntent }; + const retirement = await import("../state/portable-uninstall-retirement"); + return { + command, + onboardModule, + session, + checkpointMigration, + resumeIntent, + retirement, + }; +} +function replacementReentryState(profile: "default" | "portable", phase: string) { + const { checkpointMigration, retirement, session } = boundaryModules; + const stateDir = path.join(tempHome, ".nemoclaw"); + const sessionFile = session.SESSION_FILE; + const registryFile = path.join(stateDir, "sandboxes.json"); + const receiptDir = path.join(stateDir, "portable-demo-lifecycle"); + const receiptName = `${"a".repeat(64)}.json`; + const configDir = path.join(tempHome, ".config/nemoclaw/portable"); + const configFile = path.join(configDir, "containers.conf"); + fs.rmSync(stateDir, { recursive: true, force: true }); + fs.rmSync(configDir, { recursive: true, force: true }); + fs.mkdirSync(receiptDir, { mode: 0o700, recursive: true }); + fs.mkdirSync(configDir, { mode: 0o700, recursive: true }); + fs.writeFileSync(path.join(receiptDir, receiptName), "{}\n", { mode: 0o600 }); + fs.writeFileSync(registryFile, '{"sandboxes":{"alpha":{"name":"alpha"}}}\n', { + mode: 0o600, + }); + fs.writeFileSync(configFile, "[engine]\n", { mode: 0o600 }); + retirement.publishAndRetirePortableEvidence( + retirement.preparePortableRetirement(tempHome, [receiptName]), + ); + const stored = session.createSession({ + agent: "openclaw", + sandboxName: "later-sandbox", + sessionId: `replacement-${profile}-${phase}`, + }); + stored.status = phase === "pre-complete" ? "failed" : "in_progress"; + stored.resumable = true; + const uid = process.getuid?.() ?? 1001; + stored.checkpoint = checkpointMigration.deriveCheckpointFromSession(stored, { + profile, + runtimeAuthority: + profile === "portable" + ? { + schemaVersion: 1, + kind: "podman", + ownership: "current-user", + uid, + homeDir: tempHome, + configHome: path.join(tempHome, ".config"), + runtimeDir: `/run/user/${String(uid)}`, + socketPath: `/run/user/${String(uid)}/podman/podman.sock`, + } + : null, + }); + fs.mkdirSync(path.dirname(sessionFile), { mode: 0o700, recursive: true }); + fs.writeFileSync(sessionFile, `${JSON.stringify(stored)}\n`, { mode: 0o600 }); + profile === "portable" && + (() => { + fs.mkdirSync(configDir, { mode: 0o700, recursive: true }); + fs.writeFileSync(configFile, "[engine]\nreplacement=true\n", { mode: 0o600 }); + phase !== "config" && + (() => { + fs.mkdirSync(receiptDir, { mode: 0o700, recursive: true }); + fs.writeFileSync(path.join(receiptDir, receiptName), "{}\n", { mode: 0o600 }); + })(); + })(); + (phase === "registry" || phase === "pre-complete") && + fs.writeFileSync(registryFile, "{}\n", { mode: 0o600 }); + const record = path.join(stateDir, "portable-uninstall-retirement.json"); + return { + record, + recordBytes: fs.readFileSync(record), + sessionFile, + sessionBytes: fs.readFileSync(sessionFile), + }; } - function runWithObservedPreparation( onboardModule: { onboard(options?: import("./types").OnboardOptions): Promise }, options: import("./command").OnboardCommandOptions, @@ -85,6 +165,9 @@ function runWithObservedPreparation( ...options, preparePortableHost: () => { preparationObservedLock = fs.existsSync(activeLockFile); + preparationObservedHostFence = fs.existsSync( + boundaryModules.retirement.portableHostFencePath(tempHome), + ); return preparePortableHost(); }, }); @@ -220,14 +303,264 @@ describe("portable resume command lock boundary", () => { expect(JSON.parse(fs.readFileSync(session.SESSION_FILE, "utf8"))).toEqual( JSON.parse(resolvedRaw.at(-1)!), ); + expect(fs.readFileSync(session.SESSION_FILE, "utf8")).toBe(resolvedRaw.at(-1)); + expect(resolutions).toBe(2); expect(afterFailure.snapshot?.fingerprint).toBe(resolvedFingerprints.at(-1)); expect(failure).toMatchObject({ message: STOP_AFTER_PREPARATION }); - expect(resolutions).toBe(2); expect(preparePortableHost).toHaveBeenCalledTimes(1); expect(preparationObservedLock).toBe(true); + expect(preparationObservedHostFence).toBe(true); expect(fs.readFileSync(configWriteMarker, "utf8")).toBe("prepared"); expect(fs.readFileSync(socketActivationMarker, "utf8")).toBe("activated"); expect(fs.existsSync(session.LOCK_FILE)).toBe(false); }); + + it("holds one async-owner fence across await and exact nested reentry (#9189)", async () => { + const { portableHostFencePath, withPortableHostFence } = boundaryModules.retirement; + const exitListeners = process.listenerCount("exit"); + const events: string[] = []; + let releaseFirst!: () => void; + let firstStarted!: () => void; + const started = new Promise((resolve) => (firstStarted = resolve)); + const blocked = new Promise((resolve) => (releaseFirst = resolve)); + const first = withPortableHostFence(tempHome, async () => { + events.push("first:start"); + const outer = fs.lstatSync(portableHostFencePath(tempHome), { bigint: true }).ino; + firstStarted(); + await withPortableHostFence(tempHome, async () => { + await Promise.resolve(); + expect(fs.lstatSync(portableHostFencePath(tempHome), { bigint: true }).ino).toBe(outer); + events.push("nested"); + }); + await blocked; + events.push("first:end"); + }); + await started; + const second = withPortableHostFence(tempHome, () => events.push("second")); + await Promise.resolve(); + expect(events).toEqual(["first:start", "nested"]); + releaseFirst(); + await Promise.all([first, second]); + expect(events).toEqual(["first:start", "nested", "first:end", "second"]); + expect(fs.existsSync(portableHostFencePath(tempHome))).toBe(false); + expect(process.listenerCount("exit")).toBe(exitListeners); + }); + + it("drains an admitted detached reentry before releasing the physical fence (#9189)", async () => { + const { portableHostFencePath, withPortableHostFence } = boundaryModules.retirement; + const events: string[] = []; + let nestedStarted!: () => void; + let releaseNested!: () => void; + const started = new Promise((resolve) => (nestedStarted = resolve)); + const blocked = new Promise((resolve) => (releaseNested = resolve)); + let detached!: Promise; + const first = withPortableHostFence(tempHome, async () => { + const generation = fs.lstatSync(portableHostFencePath(tempHome), { bigint: true }).ino; + detached = withPortableHostFence(tempHome, async () => { + events.push("nested:start"); + nestedStarted(); + await blocked; + expect(fs.lstatSync(portableHostFencePath(tempHome), { bigint: true }).ino).toBe( + generation, + ); + events.push("nested:end"); + }); + await started; + events.push("outer:return"); + }).then(() => events.push("outer:resolved")); + await started; + const second = withPortableHostFence(tempHome, () => events.push("second")); + await Promise.resolve(); + expect(events).toEqual(["nested:start", "outer:return"]); + expect(fs.existsSync(portableHostFencePath(tempHome))).toBe(true); + releaseNested(); + await Promise.all([detached, first, second]); + expect(events).toEqual([ + "nested:start", + "outer:return", + "nested:end", + "outer:resolved", + "second", + ]); + expect(fs.existsSync(portableHostFencePath(tempHome))).toBe(false); + }); + + it("rejects a detached async owner and releases after nested throws (#9189)", async () => { + const { portableHostFencePath, withPortableHostFence } = boundaryModules.retirement; + const exitListeners = process.listenerCount("exit"); + let trigger!: () => void; + let detached!: Promise; + await expect( + withPortableHostFence(tempHome, async () => { + const gate = new Promise((resolve) => (trigger = resolve)); + detached = gate.then(() => withPortableHostFence(tempHome, () => undefined)); + await withPortableHostFence(tempHome, () => { + throw new Error("nested failure"); + }); + }), + ).rejects.toThrow("nested failure"); + trigger(); + await expect(detached).rejects.toThrow(/inactive/); + await expect(withPortableHostFence(tempHome, () => "released")).resolves.toBe("released"); + expect(fs.existsSync(portableHostFencePath(tempHome))).toBe(false); + expect(process.listenerCount("exit")).toBe(exitListeners); + }); + + it("balances the physical fence when the owning process exits (#9189)", () => { + const { portableHostFencePath } = boundaryModules.retirement; + const moduleUrl = new URL("../state/portable-uninstall-retirement.ts", import.meta.url).href; + const script = String.raw` + const retirement = (await import(process.argv[2])).default; + await retirement.withPortableHostFence(process.argv[1], async () => process.exit(0)); + `; + const result = spawnSync( + process.execPath, + [ + "--no-warnings", + "--import", + "tsx", + "--input-type=module", + "-e", + script, + tempHome, + moduleUrl, + ], + { encoding: "utf8" }, + ); + expect(result.status, result.stderr).toBe(0); + expect(fs.existsSync(portableHostFencePath(tempHome))).toBe(false); + }); + + it("keeps the host fence through the real onboarding preparation boundary (#9189)", async () => { + const { portableHostFencePath } = boundaryModules.retirement; + const { command, onboardModule } = boundaryModules; + fs.rmSync(path.join(tempHome, ".nemoclaw"), { recursive: true, force: true }); + await expect( + command.runOnboardCommand({ + flags: { + fresh: true, + "experimental-profile": "portable", + "yes-i-accept-third-party-software": true, + }, + env: process.env, + resolveResumeIntent: () => ({ effectiveResume: false, snapshot: null }), + runOnboard: (options) => runWithObservedPreparation(onboardModule, options), + }), + ).rejects.toThrow(STOP_AFTER_PREPARATION); + expect(preparationObservedLock).toBe(true); + expect(preparationObservedHostFence).toBe(true); + expect(fs.existsSync(portableHostFencePath(tempHome))).toBe(false); + }); + + it.each( + ( + [ + ["--resume", { resume: true }, true, "portable"], + [ + "--fresh default", + { fresh: true, "non-interactive": true, "yes-i-accept-third-party-software": true }, + false, + "default", + ], + [ + "--fresh portable", + { + fresh: true, + "experimental-profile": "portable", + "yes-i-accept-third-party-software": true, + }, + false, + "portable", + ], + ] as const + ).flatMap(([name, flags, resume, profile]) => + (["config", "receipt", "registry", "pre-complete"] as const).map( + (phase) => [`${name} after ${phase}`, flags, resume, profile, phase] as const, + ), + ), + )( + "admits %s replacement publication before session mutation (#9189)", + async (_case, flags, resume, profile, phase) => { + const { command, onboardModule, resumeIntent } = boundaryModules; + const state = replacementReentryState(profile, phase); + const expected = { + experimentalProfile: profile === "portable" ? "portable" : null, + fresh: !resume, + resume, + }; + const prepare = vi + .spyOn(onboardModule.onboardSession, "loadSession") + .mockImplementation(() => { + throw new Error(STOP_AFTER_PREPARATION); + }); + const exitListeners = process.listenerCount("exit"); + await expect( + command.runOnboardCommand({ + flags, + env: process.env, + loadPortableInferenceDescriptor: async () => null, + resolveResumeIntent: (intent) => + resumeIntent.resolveOnboardResumeIntent({ ...intent, sessionFile: state.sessionFile }), + runOnboard: (options) => { + expect(options).toMatchObject(expected); + return onboardModule.onboard(options); + }, + }), + ).rejects.toThrow(STOP_AFTER_PREPARATION); + expect(process.listenerCount("exit")).toBe(exitListeners); + expect(prepare).toHaveBeenCalledTimes(1); + expect(fs.readFileSync(state.record)).toEqual(state.recordBytes); + expect(fs.readFileSync(state.sessionFile)).toEqual(state.sessionBytes); + expect(preparePortableHost).not.toHaveBeenCalled(); + }, + ); + + it("reenters one host-fence generation from the real rebuild into real onboarding (#9189)", async () => { + fs.rmSync(path.join(tempHome, ".nemoclaw"), { recursive: true, force: true }); + const createdDirectories: string[] = []; + const originalMkdtemp = fs.mkdtempSync.bind(fs); + const mkdtemp = vi.spyOn(fs, "mkdtempSync").mockImplementation(((prefix: string) => { + const directory = originalMkdtemp(prefix); + createdDirectories.push(directory); + return directory; + }) as typeof fs.mkdtempSync); + const harnessModule = await import("../../../test/helpers/rebuild-flow-generic-harness"); + const { rebuildOnboardDependencies } = + await import("../../../test/helpers/rebuild-flow-harness"); + const actualOnboard = rebuildOnboardDependencies.onboard.bind(rebuildOnboardDependencies) as ( + options: import("./types").OnboardOptions, + ) => Promise; + const { retirement } = boundaryModules; + let innerObserved = false; + const harness = harnessModule.createRebuildFlowHarness({ + onboard: async (_session, options) => { + const lockPath = retirement.portableHostFencePath(tempHome); + const outerInode = fs.lstatSync(lockPath, { bigint: true }).ino; + const stateDir = path.join(tempHome, ".nemoclaw"); + fs.mkdirSync(stateDir, { mode: 0o700, recursive: true }); + fs.writeFileSync(path.join(stateDir, "portable-uninstall-retirement.json"), "{}", { + mode: 0o600, + }); + try { + await actualOnboard(options); + } catch (error) { + innerObserved = String(error).includes("retirement record"); + expect(fs.lstatSync(lockPath, { bigint: true }).ino).toBe(outerInode); + throw error; + } + }, + }); + + try { + await harness.rebuildSandbox("alpha", ["--yes"], { throwOnError: true }).catch(() => {}); + expect(innerObserved).toBe(true); + expect(fs.existsSync(retirement.portableHostFencePath(tempHome))).toBe(false); + } finally { + mkdtemp.mockRestore(); + for (const directory of createdDirectories.reverse()) { + fs.rmSync(directory, { recursive: true, force: true }); + } + } + }, 30_000); }); diff --git a/src/lib/onboard/portable-retirement-authority.ts b/src/lib/onboard/portable-retirement-authority.ts new file mode 100644 index 00000000000..bb4184c1b0e --- /dev/null +++ b/src/lib/onboard/portable-retirement-authority.ts @@ -0,0 +1,549 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { isDeepStrictEqual, TextDecoder } from "node:util"; + +import { acquireOnboardLock, normalizeSession, releaseOnboardLock } from "../state/onboard-session"; +import { + inspectPortableOnboardSupersession, + inspectPortableRetirementRecovery, + PORTABLE_RETIREMENT_STATE_ENTRIES, + provePortableOnboardAuthority, + readPortableAuthorityDirectory, + readPortableAuthoritySnapshot, + resumePortableEvidenceRetirement, + resumePortableOnboardReplacementEvidence, + samePortableAuthorityDirectory, + supersedePortableRetirementAfterOnboard, + withPortableHostFence, + type PortableRetirementRecovery, + type PortableOnboardAuthorityAdmission, +} from "../state/portable-uninstall-retirement"; +import { withRegistryLockAt } from "../state/registry/lock"; +import { listPortableDemoSandboxLifecycleReceipts } from "./experimental/portable-demo-lifecycle"; + +const UTF8 = new TextDecoder("utf-8", { fatal: true }); +const PORTABLE_ARTIFACT_NAME = /portable-uninstall/u; +const GENERATION = /^[A-Za-z0-9._:-]{1,256}$/u; + +export interface PortableOnboardRetirementBoundary { + readonly homeDir: string; + readonly registryFile: string; + readonly sessionFile: string; + readonly stateDir: string; +} +type Profile = "default" | "portable"; +interface Registry { + readonly defaultSandbox: string | null; + readonly sandboxes: Record< + string, + { + readonly agent?: string | null; + readonly dashboardPort?: number | null; + readonly gatewayName?: string | null; + readonly gatewayPort?: number | null; + readonly lifecycleGeneration?: string; + readonly name: string; + readonly openshellDriver?: string | null; + readonly pendingRouteReservation?: boolean; + } + >; +} +export interface PortableAuthorityAdmissionDeps { + readonly listReceipts?: typeof listPortableDemoSandboxLifecycleReceipts; + readonly loadRegistry: () => Registry; +} +export interface PortableRetirementAuthorityDeps extends PortableAuthorityAdmissionDeps { + readonly withLifecycleLock: ( + sandboxName: string, + operation: () => Promise | T, + ) => Promise; +} + +export interface PortableOnboardRetirementEntryOptions extends PortableRetirementAuthorityDeps { + readonly alreadyHeld: boolean; + readonly command: string; + readonly displayName: string; + readonly homeDir: string; + readonly registryFile: string; + readonly sessionFile: string; +} + +function strictJson(bytes: Buffer, description: string): Record { + try { + const value: unknown = JSON.parse(UTF8.decode(bytes)); + if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error(); + return value as Record; + } catch { + throw new Error(`${description} is malformed`); + } +} + +function completedSession(bytes: Buffer, expected?: Profile) { + const raw = strictJson(bytes, "Completed onboarding session"); + const session = normalizeSession(raw as never); + const checkpoint = session?.checkpoint; + const rawMachine = raw.machine; + const rawCheckpoint = raw.checkpoint; + const rawMetadata = raw.metadata; + if ( + raw.version !== 1 || + typeof raw.sessionId !== "string" || + raw.sessionId.length === 0 || + typeof raw.sandboxName !== "string" || + raw.sandboxName.length === 0 || + raw.status !== "complete" || + raw.resumable !== false || + !rawMachine || + typeof rawMachine !== "object" || + Array.isArray(rawMachine) || + (rawMachine as Record).version !== 1 || + (rawMachine as Record).state !== "complete" || + !Number.isSafeInteger((rawMachine as Record).revision) || + ((rawMachine as Record).revision as number) < 0 || + !rawCheckpoint || + typeof rawCheckpoint !== "object" || + Array.isArray(rawCheckpoint) || + (rawCheckpoint as Record).schemaVersion !== 4 || + (rawCheckpoint as Record).sessionId !== raw.sessionId || + (rawCheckpoint as Record).machineState !== "complete" || + !session || + session.sessionId !== raw.sessionId || + session.status !== "complete" || + session.resumable !== false || + session.machine.state !== "complete" || + session.machine.revision !== (rawMachine as Record).revision || + session.sandboxName !== raw.sandboxName || + session.agent === null || + !checkpoint || + checkpoint.machineState !== "complete" || + checkpoint.sessionId !== session.sessionId || + checkpoint.profile.kind !== "selected" || + (checkpoint.profile.value !== "default" && checkpoint.profile.value !== "portable") || + (expected !== undefined && checkpoint.profile.value !== expected) || + checkpoint.sandboxIdentity.kind !== "selected" || + checkpoint.sandboxIdentity.value.name !== session.sandboxName || + checkpoint.sandboxIdentity.value.agent !== session.agent || + checkpoint.gatewayAuthority.kind !== "selected" || + !rawMetadata || + typeof rawMetadata !== "object" || + Array.isArray(rawMetadata) || + typeof (rawMetadata as Record).gatewayName !== "string" || + session.metadata.gatewayName !== checkpoint.gatewayAuthority.value.gatewayName + ) + throw new Error("Completed onboarding session authority is incomplete"); + return session; +} + +function verifyAuthority( + boundary: PortableOnboardRetirementBoundary, + expected: Profile, + snapshots: ReadonlyMap, + deps: PortableAuthorityAdmissionDeps, + recovery: PortableRetirementRecovery | null, +): void { + const sessionBytes = snapshots.get(boundary.sessionFile); + const registryBytes = snapshots.get(boundary.registryFile); + if (!sessionBytes || !registryBytes) throw new Error("Completed onboarding files are missing"); + const session = completedSession(sessionBytes, expected); + const checkpoint = session.checkpoint!; + const sandboxName = session.sandboxName!; + const identity = + checkpoint.sandboxIdentity.kind === "selected" ? checkpoint.sandboxIdentity.value : null; + const gateway = + checkpoint.gatewayAuthority.kind === "selected" ? checkpoint.gatewayAuthority.value : null; + const rawRegistry = strictJson(registryBytes, "Completed onboarding registry"); + const rawSandboxes = rawRegistry.sandboxes; + const rawRow = + rawSandboxes && typeof rawSandboxes === "object" && !Array.isArray(rawSandboxes) + ? (rawSandboxes as Record)[sandboxName] + : null; + const rawEntry = + rawRow && typeof rawRow === "object" && !Array.isArray(rawRow) + ? (rawRow as Record) + : null; + if (!fs.readFileSync(boundary.registryFile).equals(registryBytes)) + throw new Error("Completed onboarding registry bytes changed before normalization"); + const registry = deps.loadRegistry(); + if (!fs.readFileSync(boundary.registryFile).equals(registryBytes)) + throw new Error("Completed onboarding registry changed while normalizing"); + const row = registry.sandboxes[sandboxName]; + const expectedAgent = identity?.agent === "openclaw" ? null : identity?.agent; + if ( + !identity || + !gateway || + !rawEntry || + rawRegistry.defaultSandbox !== sandboxName || + registry.defaultSandbox !== sandboxName || + row?.name !== sandboxName || + rawEntry.name !== sandboxName || + row.pendingRouteReservation === true || + row.agent !== expectedAgent || + rawEntry.agent !== expectedAgent || + row.openshellDriver !== "docker" || + rawEntry.openshellDriver !== "docker" || + row.gatewayName !== gateway.gatewayName || + rawEntry.gatewayName !== gateway.gatewayName || + row.gatewayPort !== gateway.gatewayPort || + rawEntry.gatewayPort !== gateway.gatewayPort || + typeof row.lifecycleGeneration !== "string" || + !GENERATION.test(row.lifecycleGeneration) || + rawEntry.lifecycleGeneration !== row.lifecycleGeneration || + rawEntry.dashboardPort !== row.dashboardPort || + (row.dashboardPort !== null && !Number.isInteger(row.dashboardPort)) + ) + throw new Error("Completed onboarding registry authority is incomplete"); + const portableStateRoot = path.join(boundary.homeDir, ".nemoclaw"); + const receiptDirectory = path.join(portableStateRoot, "portable-demo-lifecycle"); + const configDirectory = readPortableAuthorityDirectory( + path.join(boundary.homeDir, ".config/nemoclaw/portable"), + expected === "portable", + ); + const receiptDirectoryBefore = readPortableAuthorityDirectory( + receiptDirectory, + expected === "portable", + ); + const admittedArtifacts = (root: "config" | "receipt" | "registry") => + new Set( + recovery?.artifacts + .filter((artifact) => artifact.root === root) + .map((artifact) => artifact.basename) ?? [], + ); + const receiptArtifacts = admittedArtifacts("receipt"); + const configArtifacts = admittedArtifacts("config"); + if (receiptArtifacts.size || configArtifacts.size) + throw new Error("Completed onboarding still has staged portable uninstall evidence"); + const receipts = (deps.listReceipts ?? listPortableDemoSandboxLifecycleReceipts)( + portableStateRoot, + ); + const receiptDirectoryAfter = readPortableAuthorityDirectory( + receiptDirectory, + expected === "portable", + ); + if (!samePortableAuthorityDirectory(receiptDirectoryBefore, receiptDirectoryAfter)) + throw new Error("Portable lifecycle receipt directory changed while normalizing"); + const receiptEntries = receiptDirectoryBefore.entries.filter( + (entry) => !receiptArtifacts.has(entry), + ); + const configEntries = configDirectory.entries.filter((entry) => !configArtifacts.has(entry)); + if (expected === "default") { + if (checkpoint.runtimeAuthority.kind !== "unset" || receiptEntries.length || receipts.length) + throw new Error("Completed ordinary onboarding has portable receipt authority"); + if (configEntries.length) + throw new Error("Completed ordinary onboarding has portable configuration authority"); + } else { + const basename = `${createHash("sha256").update(sandboxName).digest("hex")}.json`; + const receipt = receipts.find((candidate) => candidate.sandboxName === sandboxName); + const receiptBytes = snapshots.get(path.join(receiptDirectory, basename)); + const rawReceipt = receiptBytes ? strictJson(receiptBytes, "Portable lifecycle receipt") : null; + if ( + checkpoint.runtimeAuthority.kind !== "selected" || + identity.agent !== "openclaw" || + checkpoint.runtimeAuthority.value.uid !== process.getuid?.() || + checkpoint.runtimeAuthority.value.homeDir !== boundary.homeDir || + !isDeepStrictEqual(configEntries, ["containers.conf"]) || + !readPortableAuthoritySnapshot( + path.join(boundary.homeDir, ".config/nemoclaw/portable/containers.conf"), + 64 * 1_024, + ) || + receiptEntries.length !== 1 || + receiptEntries[0] !== basename || + receipts.length !== 1 || + !receipt || + !rawReceipt || + Object.keys(rawReceipt).sort().join() !== + "containerId,dashboardPort,registryGeneration,runtimeAuthority,sandboxId,sandboxName,schemaVersion" || + rawReceipt.schemaVersion !== 4 || + rawReceipt.sandboxName !== receipt.sandboxName || + rawReceipt.sandboxId !== receipt.sandboxId || + rawReceipt.containerId !== receipt.containerId || + rawReceipt.dashboardPort !== receipt.dashboardPort || + rawReceipt.registryGeneration !== receipt.registryGeneration || + !isDeepStrictEqual(rawReceipt.runtimeAuthority, receipt.runtimeAuthority) || + !isDeepStrictEqual(receipt.runtimeAuthority, checkpoint.runtimeAuthority.value) || + receipt.registryGeneration !== row.lifecycleGeneration || + receipt.dashboardPort !== row.dashboardPort + ) + throw new Error("Completed portable onboarding authority is incomplete"); + } + rejectUnknownRetirementArtifacts(boundary.homeDir, recovery); +} + +function artifactDirectory(homeDir: string, root: "config" | "receipt" | "registry"): string { + return root === "config" + ? path.join(homeDir, ".config/nemoclaw/portable") + : root === "receipt" + ? path.join(homeDir, ".nemoclaw/portable-demo-lifecycle") + : path.join(homeDir, ".nemoclaw"); +} + +function rejectUnknownRetirementArtifacts( + homeDir: string, + recovery: PortableRetirementRecovery | null, + required = true, +): void { + const directories = new Map>([ + [path.join(homeDir, ".nemoclaw"), new Set(PORTABLE_RETIREMENT_STATE_ENTRIES)], + [path.join(homeDir, ".nemoclaw/portable-demo-lifecycle"), new Set()], + [path.join(homeDir, ".config/nemoclaw/portable"), new Set()], + ]); + for (const artifact of recovery?.artifacts ?? []) + directories.get(artifactDirectory(homeDir, artifact.root))!.add(artifact.basename); + for (const [directory, allowed] of directories) { + if ( + readPortableAuthorityDirectory( + directory, + required && directory === path.join(homeDir, ".nemoclaw"), + ).entries.some((name) => PORTABLE_ARTIFACT_NAME.test(name) && !allowed.has(name)) + ) + throw new Error("Onboarding state contains an unknown portable uninstall artifact"); + } +} + +function recordedSandboxNames(registryBytes: Buffer): string[] { + const registry = strictJson(registryBytes, "Recorded portable registry"); + const sandboxes = registry.sandboxes; + if (!sandboxes || typeof sandboxes !== "object" || Array.isArray(sandboxes)) + throw new Error("Recorded portable registry authority is incomplete"); + const names = Object.entries(sandboxes).map(([name, value]) => { + if ( + name.length < 1 || + name.length > 256 || + !value || + typeof value !== "object" || + Array.isArray(value) || + (value as Record).name !== name + ) + throw new Error("Recorded portable registry sandbox identity is invalid"); + return name; + }); + if (!names.length) throw new Error("Recorded portable registry has no sandbox authority"); + return names.sort(); +} + +async function resumeBeforeOnboard( + boundary: PortableOnboardRetirementBoundary, + recovery: PortableRetirementRecovery, + deps: PortableRetirementAuthorityDeps, + replacement = false, +): Promise { + if (!recovery.registryBytes) return; + const names = recordedSandboxNames(recovery.registryBytes); + const lock = (index: number): Promise => + index === names.length + ? Promise.resolve( + withRegistryLockAt(boundary.registryFile, () => { + const current = replacement + ? inspectPortableOnboardSupersession(boundary.homeDir) + : inspectPortableRetirementRecovery(boundary.homeDir); + rejectUnknownRetirementArtifacts(boundary.homeDir, current); + if (!current) throw new Error("Portable uninstall recovery authority disappeared"); + (replacement + ? resumePortableOnboardReplacementEvidence + : resumePortableEvidenceRetirement)(boundary.homeDir); + }), + ) + : deps.withLifecycleLock(names[index]!, () => lock(index + 1)); + await lock(0); +} + +function admission( + boundary: PortableOnboardRetirementBoundary, + expected: Profile, + deps: PortableAuthorityAdmissionDeps, + recovery: PortableRetirementRecovery | null = null, +): PortableOnboardAuthorityAdmission { + const sessionBytes = readPortableAuthoritySnapshot(boundary.sessionFile); + if (!sessionBytes) throw new Error("Completed onboarding session is missing"); + const session = completedSession(sessionBytes, expected); + const receipt = + expected === "portable" + ? path.join( + path.join(boundary.homeDir, ".nemoclaw"), + "portable-demo-lifecycle", + `${createHash("sha256").update(session.sandboxName!).digest("hex")}.json`, + ) + : null; + const files = [boundary.sessionFile, boundary.registryFile, ...(receipt ? [receipt] : [])]; + return { + files, + verify: (snapshots) => verifyAuthority(boundary, expected, snapshots, deps, recovery), + }; +} + +function lockedAuthority( + boundary: PortableOnboardRetirementBoundary, + expected: Profile, + deps: PortableRetirementAuthorityDeps, + operation: (authority: PortableOnboardAuthorityAdmission) => T, +): Promise { + const initial = readPortableAuthoritySnapshot(boundary.sessionFile); + if (!initial) throw new Error("Completed onboarding session is missing"); + const sandboxName = completedSession(initial, expected).sandboxName!; + return deps.withLifecycleLock(sandboxName, () => + withRegistryLockAt(boundary.registryFile, () => { + const current = readPortableAuthoritySnapshot(boundary.sessionFile); + if (!current || completedSession(current, expected).sandboxName !== sandboxName) + throw new Error("Completed onboarding sandbox authority changed before locking"); + return operation( + admission(boundary, expected, deps, inspectPortableOnboardSupersession(boundary.homeDir)), + ); + }), + ); +} + +async function recover( + boundary: PortableOnboardRetirementBoundary, + deps: PortableRetirementAuthorityDeps, +): Promise { + readPortableAuthorityDirectory(boundary.stateDir, false); + let recovery: PortableRetirementRecovery | null; + try { + recovery = inspectPortableRetirementRecovery(boundary.homeDir); + } catch (error) { + const replacement = inspectPortableOnboardSupersession(boundary.homeDir); + rejectUnknownRetirementArtifacts(boundary.homeDir, replacement, true); + const bytes = readPortableAuthoritySnapshot(boundary.sessionFile); + if (!bytes) throw error; + const raw = strictJson(bytes, "Onboarding session"); + if (raw.status !== "complete" || raw.resumable !== false) { + const session = normalizeSession(raw as never); + if ( + replacement?.fixedState !== "1000" || + !session || + session.resumable !== true || + session.status === "complete" + ) + throw error; + await resumeBeforeOnboard(boundary, replacement, deps, true); + const current = inspectPortableOnboardSupersession(boundary.homeDir); + rejectUnknownRetirementArtifacts(boundary.homeDir, current, true); + if (!current || current.artifacts.length || current.registryBytes) + throw new Error("Portable onboarding replacement recovery is incomplete"); + return; + } + if (replacement?.fixedState === "1000" && replacement.registryBytes) + await resumeBeforeOnboard(boundary, replacement, deps, true); + const session = completedSession(bytes); + await lockedAuthority(boundary, session.checkpoint!.profile.value, deps, (authority) => + supersedePortableRetirementAfterOnboard(boundary.homeDir, authority), + ); + return; + } + rejectUnknownRetirementArtifacts(boundary.homeDir, recovery, recovery !== null); + if (!recovery) return; + await resumeBeforeOnboard(boundary, recovery, deps); + const completed = inspectPortableRetirementRecovery(boundary.homeDir); + rejectUnknownRetirementArtifacts(boundary.homeDir, completed); +} + +export function hasPortableUninstallAuthority( + boundary: PortableOnboardRetirementBoundary, + deps: PortableAuthorityAdmissionDeps, +): boolean { + readPortableAuthorityDirectory(boundary.stateDir, true); + const receiptDirectory = readPortableAuthorityDirectory( + path.join(boundary.stateDir, "portable-demo-lifecycle"), + false, + ); + const configDirectory = readPortableAuthorityDirectory( + path.join(boundary.homeDir, ".config/nemoclaw/portable"), + false, + ); + const recovery = inspectPortableRetirementRecovery(boundary.homeDir); + rejectUnknownRetirementArtifacts(boundary.homeDir, recovery); + if (recovery) return true; + if (receiptDirectory.entries.length) { + const receipts = (deps.listReceipts ?? listPortableDemoSandboxLifecycleReceipts)( + boundary.stateDir, + ); + const receiptDirectoryAfter = readPortableAuthorityDirectory( + path.join(boundary.stateDir, "portable-demo-lifecycle"), + true, + ); + if ( + !samePortableAuthorityDirectory(receiptDirectory, receiptDirectoryAfter) || + receipts.length !== receiptDirectory.entries.length || + !isDeepStrictEqual(configDirectory.entries, ["containers.conf"]) || + !readPortableAuthoritySnapshot( + path.join(boundary.homeDir, ".config/nemoclaw/portable/containers.conf"), + 64 * 1_024, + ) + ) + throw new Error("Portable uninstall authority is incomplete"); + return true; + } + if (configDirectory.entries.length) + throw new Error("Portable uninstall configuration has no lifecycle authority"); + const sessionBytes = readPortableAuthoritySnapshot(boundary.sessionFile); + if (!sessionBytes) throw new Error("Completed onboarding session is missing"); + const session = completedSession(sessionBytes); + const profile = session.checkpoint!.profile.value; + provePortableOnboardAuthority(admission(boundary, profile, deps)); + return profile === "portable"; +} + +export async function withPortableOnboardRetirementBoundary( + boundary: PortableOnboardRetirementBoundary, + operation: () => Promise | T, + deps: PortableRetirementAuthorityDeps, +): Promise { + return await withPortableHostFence(boundary.homeDir, async () => { + await recover(boundary, deps); + return await operation(); + }); +} + +export async function supersedePortableRetirementAfterCompletedOnboard( + boundary: PortableOnboardRetirementBoundary, + expected: Profile, + deps: PortableRetirementAuthorityDeps, +): Promise { + const recovery = inspectPortableOnboardSupersession(boundary.homeDir); + if (recovery === null) return rejectUnknownRetirementArtifacts(boundary.homeDir, recovery); + await lockedAuthority(boundary, expected, deps, (authority) => + supersedePortableRetirementAfterOnboard(boundary.homeDir, authority), + ); +} + +export function beginPortableOnboardRetirementEntry( + options: PortableOnboardRetirementEntryOptions, +) { + const ownsOnboardLock = !options.alreadyHeld; + const lockResult = ownsOnboardLock + ? acquireOnboardLock(options.command) + : { acquired: true as const }; + if (!lockResult.acquired) { + console.error(` Another ${options.displayName} onboarding run is already in progress.`); + if (lockResult.holderPid) console.error(` Lock holder PID: ${lockResult.holderPid}`); + if (lockResult.holderStartedAt) console.error(` Started: ${lockResult.holderStartedAt}`); + console.error(" Wait for it to finish, or remove the stale lock if the previous run crashed:"); + console.error(` rm -f "${lockResult.lockFile}"`); + process.exit(1); + } + const boundary: PortableOnboardRetirementBoundary = { + homeDir: options.homeDir, + registryFile: options.registryFile, + sessionFile: options.sessionFile, + stateDir: path.dirname(options.sessionFile), + }; + const deps: PortableRetirementAuthorityDeps = options; + let released = false; + const release = () => { + if (released || !ownsOnboardLock) return; + released = true; + process.removeListener("exit", release); + releaseOnboardLock(); + }; + if (ownsOnboardLock) process.once("exit", release); + return { + release, + run: (operation: () => Promise | T) => + withPortableOnboardRetirementBoundary(boundary, operation, deps), + supersede: (expected: Profile) => + supersedePortableRetirementAfterCompletedOnboard(boundary, expected, deps), + }; +} diff --git a/src/lib/state/portable-uninstall-retirement.test.ts b/src/lib/state/portable-uninstall-retirement.test.ts new file mode 100644 index 00000000000..448d4338804 --- /dev/null +++ b/src/lib/state/portable-uninstall-retirement.test.ts @@ -0,0 +1,477 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawn } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, assert, describe, expect, it, vi } from "vitest"; + +import { + hasPortableRetirementRecord, + inspectPortableRetirementRecovery, + portableRetirementFingerprint, + preparePortableRetirement, + publishAndRetirePortableEvidence, + resumePortableEvidenceRetirement, + supersedePortableRetirementAfterOnboard, + type PortableOnboardAuthorityAdmission, +} from "./portable-uninstall-retirement"; + +const RECEIPT_BASENAME = `${"d".repeat(64)}.json`; +const temporaryDirectories: string[] = []; + +function fixture() { + const homeDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-portable-retirement-")); + temporaryDirectories.push(homeDir); + const stateDir = path.join(homeDir, ".nemoclaw"); + const receipt = path.join(stateDir, "portable-demo-lifecycle", RECEIPT_BASENAME); + const registryFile = path.join(stateDir, "sandboxes.json"); + const config = path.join(homeDir, ".config/nemoclaw/portable/containers.conf"); + fs.mkdirSync(path.dirname(receipt), { mode: 0o700, recursive: true }); + fs.mkdirSync(path.dirname(config), { mode: 0o700, recursive: true }); + fs.writeFileSync(receipt, "{}\n", { mode: 0o600 }); + fs.writeFileSync(registryFile, "{}\n", { mode: 0o600 }); + fs.writeFileSync(config, "[engine]\n", { mode: 0o600 }); + return { config, homeDir, receipt, registryFile, stateDir }; +} + +type Fixture = ReturnType; +type TargetRole = "config" | "receipt" | "registry"; + +function prepareFixture(test: Fixture) { + return preparePortableRetirement(test.homeDir, [RECEIPT_BASENAME]); +} + +function retirementRecordPath(test: Fixture): string { + return path.join(test.stateDir, "portable-uninstall-retirement.json"); +} + +function targetPath(test: Fixture, role: TargetRole): string { + return role === "config" ? test.config : role === "receipt" ? test.receipt : test.registryFile; +} + +function retiredFixture() { + const test = fixture(); + publishAndRetirePortableEvidence(prepareFixture(test)); + const files = [ + path.join(test.stateDir, "onboard-session.json"), + test.registryFile, + path.join(test.stateDir, "portable-demo-lifecycle", `${"e".repeat(64)}.json`), + ]; + fs.mkdirSync(path.dirname(files[2]!), { mode: 0o700, recursive: true }); + for (const target of files) fs.writeFileSync(target, "{}\n", { mode: 0o600 }); + const admission: PortableOnboardAuthorityAdmission = { files, verify: () => {} }; + return { admission, test }; +} + +function spawnForSignal( + args: readonly string[], +): Promise<{ signal: NodeJS.Signals | null; stderr: string }> { + return new Promise((resolve) => { + const child = spawn(process.execPath, args, { stdio: ["ignore", "ignore", "pipe"] }); + let stderr = ""; + child.stderr.setEncoding("utf8").on("data", (chunk) => (stderr += String(chunk))); + child.once("close", (_code, signal) => resolve({ signal, stderr })); + }); +} + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) + fs.rmSync(directory, { recursive: true, force: true }); + vi.restoreAllMocks(); +}); + +describe("portable uninstall retirement state", () => { + it("publishes the sole private retry record without raw cleanup authority (#9189)", () => { + const test = fixture(); + const prepared = prepareFixture(test); + const parsed = JSON.parse(prepared.recordBytes.toString("utf8")) as { targets: unknown[] }; + + expect( + portableRetirementFingerprint( + "a".repeat(64), + "receipt", + `${"b".repeat(64)}.json`, + Buffer.from('{"a":1}\n'), + ), + ).toBe("6316a6763402558ab9baccade86232099b32cdc73a52e6627b5e9098a4bf314e"); + expect( + portableRetirementFingerprint( + "a".repeat(64), + "receipt", + `${"b".repeat(64)}.json`, + Buffer.from("c"), + ), + ).not.toBe( + portableRetirementFingerprint( + "a".repeat(64), + "receipt", + `${"c".repeat(64)}.json`, + Buffer.from("bc"), + ), + ); + for (const [size, vector] of [ + [255, "09adecb5bfc2c496d9e1f4e737b4973699fa3f6f4a9027c0633bda893f7d9cfb"], + [256, "a62d83f4aa319e94abbccbb75d1dea277e450e5167be6872d87eb52d2e7a8b30"], + [65_535, "ab48f19398e2af46d86be80dea98e8326b67360e6cd8d3a171cc8c1a2b67a1b7"], + [65_536, "e868451822f03cecc74591d7785d6799f01d6742a83082d45d9f033c970afdbd"], + ] as const) { + expect( + portableRetirementFingerprint( + "a".repeat(64), + "config", + "containers.conf", + Buffer.alloc(size, 1), + ), + ).toBe(vector); + } + for (const input of [ + ["", "config", "containers.conf", Buffer.from("x")], + [new String("a".repeat(64)), "config", "containers.conf", Buffer.from("x")], + ["A".repeat(64), "config", "containers.conf", Buffer.from("x")], + ["a".repeat(64), "invalid", "containers.conf", Buffer.from("x")], + ["a".repeat(64), "config", "containers.conf\0", Buffer.from("x")], + ["a".repeat(64), "receipt", "../receipt.json", Buffer.from("x")], + ["a".repeat(64), "registry", "sandboxes.json", Buffer.alloc(0)], + ["a".repeat(64), "receipt", `${"b".repeat(64)}.json`, Buffer.alloc(4_097)], + ["a".repeat(64), "config", "containers.conf", Buffer.alloc(65_537)], + ["a".repeat(64), "registry", "sandboxes.json", Buffer.alloc(1_048_577)], + ] as const) { + expect(() => + portableRetirementFingerprint( + ...(input as unknown as Parameters), + ), + ).toThrow(/invalid/); + } + expect(JSON.stringify(parsed)).not.toContain("alpha"); + expect(JSON.stringify(parsed)).not.toContain(test.homeDir); + expect(JSON.stringify(parsed)).not.toContain("helper_binaries_dir"); + expect(parsed.targets.every((target) => Array.isArray(target) && target.length === 11)).toBe( + true, + ); + const configTarget = parsed.targets[0] as string[]; + const configStat = fs.statSync(test.config, { bigint: true }); + expect(configTarget.slice(2, 10)).toEqual( + [ + configStat.dev, + configStat.ino, + configStat.size, + configStat.uid, + configStat.mode, + configStat.nlink, + configStat.mtimeNs, + configStat.ctimeNs, + ].map(String), + ); + + publishAndRetirePortableEvidence(prepared); + const record = retirementRecordPath(test); + expect(fs.statSync(test.stateDir).mode & 0o777).toBe(0o700); + expect(fs.statSync(record).mode & 0o777).toBe(0o600); + expect( + [test.config, test.receipt, test.registryFile].every((target) => !fs.existsSync(target)), + ).toBe(true); + expect(inspectPortableRetirementRecovery(test.homeDir)).toEqual({ + artifacts: [], + fixedState: "1000", + registryBytes: null, + }); + expect(() => resumePortableEvidenceRetirement(test.homeDir)).not.toThrow(); + expect(hasPortableRetirementRecord(test.homeDir)).toBe(true); + }); + + it("rejects non-UTF-8 retirement and authority JSON before cleanup (#9189)", () => { + const malformedRecord = fixture(); + fs.writeFileSync(retirementRecordPath(malformedRecord), Buffer.from([0xff]), { mode: 0o600 }); + expect(() => hasPortableRetirementRecord(malformedRecord.homeDir)).toThrow(/malformed/); + + const malformedAuthority = fixture(); + fs.writeFileSync(malformedAuthority.registryFile, Buffer.from([0xff]), { mode: 0o600 }); + expect(() => prepareFixture(malformedAuthority)).toThrow(); + }); + + it.each([ + ["config", 65_536], + ["receipt", 4_096], + ["registry", 1_048_576], + ] as const)( + "enforces the exact %s target bound before record publication (#9189)", + (role, limit) => { + const exact = fixture(); + const exactTarget = targetPath(exact, role); + fs.writeFileSync( + exactTarget, + Buffer.concat([Buffer.from("{}"), Buffer.alloc(limit - 2, 0x20)]), + { mode: 0o600 }, + ); + expect(() => prepareFixture(exact)).not.toThrow(); + + const oversized = fixture(); + const oversizedTarget = targetPath(oversized, role); + fs.writeFileSync(oversizedTarget, Buffer.alloc(limit + 1, 0x20), { mode: 0o600 }); + expect(() => prepareFixture(oversized)).toThrow(/Unsafe/); + expect(fs.existsSync(retirementRecordPath(oversized))).toBe(false); + }, + ); + + it.each([ + ["config", 65_536], + ["receipt", 4_096], + ["registry", 1_048_576], + ] as const)("enforces the exact %s target bound during recovery (#9189)", (role, limit) => { + const test = fixture(); + const rename = fs.renameSync.bind(fs); + vi.spyOn(fs, "renameSync").mockImplementation((source, destination) => { + String(source) === test.config && assert.fail("pause before target retirement"); + rename(source, destination); + }); + expect(() => publishAndRetirePortableEvidence(prepareFixture(test))).toThrow(/pause/); + vi.restoreAllMocks(); + const target = targetPath(test, role); + fs.writeFileSync(target, Buffer.alloc(limit + 1, 0x20), { mode: 0o600 }); + expect(() => inspectPortableRetirementRecovery(test.homeDir)).toThrow(/Unsafe/); + expect(hasPortableRetirementRecord(test.homeDir)).toBe(true); + }); + + it("rejects same-byte canonical and staged metadata drift (#9189)", () => { + const canonical = fixture(); + const preparedCanonical = prepareFixture(canonical); + const canonicalTime = fs.statSync(canonical.receipt).mtimeMs + 1_000; + fs.utimesSync(canonical.receipt, new Date(canonicalTime), new Date(canonicalTime)); + expect(() => publishAndRetirePortableEvidence(preparedCanonical)).toThrow(/changed/); + + const staged = fixture(); + const unlink = fs.unlinkSync.bind(fs); + vi.spyOn(fs, "unlinkSync").mockImplementation((target) => { + String(target).includes(`.${RECEIPT_BASENAME}.portable-uninstall-`) && + assert.fail("injected staged receipt crash"); + unlink(target); + }); + expect(() => publishAndRetirePortableEvidence(prepareFixture(staged))).toThrow(/injected/); + vi.restoreAllMocks(); + const artifact = inspectPortableRetirementRecovery(staged.homeDir)!.artifacts[0]!; + expect(artifact).toMatchObject({ root: "receipt" }); + const stagedReceipt = path.join(path.dirname(staged.receipt), artifact.basename); + const stagedTime = fs.statSync(stagedReceipt).mtimeMs + 1_000; + fs.utimesSync(stagedReceipt, new Date(stagedTime), new Date(stagedTime)); + expect(() => resumePortableEvidenceRetirement(staged.homeDir)).toThrow(/changed/); + }); + + it("rejects noncanonical record fields and incomplete supersession before retirement (#9189)", () => { + const invalidIdentity = fixture(); + const invalidPrepared = prepareFixture(invalidIdentity); + const invalidRecord = JSON.parse(invalidPrepared.recordBytes.toString("utf8")) as { + targets: string[][]; + }; + invalidRecord.targets[0]![2] = "01"; + fs.writeFileSync(retirementRecordPath(invalidIdentity), `${JSON.stringify(invalidRecord)}\n`, { + mode: 0o600, + }); + expect(() => hasPortableRetirementRecord(invalidIdentity.homeDir)).toThrow(/values/); + + for (const extra of [false, true]) { + const shape = fixture(); + const shaped = JSON.parse(prepareFixture(shape).recordBytes.toString("utf8")) as { + targets: string[][]; + }; + extra ? shaped.targets[0]!.push("0") : shaped.targets[0]!.pop(); + fs.writeFileSync(retirementRecordPath(shape), `${JSON.stringify(shaped)}\n`, { mode: 0o600 }); + expect(() => hasPortableRetirementRecord(shape.homeDir)).toThrow(/invalid/); + } + + const missingConfig = fixture(); + const missingPrepared = prepareFixture(missingConfig); + const missingRecord = JSON.parse(missingPrepared.recordBytes.toString("utf8")) as { + targets: unknown[]; + }; + missingRecord.targets.shift(); + fs.writeFileSync(retirementRecordPath(missingConfig), `${JSON.stringify(missingRecord)}\n`, { + mode: 0o600, + }); + expect(() => hasPortableRetirementRecord(missingConfig.homeDir)).toThrow(/order/); + + const incomplete = fixture(); + const prepared = prepareFixture(incomplete); + fs.writeFileSync( + path.join(incomplete.stateDir, ".portable-uninstall-retirement.superseded"), + prepared.recordBytes, + { mode: 0o600 }, + ); + expect(() => publishAndRetirePortableEvidence(prepared)).toThrow(/incomplete/); + expect(fs.existsSync(incomplete.receipt)).toBe(true); + expect(fs.existsSync(incomplete.registryFile)).toBe(true); + }); + + it.each([ + ["T", [".portable-uninstall-retirement.tmp"]], + ["TC", [".portable-uninstall-retirement.tmp.cleanup"]], + ["R", ["portable-uninstall-retirement.json"]], + ["S", [".portable-uninstall-retirement.superseded"]], + ["SC", [".portable-uninstall-retirement.superseded.cleanup"]], + ["R+T", ["portable-uninstall-retirement.json", ".portable-uninstall-retirement.tmp"]], + ["R+TC", ["portable-uninstall-retirement.json", ".portable-uninstall-retirement.tmp.cleanup"]], + ["R+S", ["portable-uninstall-retirement.json", ".portable-uninstall-retirement.superseded"]], + [ + "RC+S", + [ + ".portable-uninstall-retirement.canonical.cleanup", + ".portable-uninstall-retirement.superseded", + ], + ], + ])("rejects an unexpected hard link in fixed state %s (#9189)", (_state, names) => { + const test = fixture(); + const prepared = prepareFixture(test); + const first = path.join(test.stateDir, names[0]!); + fs.writeFileSync(first, prepared.recordBytes, { mode: 0o600 }); + for (const name of names.slice(1)) fs.linkSync(first, path.join(test.stateDir, name)); + fs.linkSync(first, path.join(test.stateDir, "unexpected-record-link")); + expect(() => hasPortableRetirementRecord(test.homeDir)).toThrow(); + }); + + it.each(["config", "receipt", "registry"])( + "rejects an unexpected hard link to the %s target (#9189)", + (role) => { + const test = fixture(); + fs.linkSync(targetPath(test, role as TargetRole), path.join(test.homeDir, `${role}.link`)); + expect(() => prepareFixture(test)).toThrow(/Unsafe/); + }, + ); + + it("rejects a new link that prevents the published-record survivor decrement (#9189)", () => { + const test = fixture(); + const prepared = prepareFixture(test); + const temporary = path.join(test.stateDir, ".portable-uninstall-retirement.tmp"); + const record = retirementRecordPath(test); + fs.writeFileSync(temporary, prepared.recordBytes, { mode: 0o600 }); + fs.linkSync(temporary, record); + const unlink = fs.unlinkSync.bind(fs); + vi.spyOn(fs, "unlinkSync").mockImplementation((target) => { + String(target).endsWith(".tmp.cleanup") && + fs.linkSync(record, path.join(test.stateDir, "unexpected-survivor-link")); + unlink(target); + }); + expect(() => hasPortableRetirementRecord(test.homeDir)).toThrow(/survivor|Unsafe/); + }); + + it("rejects a new link that prevents the superseded survivor decrement (#9189)", () => { + const scope = retiredFixture(); + const superseded = path.join(scope.test.stateDir, ".portable-uninstall-retirement.superseded"); + const unlink = fs.unlinkSync.bind(fs); + vi.spyOn(fs, "unlinkSync").mockImplementation((target) => { + String(target).endsWith(".canonical.cleanup") && + fs.linkSync(superseded, path.join(scope.test.stateDir, "unexpected-survivor-link")); + unlink(target); + }); + expect(() => + supersedePortableRetirementAfterOnboard(scope.test.homeDir, scope.admission), + ).toThrow(/survivor|Unsafe/); + }); + + it("recovers after SIGKILL at every record and evidence durability boundary (#9189)", async () => { + const moduleUrl = new URL("./portable-uninstall-retirement.ts", import.meta.url).href; + const childScript = String.raw` + import fs from "node:fs"; + const [home, receipt, operation, boundary, moduleUrl] = process.argv.slice(1); + const retirement = (await import(moduleUrl)).default; + const prepared = retirement.preparePortableRetirement(home, [receipt]); + const original = fs[operation].bind(fs); + let calls = 0; + fs[operation] = (...args) => { + const result = original(...args); + if (++calls === Number(boundary)) process.kill(process.pid, "SIGKILL"); + return result; + }; + retirement.publishAndRetirePortableEvidence(prepared); + `; + const boundaries = { fsyncSync: 10, linkSync: 1, renameSync: 4, unlinkSync: 4 } as const; + const cases = Object.entries(boundaries).flatMap(([operation, count]) => + Array.from({ length: count }, (_value, index) => [operation, index + 1] as const), + ); + for (let offset = 0; offset < cases.length; offset += 6) { + await Promise.all( + cases.slice(offset, offset + 6).map(async ([operation, boundary]) => { + const test = fixture(); + const result = await spawnForSignal([ + "--no-warnings", + "--import", + "tsx", + "--input-type=module", + "-e", + childScript, + test.homeDir, + RECEIPT_BASENAME, + operation, + String(boundary), + moduleUrl, + ]); + expect(result.signal, `${operation}:${String(boundary)} ${result.stderr}`).toBe( + "SIGKILL", + ); + const targets = [test.receipt, test.registryFile, test.config]; + const assertRecovered = () => { + resumePortableEvidenceRetirement(test.homeDir); + expect(targets.every((target) => !fs.existsSync(target))).toBe(true); + }; + const assertPrior = () => expect(targets.every(fs.existsSync)).toBe(true); + (hasPortableRetirementRecord(test.homeDir) ? assertRecovered : assertPrior)(); + }), + ); + } + }, 30_000); + + it("recovers completed onboarding after SIGKILL at every supersession boundary (#9189)", async () => { + const moduleUrl = new URL("./portable-uninstall-retirement.ts", import.meta.url).href; + const childScript = String.raw` + import fs from "node:fs"; + const [home, filesJson, operation, boundary, moduleUrl] = process.argv.slice(1); + const retirement = (await import(moduleUrl)).default; + if (operation !== "afterReturn") { + const original = fs[operation].bind(fs); + let calls = 0; + fs[operation] = (...args) => { + const result = original(...args); + if (++calls === Number(boundary)) process.kill(process.pid, "SIGKILL"); + return result; + }; + } + retirement.supersedePortableRetirementAfterOnboard(home, { + files: JSON.parse(filesJson), + verify: () => {}, + }); + process.kill(process.pid, "SIGKILL"); + `; + const boundaries = { afterReturn: 1, fsyncSync: 12, linkSync: 1, renameSync: 2, unlinkSync: 2 }; + const cases = Object.entries(boundaries).flatMap(([operation, count]) => + Array.from({ length: count }, (_value, index) => [operation, index + 1] as const), + ); + for (let offset = 0; offset < cases.length; offset += 6) { + await Promise.all( + cases.slice(offset, offset + 6).map(async ([operation, boundary]) => { + const scope = retiredFixture(); + const result = await spawnForSignal([ + "--no-warnings", + "--import", + "tsx", + "--input-type=module", + "-e", + childScript, + scope.test.homeDir, + JSON.stringify(scope.admission.files), + operation, + String(boundary), + moduleUrl, + ]); + expect(result.signal, `${operation}:${String(boundary)} ${result.stderr}`).toBe( + "SIGKILL", + ); + expect(() => + supersedePortableRetirementAfterOnboard(scope.test.homeDir, scope.admission), + ).not.toThrow(); + expect(hasPortableRetirementRecord(scope.test.homeDir)).toBe(false); + }), + ); + } + }, 30_000); +}); diff --git a/src/lib/state/portable-uninstall-retirement.ts b/src/lib/state/portable-uninstall-retirement.ts new file mode 100644 index 00000000000..11d46d4029c --- /dev/null +++ b/src/lib/state/portable-uninstall-retirement.ts @@ -0,0 +1,849 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { AsyncLocalStorage } from "node:async_hooks"; +import { createHash, randomBytes } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { TextDecoder } from "node:util"; +import { isErrnoException } from "../core/errno"; +import { + acquireProcessBoundLockAt, + releaseProcessBoundLock, + type ProcessBoundLockHandle, +} from "./registry/lock"; + +const NAMES = { + R: "portable-uninstall-retirement.json", + T: ".portable-uninstall-retirement.tmp", + TC: ".portable-uninstall-retirement.tmp.cleanup", + S: ".portable-uninstall-retirement.superseded", + RC: ".portable-uninstall-retirement.canonical.cleanup", + SC: ".portable-uninstall-retirement.superseded.cleanup", +} as const; +const KIND = "portable-uninstall-retirement"; +const MAX_RECORD = 64 * 1024; +const MAX_AUTHORITY = 16 * 1024 * 1024; +const MAX_DIRECTORY_ENTRIES = 1024; +const HEX = /^[a-f0-9]{64}$/u; +const RECEIPT = /^[a-f0-9]{64}\.json$/u; +const DECIMAL = /^(0|[1-9][0-9]{0,19})$/u; +const MAX_UINT64 = (1n << 64n) - 1n; +const UTF8 = new TextDecoder("utf-8", { fatal: true }); + +export const PORTABLE_RETIREMENT_STATE_ENTRIES = Object.values(NAMES); +type Role = "config" | "receipt" | "registry"; +type Target = readonly [ + Role, + string, + string, + string, + string, + string, + string, + string, + string, + string, + string, +]; +interface RecordState { + readonly kind: typeof KIND; + readonly schemaVersion: 1; + readonly targets: readonly Target[]; + readonly transactionId: string; +} +interface ExactFile { + readonly bytes: Buffer; + readonly ctimeNs: bigint; + readonly dev: bigint; + readonly ino: bigint; + readonly mode: bigint; + readonly mtimeNs: bigint; + readonly nlink: bigint; + readonly size: bigint; + readonly uid: bigint; +} +export interface PreparedPortableRetirement { + readonly homeDir: string; + readonly record: RecordState; + readonly recordBytes: Buffer; +} +export interface PortableRetirementRecovery { + readonly artifacts: readonly { + readonly basename: string; + readonly root: Role; + }[]; + readonly fixedState: string; + readonly registryBytes: Buffer | null; +} +export interface PortableOnboardAuthorityAdmission { + readonly files: readonly string[]; + readonly verify: (snapshots: ReadonlyMap) => void; +} +export interface PortableAuthorityDirectorySnapshot { + readonly entries: readonly string[]; + readonly identity: fs.BigIntStats | null; +} +interface FenceOwner { + active: boolean; + readonly handle: ProcessBoundLockHandle; + readonly path: string; + references: number; + released: boolean; + readonly drained: Promise; + readonly resolveDrained: () => void; +} +const owners = new AsyncLocalStorage(); +const tails = new Map>(); + +export const portableHostFencePath = (homeDir: string): string => + path.join(homeDir, ".nemoclaw-portable-host.lock"); + +function releaseFenceReference(owner: FenceOwner): void { + owner.references -= 1; + if (owner.references === 0) owner.resolveDrained(); +} + +export async function withPortableHostFence( + homeDir: string, + operation: () => Promise | T, +): Promise { + const lockPath = portableHostFencePath(homeDir); + const inherited = owners.getStore(); + if (inherited?.path === lockPath) { + if (!inherited.active) throw new Error("Portable host fence owner is inactive"); + inherited.references += 1; + try { + return await operation(); + } finally { + releaseFenceReference(inherited); + } + } + const previous = tails.get(lockPath) ?? Promise.resolve(); + let finish!: () => void; + const turn = new Promise((resolve) => (finish = resolve)); + const tail = previous.then(() => turn); + tails.set(lockPath, tail); + await previous; + let owner: FenceOwner | null = null; + let onExit: (() => void) | null = null; + try { + let resolveDrained!: () => void; + const drained = new Promise((resolve) => (resolveDrained = resolve)); + owner = { + active: true, + handle: acquireProcessBoundLockAt(lockPath), + path: lockPath, + references: 1, + released: false, + drained, + resolveDrained, + }; + onExit = () => { + if (!owner?.active) return; + owner.active = false; + owner.released = true; + try { + releaseProcessBoundLock(owner.handle); + } catch { + // Exit cannot recover; generation-safe release preserves ambiguity. + } + }; + process.once("exit", onExit); + return await owners.run(owner, operation); + } finally { + try { + if (owner?.active) { + owner.active = false; + } + if (owner) { + releaseFenceReference(owner); + await owner.drained; + } + if (onExit) process.removeListener("exit", onExit); + if (owner && !owner.released) { + owner.released = true; + releaseProcessBoundLock(owner.handle); + } + } finally { + finish(); + if (tails.get(lockPath) === tail) tails.delete(lockPath); + } + } +} + +const root = (homeDir: string): string => path.join(homeDir, ".nemoclaw"); +const paths = (homeDir: string) => + Object.fromEntries( + Object.entries(NAMES).map(([key, name]) => [key, path.join(root(homeDir), name)]), + ) as { + [Key in keyof typeof NAMES]: string; + }; +function fsyncDirectory(directory: string): void { + const descriptor = fs.openSync(directory, fs.constants.O_RDONLY); + try { + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } +} +function privateRoot(homeDir: string): string { + const stateRoot = root(homeDir); + fs.mkdirSync(stateRoot, { mode: 0o700, recursive: true }); + const stat = fs.lstatSync(stateRoot, { bigint: true }); + const uid = process.getuid?.(); + if (uid === undefined || !stat.isDirectory() || stat.isSymbolicLink() || stat.uid !== BigInt(uid)) + throw new Error("Portable uninstall requires a current-user state directory"); + if ((stat.mode & 0o777n) !== 0o700n) fs.chmodSync(stateRoot, 0o700); + return stateRoot; +} +function sameStat(left: fs.BigIntStats, right: fs.BigIntStats): boolean { + return ["dev", "ino", "size", "mode", "uid", "nlink", "mtimeNs", "ctimeNs"].every( + (key) => left[key as keyof fs.BigIntStats] === right[key as keyof fs.BigIntStats], + ); +} +export function readPortableAuthorityDirectory( + directory: string, + required: boolean, +): PortableAuthorityDirectorySnapshot { + let descriptor: number | null = null; + try { + descriptor = fs.openSync( + directory, + fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_DIRECTORY, + ); + const before = fs.fstatSync(descriptor, { bigint: true }); + const named = fs.lstatSync(directory, { bigint: true }); + const uid = process.getuid?.(); + if ( + uid === undefined || + !before.isDirectory() || + named.isSymbolicLink() || + !sameStat(before, named) || + before.uid !== BigInt(uid) || + (before.mode & 0o777n) !== 0o700n || + before.nlink < 1n + ) + throw new Error(`Unsafe portable authority directory: ${directory}`); + const entries = fs.readdirSync(directory).sort(); + if (entries.length > MAX_DIRECTORY_ENTRIES) + throw new Error(`Portable authority directory has too many entries: ${directory}`); + if ( + !sameStat(before, fs.fstatSync(descriptor, { bigint: true })) || + !sameStat(before, fs.lstatSync(directory, { bigint: true })) + ) + throw new Error(`Portable authority directory changed while reading: ${directory}`); + return { entries, identity: before }; + } catch (error) { + if (isErrnoException(error) && error.code === "ENOENT" && !required) + return { entries: [], identity: null }; + throw error; + } finally { + if (descriptor !== null) fs.closeSync(descriptor); + } +} +export function samePortableAuthorityDirectory( + left: PortableAuthorityDirectorySnapshot, + right: PortableAuthorityDirectorySnapshot, +): boolean { + return ( + (left.identity === null) === (right.identity === null) && + left.entries.length === right.entries.length && + left.entries.every((entry, index) => entry === right.entries[index]) && + (left.identity === null || (right.identity !== null && sameStat(left.identity, right.identity))) + ); +} +function readFile(target: string, limit: number, expectedLinks = 1n): ExactFile | null { + let descriptor: number | null = null; + try { + descriptor = fs.openSync(target, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + const uid = process.getuid?.(); + if (uid === undefined) throw new Error("Current-user ownership is unavailable"); + const before = fs.fstatSync(descriptor, { bigint: true }); + const named = fs.lstatSync(target, { bigint: true }); + if ( + !before.isFile() || + named.isSymbolicLink() || + !sameStat(before, named) || + before.nlink !== expectedLinks || + before.uid !== BigInt(uid) || + (before.mode & 0o777n) !== 0o600n || + before.size < 1n || + before.size > BigInt(limit) + ) + throw new Error(`Unsafe portable uninstall file: ${target}`); + const bytes = Buffer.alloc(Number(before.size)); + if ( + fs.readSync(descriptor, bytes, 0, bytes.length, 0) !== bytes.length || + !sameStat(before, fs.fstatSync(descriptor, { bigint: true })) || + !sameStat(before, fs.lstatSync(target, { bigint: true })) + ) + throw new Error(`Portable uninstall file changed while reading: ${target}`); + return { + bytes, + ctimeNs: before.ctimeNs, + dev: before.dev, + ino: before.ino, + mode: before.mode, + mtimeNs: before.mtimeNs, + nlink: before.nlink, + size: before.size, + uid: before.uid, + }; + } catch (error) { + if (isErrnoException(error) && error.code === "ENOENT") return null; + throw error; + } finally { + if (descriptor !== null) fs.closeSync(descriptor); + } +} +function roleLimit(role: Role): number { + return role === "receipt" ? 4_096 : role === "config" ? 64 * 1_024 : 1_024 * 1_024; +} +function decimal(value: string): bigint { + if (!DECIMAL.test(value)) throw new Error("Portable uninstall decimal value is invalid"); + const parsed = BigInt(value); + if (parsed > MAX_UINT64) throw new Error("Portable uninstall decimal value is too large"); + return parsed; +} +function frame(value: Buffer | string): Buffer { + const bytes = Buffer.isBuffer(value) ? value : Buffer.from(value); + const length = Buffer.alloc(8); + length.writeBigUInt64BE(BigInt(bytes.length)); + return Buffer.concat([length, bytes]); +} +export function portableRetirementFingerprint( + transactionId: string, + role: Role, + basename: string, + bytes: Buffer, +): string { + const expectedBasename = + typeof basename === "string" && role === "config" + ? basename === "containers.conf" + : role === "registry" + ? basename === "sandboxes.json" + : role === "receipt" + ? RECEIPT.test(basename) + : false; + if ( + typeof transactionId !== "string" || + typeof role !== "string" || + !HEX.test(transactionId) || + expectedBasename !== true || + !Buffer.isBuffer(bytes) || + bytes.length < 1 || + bytes.length > roleLimit(role) + ) + throw new Error("Portable uninstall fingerprint input is invalid"); + return createHash("sha256") + .update( + Buffer.concat([ + frame("NEMOCLAW-PORTABLE-UNINSTALL-TARGET-V1"), + frame(transactionId), + frame(role), + frame(basename), + frame(bytes), + ]), + ) + .digest("hex"); +} +function canonical(homeDir: string, target: Target): string { + if (target[0] === "config") + return path.join(homeDir, ".config/nemoclaw/portable/containers.conf"); + if (target[0] === "registry") return path.join(root(homeDir), "sandboxes.json"); + return path.join(root(homeDir), "portable-demo-lifecycle", target[1]); +} +const cleanup = (homeDir: string, record: RecordState, target: Target): string => { + const source = canonical(homeDir, target); + return path.join( + path.dirname(source), + `.${target[1]}.portable-uninstall-${record.transactionId}.cleanup`, + ); +}; +function matches(record: RecordState, target: Target, file: ExactFile, staged = false): boolean { + return ( + file.dev === decimal(target[2]) && + file.ino === decimal(target[3]) && + file.size === decimal(target[4]) && + file.uid === decimal(target[5]) && + file.mode === decimal(target[6]) && + file.nlink === decimal(target[7]) && + file.mtimeNs === decimal(target[8]) && + (file.ctimeNs === decimal(target[9]) || (staged && file.ctimeNs >= decimal(target[9]))) && + portableRetirementFingerprint(record.transactionId, target[0], target[1], file.bytes) === + target[10] + ); +} + +function parseRecord(bytes: Buffer): RecordState { + let value: unknown; + try { + value = JSON.parse(UTF8.decode(bytes)); + } catch { + throw new Error("Portable uninstall retirement record is malformed"); + } + if (!value || typeof value !== "object" || Array.isArray(value)) + throw new Error("Portable uninstall retirement record is invalid"); + const record = value as Record; + if ( + Object.keys(record).sort().join() !== "kind,schemaVersion,targets,transactionId" || + record.kind !== KIND || + record.schemaVersion !== 1 || + typeof record.transactionId !== "string" || + !HEX.test(record.transactionId) || + !Array.isArray(record.targets) || + record.targets.length < 2 || + record.targets.length > 258 + ) + throw new Error("Portable uninstall retirement record values are invalid"); + const targets = record.targets.map((item): Target => { + if ( + !Array.isArray(item) || + item.length !== 11 || + !["config", "receipt", "registry"].includes(item[0]) || + item.slice(1).some((part) => typeof part !== "string") + ) + throw new Error("Portable uninstall retirement target is invalid"); + const target = item as unknown as Target; + if ( + (target[0] === "config" && target[1] !== "containers.conf") || + (target[0] === "registry" && target[1] !== "sandboxes.json") || + (target[0] === "receipt" && !RECEIPT.test(target[1])) || + target.slice(2, 10).some((part) => !DECIMAL.test(part) || decimal(part) > MAX_UINT64) || + decimal(target[4]) < 1n || + decimal(target[4]) > BigInt(roleLimit(target[0])) || + (decimal(target[6]) & 0o777n) !== 0o600n || + decimal(target[7]) !== 1n || + !HEX.test(target[10]) + ) + throw new Error("Portable uninstall retirement target values are invalid"); + return target; + }); + const receipts = targets.filter((target) => target[0] === "receipt"); + if ( + targets.at(-1)?.[0] !== "registry" || + targets.filter((target) => target[0] === "registry").length !== 1 || + targets.filter((target) => target[0] === "config").length !== 1 || + targets.some((target, index) => target[0] === "config" && index !== 0) || + receipts.length < 1 || + receipts.some((target, index) => index > 0 && receipts[index - 1]![1] >= target[1]) + ) + throw new Error("Portable uninstall retirement target order is invalid"); + return { kind: KIND, schemaVersion: 1, targets, transactionId: record.transactionId }; +} +function readRecord(target: string, links = 1n): { file: ExactFile; record: RecordState } | null { + const file = readFile(target, MAX_RECORD, links); + return file ? { file, record: parseRecord(file.bytes) } : null; +} +function snapshot(homeDir: string, id: string, role: Role, basename: string): Target | null { + const shell = [role, basename] as unknown as Target; + const file = readFile(canonical(homeDir, shell), roleLimit(role)); + if (file && role !== "config") JSON.parse(UTF8.decode(file.bytes)); + return file + ? [ + role, + basename, + String(file.dev), + String(file.ino), + String(file.size), + String(file.uid), + String(file.mode), + String(file.nlink), + String(file.mtimeNs), + String(file.ctimeNs), + portableRetirementFingerprint(id, role, basename, file.bytes), + ] + : null; +} + +export function preparePortableRetirement( + homeDir: string, + receiptBasenames: readonly string[], +): PreparedPortableRetirement { + privateRoot(homeDir); + if ( + receiptBasenames.length < 1 || + receiptBasenames.length > 256 || + new Set(receiptBasenames).size !== receiptBasenames.length || + receiptBasenames.some((name) => !RECEIPT.test(name)) + ) + throw new Error("Portable uninstall receipt identities are invalid"); + const transactionId = randomBytes(32).toString("hex"); + const targets = [ + snapshot(homeDir, transactionId, "config", "containers.conf"), + ...[...receiptBasenames] + .sort() + .map((name) => snapshot(homeDir, transactionId, "receipt", name)), + snapshot(homeDir, transactionId, "registry", "sandboxes.json"), + ].filter((target): target is Target => target !== null); + if ( + targets.filter((target) => target[0] === "config").length !== 1 || + targets.filter((target) => target[0] === "receipt").length !== receiptBasenames.length || + targets.at(-1)?.[0] !== "registry" + ) + throw new Error("Portable uninstall authority is incomplete"); + const record: RecordState = { kind: KIND, schemaVersion: 1, targets, transactionId }; + const recordBytes = Buffer.from(`${JSON.stringify(record)}\n`); + if (recordBytes.length > MAX_RECORD) throw new Error("Portable uninstall record is too large"); + return { homeDir, record, recordBytes }; +} + +function targetState( + homeDir: string, + record: RecordState, + target: Target, + allowCanonicalReplacement = false, +): "canonical" | "cleanup" | "retired" { + const limit = roleLimit(target[0]); + const source = readFile(canonical(homeDir, target), limit); + const staged = readFile(cleanup(homeDir, record, target), limit); + if (source && staged) { + if ( + allowCanonicalReplacement && + !matches(record, target, source) && + matches(record, target, staged, true) + ) + return "cleanup"; + throw new Error("Portable uninstall target has two generations"); + } + if (source && !matches(record, target, source)) { + if (allowCanonicalReplacement && !staged) return "retired"; + throw new Error("Portable uninstall target changed"); + } + if (staged && !matches(record, target, staged, true)) + throw new Error("Portable uninstall target changed"); + return source ? "canonical" : staged ? "cleanup" : "retired"; +} +function targetStates(homeDir: string, record: RecordState, allowCanonicalReplacement = false) { + const states = record.targets.map((target) => + targetState(homeDir, record, target, allowCanonicalReplacement), + ); + const firstPending = states.findIndex((state) => state !== "retired"); + if (firstPending >= 0 && states.slice(firstPending).includes("retired")) + throw new Error("Portable uninstall targets retired out of order"); + return states; +} +function allTargetsExact(homeDir: string, record: RecordState): boolean { + return record.targets.every((target) => targetState(homeDir, record, target) === "canonical"); +} +function sameExact(left: ExactFile | null, right: ExactFile): boolean { + return Boolean( + left && + left.dev === right.dev && + left.ino === right.ino && + left.nlink === right.nlink && + left.bytes.equals(right.bytes), + ); +} +function sameTarget(left: ExactFile | null, right: ExactFile, renamed = false): boolean { + return Boolean( + left && + left.bytes.equals(right.bytes) && + left.dev === right.dev && + left.ino === right.ino && + left.size === right.size && + left.uid === right.uid && + left.mode === right.mode && + left.nlink === right.nlink && + left.mtimeNs === right.mtimeNs && + (left.ctimeNs === right.ctimeNs || (renamed && left.ctimeNs >= right.ctimeNs)), + ); +} +function entryExists(target: string): boolean { + try { + fs.lstatSync(target); + return true; + } catch (error) { + if (isErrnoException(error) && error.code === "ENOENT") return false; + throw error; + } +} +function detachDelete( + source: string | null, + staged: string, + expected: ExactFile, + limit: number, + survivor?: { readonly expected: ExactFile; readonly path: string }, + targetMetadata = false, +): void { + const sourceFile = source ? readFile(source, limit, expected.nlink) : null; + let stagedFile = readFile(staged, limit, expected.nlink); + if (sourceFile && stagedFile) throw new Error("Portable uninstall cleanup has two generations"); + if (sourceFile) { + if (!(targetMetadata ? sameTarget(sourceFile, expected) : sameExact(sourceFile, expected))) + throw new Error(`Portable uninstall source changed: ${source}`); + fs.renameSync(source!, staged); + fsyncDirectory(path.dirname(source!)); + stagedFile = readFile(staged, limit, expected.nlink); + if (targetMetadata && !sameTarget(stagedFile, expected, true)) + throw new Error(`Portable uninstall cleanup changed: ${staged}`); + } + if ( + !(targetMetadata + ? sameTarget(stagedFile, sourceFile ? stagedFile! : expected) + : sameExact(stagedFile, expected)) + ) + throw new Error(`Portable uninstall cleanup changed: ${staged}`); + fs.unlinkSync(staged); + fsyncDirectory(path.dirname(staged)); + if (survivor) { + const remaining = readFile(survivor.path, limit, 1n); + if (!sameExact(remaining, { ...survivor.expected, nlink: 1n })) + throw new Error(`Portable uninstall survivor changed: ${survivor.path}`); + } +} +function recoverTemp(homeDir: string): ExactFile | null { + const state = paths(homeDir); + const pendingPaths = [state.T, state.TC].filter(entryExists); + if (pendingPaths.length > 1) throw new Error("Portable uninstall temporary state is ambiguous"); + const recordPresent = entryExists(state.R); + if (!pendingPaths.length) return null; + const expectedLinks = recordPresent ? 2n : 1n; + const pending = readRecord(pendingPaths[0]!, expectedLinks)!; + const published = recordPresent ? readRecord(state.R, 2n)! : null; + if ( + published ? !sameExact(pending.file, published.file) : !allTargetsExact(homeDir, pending.record) + ) + throw new Error("Portable uninstall temporary record has no exact authority"); + detachDelete( + state.T, + state.TC, + pending.file, + MAX_RECORD, + published ? { expected: published.file, path: state.R } : undefined, + ); + return published ? readRecord(state.R, 1n)!.file : null; +} +function publish(prepared: PreparedPortableRetirement): void { + const state = paths(prepared.homeDir); + if ([state.R, state.RC, state.S, state.SC].some(entryExists)) + throw new Error("Portable uninstall retirement already exists or is incomplete"); + recoverTemp(prepared.homeDir); + fs.writeFileSync(state.T, prepared.recordBytes, { flag: "wx", mode: 0o600 }); + const descriptor = fs.openSync(state.T, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + try { + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } + const temp = readRecord(state.T); + if (!temp || !temp.file.bytes.equals(prepared.recordBytes)) + throw new Error("Portable uninstall temporary record verification failed"); + fs.linkSync(state.T, state.R); + fsyncDirectory(path.dirname(state.R)); + const linkedTemp = readRecord(state.T, 2n); + const record = readRecord(state.R, 2n); + if ( + !linkedTemp || + !record || + !sameExact(record.file, linkedTemp.file) || + !record.file.bytes.equals(prepared.recordBytes) + ) + throw new Error("Portable uninstall record publication failed"); + recoverTemp(prepared.homeDir); +} +function retireTargets( + homeDir: string, + record: RecordState, + allowCanonicalReplacement = false, +): void { + const states = targetStates(homeDir, record, allowCanonicalReplacement); + const firstPending = states.findIndex((state) => state !== "retired"); + for (let index = Math.max(firstPending, 0); index < record.targets.length; index++) { + const target = record.targets[index]!; + const state = states[index]!; + if (state === "retired") { + fsyncDirectory(path.dirname(canonical(homeDir, target))); + continue; + } + const source = canonical(homeDir, target); + const staged = cleanup(homeDir, record, target); + const exact = readFile(state === "canonical" ? source : staged, roleLimit(target[0]))!; + if (!matches(record, target, exact, state === "cleanup")) + throw new Error("Portable uninstall target changed"); + const replacement = + allowCanonicalReplacement && + state === "cleanup" && + readFile(source, roleLimit(target[0])) !== null; + detachDelete(replacement ? null : source, staged, exact, roleLimit(target[0]), undefined, true); + } +} +export function publishAndRetirePortableEvidence(prepared: PreparedPortableRetirement): void { + if (!allTargetsExact(prepared.homeDir, prepared.record)) + throw new Error("Portable uninstall authority changed before publication"); + publish(prepared); + retireTargets(prepared.homeDir, prepared.record); +} + +function load(homeDir: string): { file: ExactFile; record: RecordState } | null { + const state = paths(homeDir); + if ([state.RC, state.S, state.SC].some(entryExists)) + throw new Error("Portable uninstall supersession is incomplete"); + recoverTemp(homeDir); + const record = readRecord(state.R, 1n); + if (record) fsyncDirectory(path.dirname(state.R)); + return record; +} +export const hasPortableRetirementRecord = (homeDir: string): boolean => admitSupersession(homeDir); +function inspectRetirementRecovery( + homeDir: string, + allowCanonicalReplacement = false, +): PortableRetirementRecovery | null { + const fixed = allowCanonicalReplacement ? fixedSupersessionState(homeDir) : null; + const recorded = allowCanonicalReplacement ? (fixed?.files[0] ?? null) : load(homeDir); + if (!recorded) return null; + const states = targetStates(homeDir, recorded.record, allowCanonicalReplacement); + const artifacts = recorded.record.targets.flatMap((target, index) => + states[index] === "cleanup" + ? [{ basename: path.basename(cleanup(homeDir, recorded.record, target)), root: target[0] }] + : [], + ); + const registry = recorded.record.targets.at(-1)!; + const state = states.at(-1)!; + const registryBytes = + state === "retired" + ? null + : readFile( + state === "canonical" + ? canonical(homeDir, registry) + : cleanup(homeDir, recorded.record, registry), + roleLimit("registry"), + )!.bytes; + return { artifacts, fixedState: fixed?.code ?? "1000", registryBytes }; +} +export const inspectPortableRetirementRecovery = ( + homeDir: string, +): PortableRetirementRecovery | null => inspectRetirementRecovery(homeDir); +export const inspectPortableOnboardSupersession = ( + homeDir: string, +): PortableRetirementRecovery | null => inspectRetirementRecovery(homeDir, true); +export function resumePortableOnboardReplacementEvidence(homeDir: string): void { + const fixed = fixedSupersessionState(homeDir); + if (!fixed || fixed.code !== "1000") + throw new Error("Portable onboarding replacement recovery requires the canonical record"); + retireTargets(homeDir, fixed.files[0]!.record, true); +} +export function resumePortableEvidenceRetirement(homeDir: string): void { + const recorded = load(homeDir); + if (!recorded) throw new Error("Portable uninstall retirement record is missing"); + retireTargets(homeDir, recorded.record); +} + +function durableFile(target: string): ExactFile { + const descriptor = fs.openSync(target, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + try { + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } + fsyncDirectory(path.dirname(target)); + const file = readFile(target, MAX_AUTHORITY); + if (!file) throw new Error(`Completed onboarding authority is missing: ${target}`); + return file; +} +export function readPortableAuthoritySnapshot( + target: string, + limit = MAX_AUTHORITY, +): Buffer | null { + return readFile(target, limit)?.bytes ?? null; +} +export function provePortableOnboardAuthority(admission: PortableOnboardAuthorityAdmission): void { + if ( + admission.files.length < 2 || + admission.files.length > 3 || + new Set(admission.files).size !== admission.files.length + ) + throw new Error("Completed onboarding authority paths are invalid"); + const snapshots = new Map(admission.files.map((target) => [target, durableFile(target)])); + admission.verify( + new Map([...snapshots].map(([target, file]) => [target, Buffer.from(file.bytes)])), + ); + for (const [target, expected] of snapshots) { + const current = readFile(target, MAX_AUTHORITY); + if ( + !current || + current.dev !== expected.dev || + current.ino !== expected.ino || + !current.bytes.equals(expected.bytes) + ) + throw new Error("Completed onboarding authority changed during verification"); + } +} +function fixedSupersessionState(homeDir: string): { + readonly code: string; + readonly files: readonly { file: ExactFile; record: RecordState }[]; +} | null { + const state = paths(homeDir); + recoverTemp(homeDir); + const statePaths = [state.R, state.RC, state.S, state.SC]; + const code = statePaths.map((target) => (entryExists(target) ? "1" : "0")).join(""); + if (code === "0000") return null; + const expectedLinks = new Map([ + ["1000", 1n], + ["1010", 2n], + ["0110", 2n], + ["0010", 1n], + ["0001", 1n], + ]).get(code); + if (expectedLinks === undefined) throw new Error("Portable uninstall supersession is ambiguous"); + const records = statePaths.map((target) => + entryExists(target) ? readRecord(target, expectedLinks) : null, + ); + const present = records.filter((entry): entry is NonNullable => Boolean(entry)); + if (!present.length) return null; + const admitted = present[0]!.file; + if ( + present.some( + (entry) => + entry.file.dev !== admitted.dev || + entry.file.ino !== admitted.ino || + entry.file.nlink !== admitted.nlink || + !entry.file.bytes.equals(admitted.bytes), + ) + ) + throw new Error("Portable uninstall supersession is ambiguous"); + return { code, files: present }; +} +function admitSupersession(homeDir: string): boolean { + return fixedSupersessionState(homeDir) !== null; +} +function supersede(homeDir: string): void { + const state = paths(homeDir); + const fixed = fixedSupersessionState(homeDir); + if (!fixed) return; + const admitted = fixed.files[0]!.file; + if (fixed.code === "1000") { + fs.linkSync(state.R, state.S); + fsyncDirectory(path.dirname(state.R)); + const linkedR = readRecord(state.R, 2n); + const linkedS = readRecord(state.S, 2n); + if (!linkedR || !linkedS || !sameExact(linkedR.file, linkedS.file)) + throw new Error("Portable uninstall supersession link publication failed"); + return supersede(homeDir); + } else if (fixed.code === "1010" || fixed.code === "0110") { + const survivor = readRecord(state.S, 2n)!; + fsyncDirectory(path.dirname(state.R)); + detachDelete(state.R, state.RC, admitted, MAX_RECORD, { + expected: survivor.file, + path: state.S, + }); + return supersede(homeDir); + } else if (fixed.code === "0010" || fixed.code === "0001") { + detachDelete(state.S, state.SC, admitted, MAX_RECORD); + return supersede(homeDir); + } + throw new Error("Portable uninstall supersession is ambiguous"); +} +export function supersedePortableRetirementAfterOnboard( + homeDir: string, + admission: PortableOnboardAuthorityAdmission, +): void { + const recorded = fixedSupersessionState(homeDir)?.files[0] ?? null; + if (!recorded) return; + provePortableOnboardAuthority(admission); + retireTargets(homeDir, recorded.record, true); + provePortableOnboardAuthority(admission); + inspectRetirementRecovery(homeDir, true); + supersede(homeDir); +} diff --git a/src/lib/state/registry-lock.test.ts b/src/lib/state/registry-lock.test.ts index 32807c72fc6..c26b7d8aa76 100644 --- a/src/lib/state/registry-lock.test.ts +++ b/src/lib/state/registry-lock.test.ts @@ -1,112 +1,478 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; -import { classifyExistingLock } from "./registry"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + acquireProcessBoundLockAt, + classifyExistingLock, + releaseProcessBoundLock, + withProcessBoundRegistryLockAt, + withRegistryLockAt, + type RegistryLockDeps, +} from "./registry/lock"; const STALE = 10_000; const LOCK_MTIME = 1_000_000; +const PROCESS_IDENTITY = "12345678-1234-1234-1234-123456789abc 123456"; +const RECYCLED_IDENTITY = "12345678-1234-1234-1234-123456789abc 123457"; +const OTHER_IDENTITY = "87654321-4321-4321-4321-cba987654321 123456"; +const temporaryDirectories: string[] = []; + +function fixture(prefix: string) { + const homeDir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + temporaryDirectories.push(homeDir); + const registryFile = path.join(homeDir, ".nemoclaw", "sandboxes.json"); + const lockDir = `${registryFile}.lock`; + return { + homeDir, + lockDir, + ownerFile: path.join(lockDir, "owner"), + processStartFile: path.join(lockDir, "process-start"), + registryFile, + }; +} + +function writeOrdinaryGeneration(test: ReturnType, ownerPid: number): void { + fs.mkdirSync(test.lockDir, { recursive: true, mode: 0o700 }); + fs.writeFileSync(test.ownerFile, String(ownerPid), { mode: 0o600 }); +} + +function writeExactGeneration( + test: ReturnType, + ownerPid: number, + identity: string, +): void { + writeOrdinaryGeneration(test, ownerPid); + fs.writeFileSync(test.processStartFile, `${String(ownerPid)} ${identity}\n`, { mode: 0o600 }); +} + +function markStale(lockDir: string): void { + fs.utimesSync(lockDir, new Date(LOCK_MTIME), new Date(LOCK_MTIME)); +} + +function quarantineDirectories(lockDir: string): string[] { + const parent = path.dirname(lockDir); + const prefix = `${path.basename(lockDir)}.quarantine.`; + return fs + .readdirSync(parent) + .filter((entry) => entry.startsWith(prefix)) + .map((entry) => path.join(parent, entry)); +} + +function interceptDetach( + lockDir: string, + replace: (rename: typeof fs.renameSync, quarantine: string) => void, +): void { + const rename = fs.renameSync.bind(fs) as typeof fs.renameSync; + vi.spyOn(fs, "renameSync").mockImplementation((source, destination) => + String(source) === lockDir && String(destination).startsWith(`${lockDir}.quarantine.`) + ? replace(rename, String(destination)) + : rename(source, destination), + ); +} -describe("registry lock staleness (PID-recycle wedge)", () => { - it("breaks a lock whose owner process is dead", () => { +function exactDeps(overrides: RegistryLockDeps = {}): RegistryLockDeps { + return { + isProcessAlive: () => true, + now: () => Number.MAX_SAFE_INTEGER, + readProcessIdentity: () => PROCESS_IDENTITY, + wait: () => undefined, + ...overrides, + }; +} + +afterEach(() => { + vi.restoreAllMocks(); + for (const directory of temporaryDirectories.splice(0)) { + fs.rmSync(directory, { force: true, recursive: true }); + } +}); + +describe("registry lock ownership decisions", () => { + it("breaks a dead owner and an exact recycled owner without waiting", () => { expect( classifyExistingLock({ - ownerPid: 4242, ownerAlive: false, - processStartMs: null, + ownerPid: 4242, + ownerStatus: "ordinary", lockMtimeMs: LOCK_MTIME, - nowMs: LOCK_MTIME + 50, + nowMs: LOCK_MTIME + 1, staleMs: STALE, }), ).toBe("break"); - }); - - it("breaks a lock held by a RECYCLED pid that started after the lock (self-found wedge)", () => { - // kill(pid,0) succeeds, but the live process started after the lock was - // taken -> the original holder crashed and the pid was reused. A - // liveness-only check would wedge here forever. expect( classifyExistingLock({ - ownerPid: 4242, ownerAlive: true, - processStartMs: LOCK_MTIME + 5_000, + ownerPid: 4242, + ownerStatus: "recycled", lockMtimeMs: LOCK_MTIME, - nowMs: LOCK_MTIME + 5_100, + nowMs: LOCK_MTIME + 1, staleMs: STALE, }), ).toBe("break"); }); - it("waits on a live original holder with a fresh lock", () => { + it("never age-breaks an exact original or unverifiable process-bound owner", () => { + for (const ownerStatus of ["original", "unverifiable"] as const) { + expect( + classifyExistingLock({ + ownerAlive: true, + ownerPid: 4242, + ownerStatus, + lockMtimeMs: LOCK_MTIME, + nowMs: Number.MAX_SAFE_INTEGER, + staleMs: STALE, + }), + ).toBe("wait"); + } + }); + + it("retains the bounded age rule for ordinary or unreadable owners", () => { + for (const [ownerPid, nowMs, expected] of [ + [4242, LOCK_MTIME + 1, "wait"], + [4242, LOCK_MTIME + STALE + 1, "break"], + [null, LOCK_MTIME + 1, "wait"], + [null, LOCK_MTIME + STALE + 1, "break"], + ] as const) { + expect( + classifyExistingLock({ + ownerAlive: ownerPid !== null, + ownerPid, + ownerStatus: "ordinary", + lockMtimeMs: LOCK_MTIME, + nowMs, + staleMs: STALE, + }), + ).toBe(expected); + } + }); +}); + +describe("process-bound registry locking", () => { + it("holds beyond ten seconds and makes a contender exhaust 120 bounded retries", () => { + const test = fixture("nemoclaw-process-bound-lock-"); + const wait = vi.fn(); + const contender = vi.fn(); + + withProcessBoundRegistryLockAt( + test.registryFile, + () => { + expect(fs.readFileSync(test.ownerFile, "utf8")).toBe(String(process.pid)); + expect(fs.readFileSync(test.processStartFile, "utf8")).toBe( + `${String(process.pid)} ${PROCESS_IDENTITY}\n`, + ); + expect(() => withRegistryLockAt(test.registryFile, contender, exactDeps({ wait }))).toThrow( + /after 120 retries/, + ); + }, + exactDeps(), + ); + + expect(wait).toHaveBeenCalledTimes(120); + expect(contender).not.toHaveBeenCalled(); + expect(fs.existsSync(test.lockDir)).toBe(false); + }); + + it("fails before effects without an exact process identity", () => { + const test = fixture("nemoclaw-process-bound-precondition-"); + const callback = vi.fn(); + + expect(() => + withProcessBoundRegistryLockAt(test.registryFile, callback, { + readProcessIdentity: () => null, + }), + ).toThrow(/exact Linux process-start identity/); + expect(callback).not.toHaveBeenCalled(); + expect(fs.existsSync(path.dirname(test.registryFile))).toBe(false); + }); + + it("fails before effects when current-user identity is unavailable", () => { + const test = fixture("nemoclaw-process-bound-no-uid-"); + const callback = vi.fn(); + vi.spyOn(process, "getuid").mockImplementation(() => undefined as never); + + expect(() => + withProcessBoundRegistryLockAt(test.registryFile, callback, { + readProcessIdentity: () => PROCESS_IDENTITY, + }), + ).toThrow(/exact Linux process-start identity/); + expect(callback).not.toHaveBeenCalled(); + expect(fs.existsSync(path.dirname(test.registryFile))).toBe(false); + }); + + it("breaks a crashed owner and acquires the replacement generation", () => { + const test = fixture("nemoclaw-dead-lock-"); + writeExactGeneration(test, 4242, PROCESS_IDENTITY); + expect( - classifyExistingLock({ - ownerPid: 4242, - ownerAlive: true, - processStartMs: LOCK_MTIME - 2_000, - lockMtimeMs: LOCK_MTIME, - nowMs: LOCK_MTIME + 200, - staleMs: STALE, + withRegistryLockAt(test.registryFile, () => "acquired", { + ...exactDeps(), + isProcessAlive: () => false, + maxRetries: 2, }), - ).toBe("wait"); + ).toBe("acquired"); + expect(fs.existsSync(test.lockDir)).toBe(false); }); - it("breaks a live original holder once the lock is clearly stale (wedged holder)", () => { + it("breaks a rapidly recycled PID from exact tick identity without a time tolerance", () => { + const test = fixture("nemoclaw-recycled-lock-"); + writeExactGeneration(test, 4242, PROCESS_IDENTITY); + expect( - classifyExistingLock({ - ownerPid: 4242, - ownerAlive: true, - processStartMs: LOCK_MTIME - 2_000, - lockMtimeMs: LOCK_MTIME, - nowMs: LOCK_MTIME + STALE + 1, - staleMs: STALE, + withRegistryLockAt(test.registryFile, () => "acquired", { + ...exactDeps({ readProcessIdentity: () => RECYCLED_IDENTITY }), + maxRetries: 2, + now: () => Number(fs.lstatSync(test.lockDir).mtimeMs) + 1, }), - ).toBe("break"); + ).toBe("acquired"); }); - it("falls back to age when the owner pid is unreadable", () => { + it("waits on a live process-bound owner when its current identity is unavailable", () => { + const test = fixture("nemoclaw-unverifiable-lock-"); + writeExactGeneration(test, 4242, PROCESS_IDENTITY); + markStale(test.lockDir); + const wait = vi.fn(); + + expect(() => + withRegistryLockAt(test.registryFile, () => undefined, { + ...exactDeps({ readProcessIdentity: () => null, wait }), + maxRetries: 1, + }), + ).toThrow(/after 1 retries/); + expect(wait).toHaveBeenCalledOnce(); + expect(fs.existsSync(test.lockDir)).toBe(true); + }); + + it("keeps ordinary non-Linux acquisition and release behavior", () => { + const test = fixture("nemoclaw-ordinary-no-proc-"); + vi.spyOn(process, "getuid").mockImplementation(() => undefined as never); + expect( - classifyExistingLock({ - ownerPid: null, - ownerAlive: false, - processStartMs: null, - lockMtimeMs: LOCK_MTIME, - nowMs: LOCK_MTIME + 200, - staleMs: STALE, + withRegistryLockAt(test.registryFile, () => "ordinary", { + readProcessIdentity: () => null, }), - ).toBe("wait"); + ).toBe("ordinary"); + expect(fs.existsSync(test.lockDir)).toBe(false); + }); + + it("uses the ordinary age rule for a legacy owner without a process sidecar", () => { + const test = fixture("nemoclaw-legacy-age-lock-"); + writeOrdinaryGeneration(test, 4242); + markStale(test.lockDir); + expect( - classifyExistingLock({ - ownerPid: null, - ownerAlive: false, - processStartMs: null, - lockMtimeMs: LOCK_MTIME, - nowMs: LOCK_MTIME + STALE + 1, - staleMs: STALE, + withRegistryLockAt(test.registryFile, () => "acquired", { + ...exactDeps(), + maxRetries: 2, }), - ).toBe("break"); + ).toBe("acquired"); }); - it("falls back to age when start time is unavailable (non-/proc host)", () => { - // alive, but processStartMs null (no /proc): fresh -> wait, stale -> break. + it.each([ + [ + "numeric owner with trailing text", + (test: ReturnType) => { + fs.writeFileSync(test.ownerFile, "4242junk", { mode: 0o600 }); + }, + ], + [ + "wrong-mode owner", + (test: ReturnType) => { + fs.chmodSync(test.ownerFile, 0o644); + }, + ], + [ + "symbolic-link owner", + (test: ReturnType) => { + const target = path.join(test.homeDir, "owner-target"); + fs.writeFileSync(target, "4242", { mode: 0o600 }); + fs.unlinkSync(test.ownerFile); + fs.symlinkSync(target, test.ownerFile); + }, + ], + ])("does not trust a %s", (_case, mutateOwner) => { + const test = fixture("nemoclaw-unsafe-owner-"); + writeOrdinaryGeneration(test, 4242); + mutateOwner(test); + markStale(test.lockDir); + expect( - classifyExistingLock({ - ownerPid: 4242, - ownerAlive: true, - processStartMs: null, - lockMtimeMs: LOCK_MTIME, - nowMs: LOCK_MTIME + 200, - staleMs: STALE, + withRegistryLockAt(test.registryFile, () => "acquired", { + ...exactDeps(), + maxRetries: 2, }), - ).toBe("wait"); + ).toBe("acquired"); + }); +}); + +describe("generation-safe registry lock removal", () => { + it("holds and releases one opaque process-bound generation", () => { + const test = fixture("nemoclaw-opaque-handle-"); + const handle = acquireProcessBoundLockAt(test.lockDir, exactDeps()); + + expect(fs.readFileSync(test.ownerFile, "utf8")).toBe(String(process.pid)); + expect(() => acquireProcessBoundLockAt(test.lockDir, exactDeps({ maxRetries: 1 }))).toThrow( + /after 1 retries/, + ); + releaseProcessBoundLock(handle); + expect(fs.existsSync(test.lockDir)).toBe(false); + expect(() => releaseProcessBoundLock(handle)).toThrow(/inactive/); + }); + + it("preserves a replacement when an opaque-handle release detaches a mismatch", () => { + const test = fixture("nemoclaw-opaque-release-replacement-"); + const handle = acquireProcessBoundLockAt(test.lockDir, exactDeps()); + interceptDetach(test.lockDir, (rename, quarantine) => { + rename(test.lockDir, quarantine); + fs.mkdirSync(test.lockDir, { mode: 0o700 }); + fs.writeFileSync(test.ownerFile, "4343", { mode: 0o600 }); + }); + + expect(() => releaseProcessBoundLock(handle)).not.toThrow(); + expect(fs.readFileSync(test.ownerFile, "utf8")).toBe("4343"); + expect(quarantineDirectories(test.lockDir)).toEqual([]); + }); + + it("preserves a replacement that wins before stale detach", () => { + const test = fixture("nemoclaw-before-detach-"); + const displaced = `${test.lockDir}.displaced`; + writeOrdinaryGeneration(test, 4242); + markStale(test.lockDir); + interceptDetach(test.lockDir, (rename, quarantine) => { + rename(test.lockDir, displaced); + fs.mkdirSync(test.lockDir, { mode: 0o700 }); + fs.writeFileSync(test.ownerFile, "4343", { mode: 0o600 }); + rename(test.lockDir, quarantine); + }); + + expect(() => + withRegistryLockAt(test.registryFile, () => undefined, { + ...exactDeps(), + maxRetries: 1, + }), + ).toThrow(/after 1 retries/); + expect(fs.existsSync(test.lockDir)).toBe(false); + expect(fs.readFileSync(path.join(displaced, "owner"), "utf8")).toBe("4242"); + const [quarantine] = quarantineDirectories(test.lockDir); + expect(fs.readFileSync(path.join(quarantine!, "owner"), "utf8")).toBe("4343"); + }); + + it("leaves a detached mismatch when another canonical generation appears", () => { + const test = fixture("nemoclaw-restore-winner-"); + const displaced = `${test.lockDir}.displaced`; + writeOrdinaryGeneration(test, 4242); + markStale(test.lockDir); + interceptDetach(test.lockDir, (rename, quarantine) => { + rename(test.lockDir, displaced); + fs.mkdirSync(test.lockDir, { mode: 0o700 }); + fs.writeFileSync(test.ownerFile, "4343", { mode: 0o600 }); + rename(test.lockDir, quarantine); + fs.mkdirSync(test.lockDir, { mode: 0o700 }); + fs.writeFileSync(test.ownerFile, "4444", { mode: 0o600 }); + }); + + expect(() => + withRegistryLockAt(test.registryFile, () => undefined, { + ...exactDeps(), + maxRetries: 1, + }), + ).toThrow(/after 1 retries/); + const [quarantine] = quarantineDirectories(test.lockDir); + expect(fs.readFileSync(test.ownerFile, "utf8")).toBe("4444"); + expect(fs.readFileSync(path.join(quarantine!, "owner"), "utf8")).toBe("4343"); + expect(fs.readFileSync(path.join(displaced, "owner"), "utf8")).toBe("4242"); + }); + + it("does not delete a simultaneous stale breaker's generation after detach", () => { + const test = fixture("nemoclaw-simultaneous-breakers-"); + writeOrdinaryGeneration(test, 4242); + markStale(test.lockDir); + interceptDetach(test.lockDir, (rename, quarantine) => { + rename(test.lockDir, quarantine); + fs.mkdirSync(test.lockDir, { mode: 0o700 }); + fs.writeFileSync(test.ownerFile, "4343", { mode: 0o600 }); + fs.writeFileSync(test.processStartFile, `4343 ${OTHER_IDENTITY}\n`, { mode: 0o600 }); + }); + const ownerLiveness = new Map([ + [4242, false], + [4343, true], + ]); + const currentIdentities = new Map([[4343, OTHER_IDENTITY]]); + + expect(() => + withRegistryLockAt(test.registryFile, () => undefined, { + isProcessAlive: (pid) => ownerLiveness.get(pid) ?? true, + maxRetries: 2, + now: () => Number.MAX_SAFE_INTEGER, + readProcessIdentity: (pid) => currentIdentities.get(pid) ?? PROCESS_IDENTITY, + wait: () => undefined, + }), + ).toThrow(/after 2 retries/); + expect(fs.readFileSync(test.ownerFile, "utf8")).toBe("4343"); + expect(quarantineDirectories(test.lockDir)).toEqual([]); + }); + + it("removes only the released generation after canonical recreation", () => { + const test = fixture("nemoclaw-release-recreation-"); + expect( - classifyExistingLock({ - ownerPid: 4242, - ownerAlive: true, - processStartMs: null, - lockMtimeMs: LOCK_MTIME, - nowMs: LOCK_MTIME + STALE + 1, - staleMs: STALE, + withRegistryLockAt( + test.registryFile, + () => { + interceptDetach(test.lockDir, (rename, quarantine) => { + rename(test.lockDir, quarantine); + fs.mkdirSync(test.lockDir, { mode: 0o700 }); + fs.writeFileSync(test.ownerFile, "4343", { mode: 0o600 }); + }); + return "released"; + }, + exactDeps(), + ), + ).toBe("released"); + expect(fs.readFileSync(test.ownerFile, "utf8")).toBe("4343"); + expect(quarantineDirectories(test.lockDir)).toEqual([]); + }); + + it("quarantines an owner-file replacement and rejects release", () => { + const test = fixture("nemoclaw-owner-replacement-"); + + expect(() => + withRegistryLockAt( + test.registryFile, + () => { + const replacement = path.join(test.lockDir, "replacement-owner"); + fs.writeFileSync(replacement, String(process.pid), { mode: 0o600 }); + fs.renameSync(replacement, test.ownerFile); + }, + exactDeps(), + ), + ).toThrow(/changed ownership/); + expect(fs.existsSync(test.lockDir)).toBe(false); + const [quarantine] = quarantineDirectories(test.lockDir); + expect(fs.readFileSync(path.join(quarantine!, "owner"), "utf8")).toBe(String(process.pid)); + expect(fs.existsSync(path.join(quarantine!, "process-start"))).toBe(true); + }); + + it("cleans only its detached failed-acquisition generation", () => { + const test = fixture("nemoclaw-acquisition-cleanup-"); + const identities = [PROCESS_IDENTITY, RECYCLED_IDENTITY]; + const readProcessIdentity = vi.fn(() => identities.shift() ?? RECYCLED_IDENTITY); + interceptDetach(test.lockDir, (rename, quarantine) => { + rename(test.lockDir, quarantine); + fs.mkdirSync(test.lockDir, { mode: 0o700 }); + fs.writeFileSync(test.ownerFile, "4343", { mode: 0o600 }); + }); + + expect(() => + withProcessBoundRegistryLockAt(test.registryFile, () => undefined, { + readProcessIdentity, }), - ).toBe("break"); + ).toThrow(/identity changed during acquisition/); + expect(fs.readFileSync(test.ownerFile, "utf8")).toBe("4343"); + expect(quarantineDirectories(test.lockDir)).toEqual([]); }); }); diff --git a/src/lib/state/registry/lock.ts b/src/lib/state/registry/lock.ts index bdf1af16498..79841ce896f 100644 --- a/src/lib/state/registry/lock.ts +++ b/src/lib/state/registry/lock.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import crypto from "node:crypto"; import fs from "node:fs"; import path from "node:path"; import { isErrnoException } from "../../core/errno"; @@ -9,11 +10,34 @@ import { REGISTRY_FILE } from "./persistence"; export const LOCK_DIR = `${REGISTRY_FILE}.lock`; export const LOCK_OWNER = path.join(LOCK_DIR, "owner"); +export const LOCK_PROCESS_START = path.join(LOCK_DIR, "process-start"); export const LOCK_STALE_MS = 10_000; export const LOCK_RETRY_MS = 100; export const LOCK_MAX_RETRIES = 120; -/** kill(pid, 0) liveness probe. EPERM means the pid exists but is owned by - * another user, which still counts as alive. */ +const BOOT_ID = /^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/u; + +type Paths = { readonly directory: string; readonly owner: string; readonly processStart: string }; +type Identity = { readonly device: bigint; readonly inode: bigint }; +type Generation = Identity & { readonly paths: Paths }; +type Acquired = Generation & { readonly ownerFile: Identity; readonly processFile: Identity | null; readonly processRecord: string | null }; + +export interface RegistryLockDeps { readonly isProcessAlive?: (pid: number) => boolean; readonly maxRetries?: number; readonly now?: () => number; readonly readProcessIdentity?: (pid: number) => string | null; readonly wait?: () => void } +export type RegistryOwnerStatus = "ordinary" | "original" | "recycled" | "unverifiable"; +export type RegistryLockDecision = "break" | "wait"; +export type ProcessBoundLockHandle = object; + +const handles = new WeakMap(); +const at = (directory: string): Paths => ({ directory, owner: path.join(directory, "owner"), processStart: path.join(directory, "process-start") }); +const identity = (target: string): Identity | null => { + try { + const stat = fs.lstatSync(target, { bigint: true }); + return { device: stat.dev, inode: stat.ino }; + } catch { + return null; + } +}; +const sameIdentity = (left: Identity | null, right: Identity | null): boolean => left?.device === right?.device && left?.inode === right?.inode; + function isProcessAlive(pid: number): boolean { try { process.kill(pid, 0); @@ -23,190 +47,238 @@ function isProcessAlive(pid: number): boolean { } } -/** Wall-clock start time (ms since epoch) of `pid` from /proc, or null when it - * cannot be read (process gone, or a non-Linux host without /proc). Mirrors the - * onboard-session lock's recycle check. */ -function readProcessStartMs(pid: number): number | null { +function processIdentity(pid: number): string | null { try { - const statText = fs.readFileSync(`/proc/${pid}/stat`, "utf8"); - const btimeLine = fs - .readFileSync("/proc/stat", "utf8") - .split("\n") - .find((line) => line.startsWith("btime ")); - const bootSeconds = btimeLine ? Number(btimeLine.trim().split(/\s+/)[1]) : NaN; - const closeParen = statText.lastIndexOf(")"); - if (!Number.isFinite(bootSeconds) || closeParen < 0) return null; - const fieldsAfterComm = statText - .slice(closeParen + 2) - .trim() - .split(/\s+/); - const startTicks = Number(fieldsAfterComm[19]); - if (!Number.isFinite(startTicks)) return null; - // /proc//stat starttime is in USER_HZ ticks (100 on supported hosts). - const clockTicksPerSecond = 100; - return (bootSeconds + startTicks / clockTicksPerSecond) * 1000; + const stat = fs.readFileSync(`/proc/${String(pid)}/stat`, "utf8"); + const close = stat.lastIndexOf(")"); + const tick = close < 0 ? undefined : stat.slice(close + 2).trim().split(/\s+/u)[19]; + const boot = fs.readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim(); + return tick && /^[1-9][0-9]*$/u.test(tick) && BOOT_ID.test(boot) ? `${boot} ${tick}` : null; } catch { return null; } } -export type RegistryLockDecision = "break" | "wait"; +function sameFile(left: fs.BigIntStats, right: fs.BigIntStats): boolean { + return (["dev", "ino", "size", "mode", "uid", "nlink", "mtimeNs", "ctimeNs"] as const).every((key) => left[key] === right[key]); +} -/** - * Decide whether an existing registry lock should be broken (stale) or waited - * on. Exported for tests. - * - * The PID-recycle wedge this guards against: a holder that crashes without - * releasing leaves `LOCK_DIR` + the owner pid behind. If that pid is later - * reused by an unrelated live process, `kill(pid, 0)` succeeds, so a - * liveness-only check treats the lock as held forever and every registry write - * wedges (retries exhausted -> "Failed to acquire lock"). When the owner looks - * alive we therefore also confirm it started BEFORE it took the lock: a process - * whose /proc start time is after the lock's mtime is a recycled pid, so the - * lock is stale. When the owner pid or its start time cannot be read (missing - * owner file, non-Linux host), fall back to breaking the lock once it is older - * than a registry op could legitimately take. - */ -export function classifyExistingLock(opts: { - ownerPid: number | null; - ownerAlive: boolean; - processStartMs: number | null; - lockMtimeMs: number; - nowMs: number; - staleMs: number; -}): RegistryLockDecision { - const ageMs = opts.nowMs - opts.lockMtimeMs; - if (opts.ownerPid === null) { - // Owner file missing or unreadable: decide on age alone. - return ageMs > opts.staleMs ? "break" : "wait"; - } - if (!opts.ownerAlive) { - return "break"; +function readOwnedFile(target: string, limit: number, requireUid: boolean): string | null { + let descriptor: number | null = null; + try { + const uid = process.getuid?.(); + if (requireUid && uid === undefined) return null; + descriptor = fs.openSync(target, fs.constants.O_RDONLY | (process.platform === "win32" ? 0 : fs.constants.O_NOFOLLOW)); + const before = fs.fstatSync(descriptor, { bigint: true }); + const pathBefore = fs.lstatSync(target, { bigint: true }); + if (!before.isFile() || !pathBefore.isFile() || pathBefore.isSymbolicLink() || !sameFile(before, pathBefore) || before.nlink !== 1n || before.size < 1n || before.size > BigInt(limit) || (uid !== undefined && (before.uid !== BigInt(uid) || (before.mode & 0o777n) !== 0o600n))) + return null; + const bytes = Buffer.alloc(Number(before.size)); + const read = fs.readSync(descriptor, bytes, 0, bytes.length, 0); + const after = fs.fstatSync(descriptor, { bigint: true }); + const pathAfter = fs.lstatSync(target, { bigint: true }); + return read === bytes.length && sameFile(before, after) && sameFile(before, pathAfter) ? bytes.toString("utf8") : null; + } catch { + return null; + } finally { + if (descriptor !== null) fs.closeSync(descriptor); } - if (opts.processStartMs !== null && opts.processStartMs > opts.lockMtimeMs + 1000) { - // Live pid that started after the lock was taken -> the pid was recycled. - return "break"; +} + +function ownerPid(target: string): number | null { + const value = readOwnedFile(target, 32, false); + if (value === null || !/^[1-9][0-9]{0,15}\n?$/u.test(value)) return null; + const pid = Number(value.trim()); + return Number.isSafeInteger(pid) ? pid : null; +} + +function readProcessRecord(target: string, pid: number): string | null { + const value = readOwnedFile(target, 128, true); + const match = value && /^([1-9][0-9]{0,15}) ([0-9a-f-]{36}) ([1-9][0-9]*)\n?$/u.exec(value); + return match && match[1] === String(pid) && BOOT_ID.test(match[2]!) + ? `${match[1]} ${match[2]} ${match[3]}` + : null; +} + +export function classifyExistingLock(options: { ownerAlive: boolean; ownerPid: number | null; ownerStatus: RegistryOwnerStatus; lockMtimeMs: number; nowMs: number; staleMs: number }): RegistryLockDecision { + if (options.ownerPid === null) + return options.nowMs - options.lockMtimeMs > options.staleMs ? "break" : "wait"; + if (!options.ownerAlive || options.ownerStatus === "recycled") return "break"; + if (options.ownerStatus !== "ordinary") return "wait"; + return options.nowMs - options.lockMtimeMs > options.staleMs ? "break" : "wait"; +} + +function status(pid: number, alive: boolean, record: string | null, read: (pid: number) => string | null): RegistryOwnerStatus { + if (record === null) return "ordinary"; + if (!alive) return "recycled"; + const current = read(pid); + return current === null ? "unverifiable" : record === `${String(pid)} ${current}` ? "original" : "recycled"; +} + +function sameGeneration(generation: Generation): boolean { + const current = identity(generation.paths.directory); + return sameIdentity(current, generation); +} + +/** Atomically detach one generation; never restore or sweep an ambiguous generation. */ +function removeGeneration(generation: Generation, files: readonly (readonly [string, Identity])[], validate: (paths: Paths) => boolean): void { + const quarantine = `${generation.paths.directory}.quarantine.${String(process.pid)}.${crypto.randomUUID()}`; + fs.renameSync(generation.paths.directory, quarantine); + const detached = { ...identity(quarantine)!, paths: at(quarantine) }; + const mapped = files.map(([target, expected]) => [path.join(quarantine, path.basename(target)), expected] as const); + const names = mapped.map(([target]) => path.basename(target)).sort(); + const exactNames = JSON.stringify(fs.readdirSync(quarantine).sort()) === JSON.stringify(names); + if (!sameIdentity(detached, generation) || !sameGeneration(detached) || !exactNames || mapped.some(([target, expected]) => !sameIdentity(identity(target), expected)) || !validate(detached.paths)) + throw new Error("Registry lock changed ownership"); + fs.rmSync(quarantine, { recursive: true }); +} + +function tryRemove(generation: Generation, pid: number | null, owner: Identity | null, record: string | null, processFile: Identity | null, mtimeNs: bigint): boolean { + const files: [string, Identity][] = []; + if (owner) files.push([generation.paths.owner, owner]); + if (processFile) files.push([generation.paths.processStart, processFile]); + try { + removeGeneration(generation, files, (paths) => fs.lstatSync(paths.directory, { bigint: true }).mtimeNs === mtimeNs && ownerPid(paths.owner) === pid && (pid === null || readProcessRecord(paths.processStart, pid) === record)); + return true; + } catch { + return false; } - // Live original holder (or start time unknown): only break once the lock is - // clearly older than a registry op could take, which also covers hosts where - // recycle cannot be detected directly. - return ageMs > opts.staleMs ? "break" : "wait"; } -/** Acquire an advisory lock using mkdir (atomic on POSIX). */ -export function acquireLock(): void { - ensureConfigDir(path.dirname(REGISTRY_FILE)); - const sleepBuf = new Int32Array(new SharedArrayBuffer(4)); - for (let i = 0; i < LOCK_MAX_RETRIES; i++) { +function acquire(directory: string, exact: boolean, deps: RegistryLockDeps): Acquired { + const readIdentity = deps.readProcessIdentity ?? processIdentity; + const initialIdentity = readIdentity(process.pid); + if (exact && (initialIdentity === null || process.getuid?.() === undefined)) + throw new Error("Portable registry locking requires an exact Linux process-start identity"); + const paths = at(directory); + const alive = deps.isProcessAlive ?? isProcessAlive; + const retries = deps.maxRetries ?? LOCK_MAX_RETRIES; + const now = deps.now ?? Date.now; + const sleep = new Int32Array(new SharedArrayBuffer(4)); + const wait = deps.wait ?? (() => Atomics.wait(sleep, 0, 0, LOCK_RETRY_MS)); + fs.mkdirSync(path.dirname(directory), { recursive: true }); + for (let attempt = 0; attempt < retries; attempt++) { try { - fs.mkdirSync(LOCK_DIR); + fs.mkdirSync(directory, { mode: 0o700 }); } catch (error) { - if (!isErrnoException(error) || error.code !== "EEXIST") { - throw error; - } - let lockStat: fs.Stats; - try { - lockStat = fs.statSync(LOCK_DIR); - } catch { - // Lock dir vanished between the failed mkdir and this stat: another - // waiter released it, so retry immediately. - continue; - } - let ownerPid: number | null = null; + if (!isErrnoException(error) || error.code !== "EEXIST") throw error; + let lock: fs.BigIntStats; try { - const parsed = Number.parseInt(fs.readFileSync(LOCK_OWNER, "utf-8").trim(), 10); - ownerPid = Number.isFinite(parsed) && parsed > 0 ? parsed : null; - } catch { - ownerPid = null; + lock = fs.lstatSync(directory, { bigint: true }); + } catch (statError) { + if (isErrnoException(statError) && statError.code === "ENOENT") continue; + throw statError; } - const ownerAlive = ownerPid !== null ? isProcessAlive(ownerPid) : false; - const processStartMs = ownerPid !== null && ownerAlive ? readProcessStartMs(ownerPid) : null; + if (!lock.isDirectory() || lock.isSymbolicLink()) + throw new Error(`Registry lock '${directory}' is not a real directory`); + const pid = ownerPid(paths.owner); + const owner = identity(paths.owner); + const live = pid !== null && alive(pid); + const record = pid === null ? null : readProcessRecord(paths.processStart, pid); + const processFile = identity(paths.processStart); const decision = classifyExistingLock({ - ownerPid, - ownerAlive, - processStartMs, - lockMtimeMs: lockStat.mtimeMs, - nowMs: Date.now(), - staleMs: LOCK_STALE_MS, + ownerAlive: live, ownerPid: pid, + ownerStatus: pid === null ? "ordinary" : status(pid, live, record, readIdentity), + lockMtimeMs: Number(lock.mtimeMs), nowMs: now(), staleMs: LOCK_STALE_MS, }); - if (decision === "break") { - // Only break the lock if it is provably the same one we classified. - // Re-stat LOCK_DIR and require the inode + mtime to be unchanged (a - // replacement lock is a fresh mkdir, hence a new inode) and, when the - // owner pid was readable, that it still matches. Any stat/read failure - // means the identity cannot be proven, so the lock is left alone rather - // than risk clobbering an in-flight replacement that exists as LOCK_DIR - // before its owner file has been written. - let stillSameLock = false; - try { - const currentStat = fs.statSync(LOCK_DIR); - stillSameLock = - currentStat.ino === lockStat.ino && currentStat.mtimeMs === lockStat.mtimeMs; - if (stillSameLock && ownerPid !== null) { - const recheck = Number.parseInt(fs.readFileSync(LOCK_OWNER, "utf-8").trim(), 10); - stillSameLock = recheck === ownerPid; - } - } catch { - stillSameLock = false; - } - if (stillSameLock) { - fs.rmSync(LOCK_DIR, { recursive: true, force: true }); - continue; - } - } - Atomics.wait(sleepBuf, 0, 0, LOCK_RETRY_MS); + if (decision === "break" && tryRemove({ device: lock.dev, inode: lock.ino, paths }, pid, owner, record, processFile, lock.mtimeNs)) + continue; + wait(); continue; } - const ownerTmp = `${LOCK_OWNER}.tmp.${process.pid}`; + const lock = fs.lstatSync(directory, { bigint: true }); + const generation: Generation = { device: lock.dev, inode: lock.ino, paths }; + const temporary = `${paths.owner}.tmp.${String(process.pid)}.${crypto.randomUUID()}`; + let temporaryFile: Identity | null = null; + let ownerFile: Identity | null = null; + let record: string | null = null; + let processFile: Identity | null = null; try { - fs.writeFileSync(ownerTmp, String(process.pid), { mode: 0o600 }); - fs.renameSync(ownerTmp, LOCK_OWNER); - } catch (ownerError) { - try { - fs.unlinkSync(ownerTmp); - } catch { - /* best effort */ - } - try { - fs.unlinkSync(LOCK_OWNER); - } catch { - /* best effort */ + fs.writeFileSync(temporary, String(process.pid), { flag: "wx", mode: 0o600 }); + temporaryFile = identity(temporary); + fs.renameSync(temporary, paths.owner); + ownerFile = temporaryFile; + temporaryFile = null; + if (initialIdentity !== null) { + record = `${String(process.pid)} ${initialIdentity}`; + fs.writeFileSync(paths.processStart, `${record}\n`, { flag: "wx", mode: 0o600 }); + processFile = identity(paths.processStart); } + const invalidProcessFile = record !== null && + (processFile === null || !sameIdentity(identity(paths.processStart), processFile) || readProcessRecord(paths.processStart, process.pid) !== record); + if (ownerFile === null || !sameGeneration(generation) || !sameIdentity(identity(paths.owner), ownerFile) || ownerPid(paths.owner) !== process.pid || invalidProcessFile || (exact && readIdentity(process.pid) !== initialIdentity)) + throw new Error("Registry lock identity changed during acquisition"); + } catch (error) { + const files: [string, Identity][] = []; + if (temporaryFile) files.push([temporary, temporaryFile]); + if (ownerFile) files.push([paths.owner, ownerFile]); + if (processFile) files.push([paths.processStart, processFile]); try { - fs.rmdirSync(LOCK_DIR); + removeGeneration(generation, files, () => true); } catch { - /* best effort */ + // An ambiguous detached generation is intentionally preserved. } - throw ownerError; + throw error; } - return; + return { ...generation, ownerFile, processFile, processRecord: record }; } - throw new Error(`Failed to acquire lock on ${REGISTRY_FILE} after ${LOCK_MAX_RETRIES} retries`); + throw new Error(`Failed to acquire lock on ${directory} after ${String(retries)} retries`); } -export function releaseLock(): void { - try { - fs.unlinkSync(LOCK_OWNER); - } catch (error) { - if (!isErrnoException(error) || error.code !== "ENOENT") { - throw error; - } - } +function release(acquired: Acquired): void { + const files: [string, Identity][] = [[acquired.paths.owner, acquired.ownerFile]]; + if (acquired.processFile) files.push([acquired.paths.processStart, acquired.processFile]); try { - fs.rmSync(LOCK_DIR, { recursive: true, force: true }); - } catch (error) { - if (!isErrnoException(error) || error.code !== "ENOENT") { - throw error; - } + removeGeneration(acquired, files, (paths) => ownerPid(paths.owner) === process.pid && readProcessRecord(paths.processStart, process.pid) === acquired.processRecord); + } catch { + throw new Error(`Registry lock '${acquired.paths.directory}' changed ownership`); } } -export function withLock(fn: () => T): T { +let compatibility: Acquired | null = null; + +export function acquireLock(): void { + ensureConfigDir(path.dirname(REGISTRY_FILE)); + if (compatibility) throw new Error(`Registry lock '${LOCK_DIR}' is already acquired`); + compatibility = acquire(LOCK_DIR, false, {}); +} +export function releaseLock(): void { + if (!compatibility) throw new Error(`Registry lock '${LOCK_DIR}' is not acquired`); + const lock = compatibility; + compatibility = null; + release(lock); +} +export function withLock(operation: () => T): T { acquireLock(); try { - return fn(); + return operation(); } finally { releaseLock(); } } +function withAcquired(directory: string, exact: boolean, operation: () => T, deps: RegistryLockDeps): T { + const lock = acquire(directory, exact, deps); + try { + return operation(); + } finally { + release(lock); + } +} +export function withRegistryLockAt(registryFile: string, operation: () => T, deps: RegistryLockDeps = {}): T { + return withAcquired(`${registryFile}.lock`, false, operation, deps); +} +export function withProcessBoundRegistryLockAt(registryFile: string, operation: () => T, deps: RegistryLockDeps = {}): T { + return withAcquired(`${registryFile}.lock`, true, operation, deps); +} +export function acquireProcessBoundLockAt(lockDirectory: string, deps: RegistryLockDeps = {}): ProcessBoundLockHandle { + const handle = {}; + handles.set(handle, acquire(lockDirectory, process.platform === "linux", deps)); + return handle; +} +export function releaseProcessBoundLock(handle: ProcessBoundLockHandle): void { + const lock = handles.get(handle); + if (!lock) throw new Error("Process-bound registry lock handle is inactive"); + handles.delete(handle); + release(lock); +} diff --git a/test/cli/onboard-compatibility.test.ts b/test/cli/onboard-compatibility.test.ts index e342a505927..bfabcaaafa7 100644 --- a/test/cli/onboard-compatibility.test.ts +++ b/test/cli/onboard-compatibility.test.ts @@ -295,7 +295,7 @@ describe("CLI onboard compatibility", () => { const localBin = path.join(home, "bin"); const nemoclawDir = path.join(home, ".nemoclaw"); fs.mkdirSync(localBin, { recursive: true }); - fs.mkdirSync(nemoclawDir, { recursive: true }); + fs.mkdirSync(nemoclawDir, { recursive: true, mode: 0o700 }); writeOpenShellVersionStub(localBin); writeIncompleteResumeSession(nemoclawDir); diff --git a/test/e2e/live/podman-portable-uninstall.test.ts b/test/e2e/live/podman-portable-uninstall.test.ts new file mode 100644 index 00000000000..57927396fb8 --- /dev/null +++ b/test/e2e/live/podman-portable-uninstall.test.ts @@ -0,0 +1,456 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { + capturePodmanSocketAuthority, + createPodmanContainerEngine, +} from "../../../src/lib/adapters/podman"; +import { portableDemoReceiptPath } from "../../../src/lib/onboard/experimental/portable-runtime-receipt-readiness"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { REPO_ROOT } from "../fixtures/paths.ts"; +import { + cleanupPodmanLifecycle, + executableOnPath, + inspectContainer, + runCommand, + SOCKET_PATH, +} from "./podman-cpu-lifecycle-helpers.ts"; + +const BASE_IMAGE = + "ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:3265d482f67c9d81ee3a59b0bbad5eb5ea6c705fea81ece8ae888ed12794f7f1"; +const UNRELATED_IMAGE = + "docker.io/library/ubuntu@sha256:019e8eb29a85e74d64925745884f2ec79aa27e3feab36353d24656f4d6b89467"; +const UNUSED_IMAGE = + "docker.io/library/busybox@sha256:73aaf090f3d85aa34ee199857f03fa3a95c8ede2ffd4cc2cdb5b94e566b11662"; +const SANDBOX_NAME = "podman-uninstall"; +const UNRELATED_NAME = "nemoclaw-uninstall-unrelated"; +const REGISTRY_NAME = "nemoclaw-portable-registry"; +const UNINSTALL_ARGS = [ + "uninstall", + "--all-gateway-ports", + "--delete-models", + "--destroy-user-data", + "--yes", +] as const; +const E2E_PHASES = [ + "pin the current-user Podman authority", + "authenticate the workflow-owned pinned OpenShell gateway", + "create receipt-owned and unrelated resources", + "project portable selectors", + "run the exact full uninstall command", + "verify resource and lifecycle retirement", + "restart the user Podman socket", + "begin portable reinstall runtime selection", +] as const; + +function sandboxCreateArgs(): string[] { + return [ + "sandbox", + "create", + "-g", + "nemoclaw", + "--name", + SANDBOX_NAME, + "--from", + BASE_IMAGE, + "--policy", + path.join(REPO_ROOT, "test/e2e/live/podman-cpu-lifecycle-policy.yaml"), + "--no-tty", + "--", + "/bin/sh", + "-lc", + "true", + ]; +} + +function unrelatedCreateArgs(): string[] { + return [ + "create", + "--name", + UNRELATED_NAME, + "--label", + "com.nvidia.nemoclaw.e2e-unrelated=1", + UNRELATED_IMAGE, + "sleep", + "infinity", + ]; +} + +function writePortableUninstallSummary(artifactDir: string | undefined, uid: number): void { + const writeSummary = artifactDir + ? () => { + fs.mkdirSync(artifactDir, { recursive: true, mode: 0o700 }); + fs.writeFileSync( + path.join(artifactDir, "portable-uninstall-summary.json"), + `${JSON.stringify( + { + schemaVersion: 1, + command: ["nemoclaw", ...UNINSTALL_ARGS], + dockerUnavailable: true, + rootlessUid: uid, + sandboxRemoved: true, + unrelatedContainerPreserved: true, + registryRemoved: true, + sandboxImagePreserved: true, + unrelatedImagesPreserved: true, + receiptRetired: true, + configRetired: true, + exactSelectorsCleared: true, + userSelectorsPreserved: true, + socketRestarted: true, + reinstallRuntimeSelectionBegan: true, + }, + null, + 2, + )}\n`, + { mode: 0o600 }, + ); + } + : () => undefined; + writeSummary(); +} + +test( + "runs full portable uninstall before a clean socket restart and reinstall start (#9189)", + { meta: { e2ePhases: E2E_PHASES }, timeout: 300_000 }, + async ({ progress, shellProbe }) => { + progress.phase("pin the current-user Podman authority"); + expect(process.platform).toBe("linux"); + const uid = process.getuid?.() ?? -1; + expect(uid).toBeGreaterThan(0); + expect(SOCKET_PATH).toBe(path.join("/run/user", String(uid), "podman", "podman.sock")); + expect(fs.existsSync("/var/run/docker.sock")).toBe(false); + const socketAuthority = capturePodmanSocketAuthority(SOCKET_PATH); + const engine = createPodmanContainerEngine({ operation: "sandbox-lifecycle", socketAuthority }); + expect(engine.capture(["version", "--format", "json"]).status).toBe(0); + const nemoclawBin = executableOnPath("nemoclaw"); + const openshellBin = executableOnPath("openshell"); + + const homeDir = os.homedir(); + const stateDir = path.join(homeDir, ".nemoclaw"); + const configDir = path.join(homeDir, ".config", "nemoclaw"); + const registryFile = path.join(stateDir, "sandboxes.json"); + expect(fs.existsSync(stateDir)).toBe(false); + expect(fs.existsSync(configDir)).toBe(false); + const gatewayRoot = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-podman-uninstall-")); + const cliEnv: NodeJS.ProcessEnv = { + ...process.env, + XDG_CONFIG_HOME: path.join(gatewayRoot, "cli-config"), + }; + const previousPortableProfile = process.env.NEMOCLAW_EXPERIMENTAL_PROFILE; + const createdContainerIds: string[] = []; + const imageWasPresent = new Map( + [UNRELATED_IMAGE, UNUSED_IMAGE].map((image) => [ + image, + engine.capture(["image", "exists", image]).status === 0, + ]), + ); + let releaseProjectedSelectors = async (): Promise => undefined; + try { + progress.phase("authenticate the workflow-owned pinned OpenShell gateway"); + process.env.NEMOCLAW_EXPERIMENTAL_PROFILE = "portable"; + expect(cliEnv.NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR).toBeTruthy(); + expect(cliEnv.OPENSHELL_LOCAL_TLS_DIR).toBeTruthy(); + await runCommand( + shellProbe, + openshellBin, + ["gateway", "add", "https://127.0.0.1:8080", "--local", "--name", "nemoclaw"], + { artifactName: "podman-uninstall-add-gateway-nemoclaw", env: cliEnv }, + ); + const gatewayInfo = JSON.parse( + await runCommand( + shellProbe, + openshellBin, + ["gateway", "info", "-g", "nemoclaw", "-o", "json"], + { artifactName: "podman-uninstall-gateway-info", env: cliEnv, timeoutMs: 10_000 }, + ), + ) as { compute_drivers?: Array<{ name: string }>; status?: string }; + expect(gatewayInfo).toMatchObject({ status: "healthy" }); + expect(gatewayInfo.compute_drivers).toContainEqual( + expect.objectContaining({ name: "podman" }), + ); + + progress.phase("create receipt-owned and unrelated resources"); + for (const image of imageWasPresent.keys()) { + expect(engine.capture(["pull", image]).status).toBe(0); + } + await runCommand(shellProbe, openshellBin, sandboxCreateArgs(), { + artifactName: "podman-uninstall-create-sandbox", + env: cliEnv, + timeoutMs: 240_000, + }); + const sandboxInspection = inspectContainer(engine, SANDBOX_NAME); + const sandboxContainerId = sandboxInspection.Id; + const sandboxId = sandboxInspection.Config.Labels["openshell.ai/sandbox-id"]; + expect(sandboxId).toMatch(/^[A-Za-z0-9._:-]{1,256}$/u); + createdContainerIds.push(sandboxContainerId); + + const unrelatedCreate = engine.capture(unrelatedCreateArgs()); + expect(unrelatedCreate.status).toBe(0); + const unrelatedContainerId = unrelatedCreate.stdout.trim(); + expect(unrelatedContainerId).toMatch(/^[a-f0-9]{64}$/u); + createdContainerIds.push(unrelatedContainerId); + + const registryCreate = engine.capture([ + "create", + "--name", + REGISTRY_NAME, + "--label", + "com.nvidia.nemoclaw.portable=1", + BASE_IMAGE, + "sleep", + "infinity", + ]); + expect(registryCreate.status).toBe(0); + const registryContainerId = registryCreate.stdout.trim(); + expect(registryContainerId).toMatch(/^[a-f0-9]{64}$/u); + createdContainerIds.push(registryContainerId); + for (const containerId of createdContainerIds) { + expect(engine.capture(["start", containerId]).status).toBe(0); + } + + const runtimeAuthority = { + schemaVersion: 1, + kind: "podman", + ownership: "current-user", + uid, + homeDir, + configHome: path.join(homeDir, ".config"), + runtimeDir: path.join("/run/user", String(uid)), + socketPath: SOCKET_PATH, + } as const; + const receiptFile = portableDemoReceiptPath(SANDBOX_NAME, stateDir); + fs.mkdirSync(path.dirname(receiptFile), { recursive: true, mode: 0o700 }); + fs.writeFileSync( + receiptFile, + `${JSON.stringify( + { + schemaVersion: 4, + sandboxName: SANDBOX_NAME, + sandboxId, + containerId: sandboxContainerId, + dashboardPort: 18789, + registryGeneration: sandboxContainerId, + runtimeAuthority, + }, + null, + 2, + )}\n`, + { mode: 0o600 }, + ); + fs.writeFileSync( + registryFile, + `${JSON.stringify( + { + defaultSandbox: SANDBOX_NAME, + sandboxes: { + [SANDBOX_NAME]: { + name: SANDBOX_NAME, + agent: "openclaw", + gatewayName: "nemoclaw", + gatewayPort: 8080, + openshellDriver: "docker", + lifecycleGeneration: sandboxContainerId, + }, + }, + }, + null, + 2, + )}\n`, + { mode: 0o600 }, + ); + const expectedContainersConf = path.join( + runtimeAuthority.configHome, + "nemoclaw", + "portable", + "containers.conf", + ); + fs.mkdirSync(path.dirname(expectedContainersConf), { recursive: true, mode: 0o700 }); + fs.writeFileSync(expectedContainersConf, '[network]\nfirewall_driver = "iptables"\n', { + mode: 0o600, + }); + + progress.phase("project portable selectors"); + await runCommand( + shellProbe, + "systemctl", + [ + "--user", + "set-environment", + `CONTAINERS_CONF=${expectedContainersConf}`, + "NETAVARK_FW=iptables", + "CONTAINER_HOST=ssh://user-managed.invalid", + "CONTAINER_CONNECTION=user-managed", + `CONTAINER_SSHKEY=${path.join(homeDir, ".ssh", "user-managed")}`, + ], + { artifactName: "podman-uninstall-project-selectors" }, + ); + releaseProjectedSelectors = async () => { + await runCommand( + shellProbe, + "systemctl", + [ + "--user", + "unset-environment", + "CONTAINERS_CONF", + "NETAVARK_FW", + "CONTAINER_HOST", + "CONTAINER_CONNECTION", + "CONTAINER_SSHKEY", + ], + { allowFailure: true, artifactName: "podman-uninstall-clear-selectors" }, + ); + }; + + progress.phase("run the exact full uninstall command"); + await runCommand(shellProbe, nemoclawBin, UNINSTALL_ARGS, { + artifactName: "podman-exact-full-uninstall", + env: cliEnv, + timeoutMs: 240_000, + }); + + progress.phase("verify resource and lifecycle retirement"); + const postUninstallEngine = createPodmanContainerEngine({ + operation: "sandbox-lifecycle", + socketAuthority: capturePodmanSocketAuthority(SOCKET_PATH), + }); + expect(postUninstallEngine.capture(["inspect", sandboxContainerId]).status).not.toBe(0); + expect(postUninstallEngine.capture(["inspect", registryContainerId]).status).not.toBe(0); + expect(postUninstallEngine.capture(["inspect", unrelatedContainerId]).status).toBe(0); + expect(postUninstallEngine.capture(["image", "exists", BASE_IMAGE]).status).toBe(0); + expect(postUninstallEngine.capture(["image", "exists", UNRELATED_IMAGE]).status).toBe(0); + expect(postUninstallEngine.capture(["image", "exists", UNUSED_IMAGE]).status).toBe(0); + expect(fs.existsSync(receiptFile)).toBe(false); + const retirementRecord = path.join(stateDir, "portable-uninstall-retirement.json"); + expect(fs.existsSync(retirementRecord)).toBe(true); + expect(fs.statSync(stateDir).mode & 0o777).toBe(0o700); + expect(fs.statSync(retirementRecord).mode & 0o777).toBe(0o600); + const residualFiles = fs + .readdirSync(stateDir, { recursive: true, withFileTypes: true }) + .filter((entry) => entry.isFile() || entry.isSymbolicLink()) + .map((entry) => path.join(entry.parentPath, entry.name)); + expect(residualFiles).toEqual([retirementRecord]); + expect(fs.existsSync(configDir)).toBe(false); + const managerEnvironment = await runCommand( + shellProbe, + "systemctl", + ["--user", "show-environment"], + { artifactName: "podman-uninstall-selectors-after-full-command" }, + ); + expect(managerEnvironment).not.toContain("CONTAINERS_CONF="); + expect(managerEnvironment).not.toContain("NETAVARK_FW="); + expect(managerEnvironment).toContain("CONTAINER_HOST=ssh://user-managed.invalid"); + expect(managerEnvironment).toContain("CONTAINER_CONNECTION=user-managed"); + expect(managerEnvironment).toContain( + `CONTAINER_SSHKEY=${path.join(homeDir, ".ssh", "user-managed")}`, + ); + + await runCommand( + shellProbe, + "systemctl", + [ + "--user", + "unset-environment", + "CONTAINER_HOST", + "CONTAINER_CONNECTION", + "CONTAINER_SSHKEY", + ], + { artifactName: "podman-uninstall-release-test-selectors" }, + ); + releaseProjectedSelectors = async (): Promise => undefined; + + progress.phase("restart the user Podman socket"); + await runCommand( + shellProbe, + "bash", + [ + "-ceu", + ` +socket_path="$1" +systemctl --user stop podman.service +systemctl --user reset-failed podman.service podman.socket +systemctl --user restart podman.socket +for attempt in $(seq 1 30); do + if podman --url "unix://$socket_path" version --format json >/dev/null; then + break + fi + test "$attempt" -lt 30 + sleep 1 +done +systemctl --user is-active --quiet podman.socket +systemctl --user show podman.socket --property=ActiveState --property=Result +`, + "podman-uninstall-socket-restart", + SOCKET_PATH, + ], + { artifactName: "podman-uninstall-restart-user-socket", timeoutMs: 60_000 }, + ); + + progress.phase("begin portable reinstall runtime selection"); + await runCommand( + shellProbe, + "env", + [ + "-u", + "CONTAINERS_CONF", + "-u", + "NETAVARK_FW", + "-u", + "CONTAINER_HOST", + "-u", + "CONTAINER_CONNECTION", + "-u", + "CONTAINER_SSHKEY", + "bash", + "-ceu", + ` +source "$1" +export NEMOCLAW_EXPERIMENTAL_PROFILE=portable +prepare_portable_experimental_runtime_override +test "$DOCKER_HOST" = "unix://$2" +`, + "podman-uninstall-reinstall-start", + path.join(REPO_ROOT, "scripts", "install.sh"), + SOCKET_PATH, + ], + { artifactName: "podman-uninstall-begin-reinstall", timeoutMs: 60_000 }, + ); + + writePortableUninstallSummary(process.env.E2E_ARTIFACT_DIR, uid); + } finally { + await releaseProjectedSelectors(); + try { + const cleanupAuthority = capturePodmanSocketAuthority(SOCKET_PATH); + const cleanupEngine = createPodmanContainerEngine({ + operation: "sandbox-lifecycle", + socketAuthority: cleanupAuthority, + }); + for (const containerId of createdContainerIds.reverse()) { + cleanupEngine.capture(["rm", "--force", containerId]); + } + for (const [image, existed] of imageWasPresent) { + existed || cleanupEngine.capture(["image", "rm", image]); + } + } catch { + // The workflow's always-run cleanup owns any resources left after a socket failure. + } + fs.rmSync(stateDir, { recursive: true, force: true }); + fs.rmSync(configDir, { recursive: true, force: true }); + await cleanupPodmanLifecycle({ + cliEnv, + completed: true, + createdSandboxes: [], + engine, + gateway: null, + openshellBin, + previousPortableProfile, + root: gatewayRoot, + shellProbe, + }); + } + }, +); diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index bbfec6b1fad..5d8d2cc7352 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -82,6 +82,14 @@ "test/runtime-provider-source-shape.test.ts" ] }, + { + "live": "test/e2e/live/podman-portable-uninstall.test.ts", + "fast": [ + "src/lib/actions/uninstall/portable-runtime-cleanup.test.ts", + "src/lib/actions/uninstall/run-plan-portable-runtime.test.ts", + "test/e2e/support/podman-cpu-proof-workflow.test.ts" + ] + }, { "live": "test/e2e/live/hermes-gpu-startup.test.ts", "fast": [ @@ -177,9 +185,7 @@ }, { "live": "test/e2e/live/openshell-gateway-auth-source-contract.test.ts", - "fast": [ - "test/e2e/support/openshell-gateway-auth-source-contract-helpers.test.ts" - ] + "fast": ["test/e2e/support/openshell-gateway-auth-source-contract-helpers.test.ts"] }, { "live": "test/e2e/live/openshell-credential-generation-window.test.ts", @@ -203,9 +209,7 @@ }, { "live": "test/e2e/live/sandbox-rlimits-connect.test.ts", - "fast": [ - "test/e2e/support/workflow-plan.test.ts" - ] + "fast": ["test/e2e/support/workflow-plan.test.ts"] }, { "live": "test/e2e/live/sandbox-operations.test.ts", @@ -240,9 +244,7 @@ }, { "live": "test/e2e/live/onboard-policy-preset-sequencing.test.ts", - "fast": [ - "test/e2e/support/onboard-interactive-pty.test.ts" - ] + "fast": ["test/e2e/support/onboard-interactive-pty.test.ts"] }, { "live": "test/e2e/live/snapshot-commands.test.ts", diff --git a/test/e2e/support/podman-cpu-proof-workflow.test.ts b/test/e2e/support/podman-cpu-proof-workflow.test.ts index b9f972f9214..0ef403c0cf3 100644 --- a/test/e2e/support/podman-cpu-proof-workflow.test.ts +++ b/test/e2e/support/podman-cpu-proof-workflow.test.ts @@ -372,6 +372,15 @@ describe("native Podman CPU proof workflow", () => { "src/lib/onboard/docker-driver-gateway-*.ts", "src/lib/onboard/managed-bootstrap/podman-*.ts", "src/lib/onboard/experimental/portable-demo-lifecycle.ts", + "src/lib/onboard/experimental/portable-runtime-*.ts", + "src/lib/onboard/portable-retirement-authority.ts", + "src/lib/onboard.ts", + "src/lib/actions/uninstall/portable-runtime-cleanup.ts", + "src/lib/actions/uninstall/all-gateway-ports.ts", + "src/lib/actions/uninstall/run-plan.ts", + "src/commands/internal/uninstall/run-plan.ts", + "src/lib/state/portable-uninstall-retirement.ts", + "src/lib/state/registry/lock.ts", "src/lib/onboard/runtime-provider/container-state-mutation.ts", "src/lib/onboard/runtime-provider/docker-state-mutation.ts", "src/lib/onboard/experimental/portable-cpu-delegation-preflight*.ts", @@ -380,6 +389,7 @@ describe("native Podman CPU proof workflow", () => { "test/e2e/live/podman-cpu-lifecycle-artifacts.ts", "test/e2e/live/podman-cpu-lifecycle-helpers.ts", "test/e2e/live/podman-cpu-lifecycle-policy.yaml", + "test/e2e/live/podman-portable-uninstall.test.ts", "test/e2e/registry/native-runtime-qualification.ts", "test/e2e/live/portable-cpu-delegation-proof.test.ts", ]), @@ -394,9 +404,10 @@ describe("native Podman CPU proof workflow", () => { expect(namedStep("Checkout").with).toMatchObject({ ref: "${{ github.event.pull_request.head.sha }}", }); - expect(namedStep("Build shared sandbox-name contract").run).toBe( - "npm run build:policy-boundary", - ); + const buildCandidate = namedStep("Build and link candidate CLI").run ?? ""; + expect(buildCandidate).toContain("npm run build:cli"); + expect(buildCandidate).toContain("npm link --ignore-scripts"); + expect(buildCandidate).toContain("nemoclaw --version"); const installPodman = namedStep("Install Podman 5 runtime").run ?? ""; expect(installPodman).toContain("apt-get install --yes"); expect(installPodman).toContain("passt"); @@ -932,9 +943,22 @@ describe("native Podman CPU proof workflow", () => { .steps?.map((step) => step.run ?? "") .join("\n"); - expect(proof.run).toBe( - "npx vitest run --project e2e-live test/e2e/live/podman-cpu-lifecycle.test.ts", + expect(proof.run).toContain( + "npx vitest run --project e2e-live \\\n test/e2e/live/podman-cpu-lifecycle.test.ts \\", ); + expect(proof.run).toContain("test/e2e/live/podman-portable-uninstall.test.ts"); + const uninstallSource = readRepoText("test/e2e/live/podman-portable-uninstall.test.ts"); + expect(uninstallSource).toContain('executableOnPath("nemoclaw")'); + expect(uninstallSource).toContain("NEMOCLAW_OPENSHELL_GATEWAY_STATE_DIR"); + expect(uninstallSource).toContain("OPENSHELL_LOCAL_TLS_DIR"); + expect(uninstallSource).toContain('["gateway", "info", "-g", "nemoclaw", "-o", "json"]'); + expect(uninstallSource).toContain('gatewayName: "nemoclaw"'); + expect(uninstallSource).toContain('"--all-gateway-ports"'); + expect(uninstallSource).toContain('"--delete-models"'); + expect(uninstallSource).toContain('"--destroy-user-data"'); + expect(uninstallSource).toContain('"--yes"'); + expect(uninstallSource).toContain("systemctl --user restart podman.socket"); + expect(uninstallSource).toContain("prepare_portable_experimental_runtime_override"); const liveSource = readRepoText("test/e2e/live/podman-cpu-lifecycle.test.ts"); const authorityIndex = liveSource.indexOf("expect(candidateAuthority())"); const enginesIndex = liveSource.indexOf("let runtimeEngines = engines()"); @@ -942,6 +966,7 @@ describe("native Podman CPU proof workflow", () => { expect(enginesIndex).toBeGreaterThanOrEqual(0); expect(authorityIndex).toBeLessThan(enginesIndex); expect(scripts).not.toContain("podman create"); + expect(scripts).toMatch(/onboard\.js"\)\)\.default[\s\S]*stopHostGatewayProcesses/u); expect(scripts).not.toContain("openshell-sandbox-$sandbox_name"); expect(scripts).not.toContain("openshell.sandbox-name"); expect(diagnostics.if).toBe("failure()"); @@ -961,5 +986,7 @@ describe("native Podman CPU proof workflow", () => { expect(cleanup.run).toContain('podman --url "$endpoint" volume rm --force'); expect(cleanup.run).toContain('podman --url "$endpoint" secret rm'); expect(cleanup.run).toContain('podman --url "$endpoint" network rm openshell-docker'); + const stopGateway = namedStep("Stop the exact portable-retirement proof gateway"); + expect(stopGateway.env?.E2E_PORTABLE_GATEWAY_STOP_SCOPE).toBe("full"); }); }); diff --git a/test/registry.test.ts b/test/registry.test.ts index 3ff892d2188..64e9133e967 100644 --- a/test/registry.test.ts +++ b/test/registry.test.ts @@ -1293,7 +1293,7 @@ describe("advisory file locking", () => { return origMkdir.apply(fs, args); }; fs.rmSync = (...args) => { - if (args[0] === lockDir) rmCalls.push(args[0]); + if (String(args[0]).startsWith(`${lockDir}.quarantine.`)) rmCalls.push(args[0]); return origRm.apply(fs, args); }; try { diff --git a/test/uninstall.test.ts b/test/uninstall.test.ts index 6206319168e..b59ad213e1a 100644 --- a/test/uninstall.test.ts +++ b/test/uninstall.test.ts @@ -15,6 +15,8 @@ import { NEMOCLAW_OPENSHELL_SANDBOX_NAMESPACE_ENV, } from "../src/lib/onboard/docker-driver-gateway-config"; import { writeDockerDriverGatewayRuntimeMarkerForStateDir } from "../src/lib/onboard/docker-driver-gateway-runtime-marker"; +import { deriveCheckpointFromSession } from "../src/lib/state/onboard-checkpoint-migrate"; +import { createSession } from "../src/lib/state/onboard-session"; const UNINSTALL_SCRIPT = path.join(import.meta.dirname, "..", "uninstall.sh"); @@ -39,7 +41,7 @@ exit 0 } function seedPreservedState(tmp: string): string { - const stateDir = path.join(tmp, ".nemoclaw"); + const stateDir = seedCompletedDefaultAuthority(tmp); fs.mkdirSync(path.join(stateDir, "rebuild-backups", "sb1", "20260101"), { recursive: true }); fs.writeFileSync( path.join(stateDir, "rebuild-backups", "sb1", "20260101", "manifest.json"), @@ -47,9 +49,57 @@ exit 0 ); fs.mkdirSync(path.join(stateDir, "backups", "20260320-120000"), { recursive: true }); fs.writeFileSync(path.join(stateDir, "backups", "20260320-120000", "USER.md"), "hello"); + return stateDir; + } + + function seedCompletedDefaultAuthority(tmp: string): string { + const stateDir = path.join(tmp, ".nemoclaw"); + const sandboxName = "ordinary-authority"; + const gateway = { + gatewayName: "nemoclaw", + gatewayPort: 8080, + mode: "nemoclaw-managed" as const, + source: "standalone" as const, + endpoint: null, + stateDir: null, + supervisor: null, + requiredCapabilities: [], + }; + fs.mkdirSync(stateDir, { recursive: true, mode: 0o700 }); + fs.chmodSync(stateDir, 0o700); + const session = createSession({ + agent: "openclaw", + sandboxName, + metadata: { gatewayName: gateway.gatewayName, fromDockerfile: null }, + }); + session.status = "complete"; + session.resumable = false; + session.machine = { ...session.machine, state: "complete", revision: 1 }; + session.checkpoint = { + ...deriveCheckpointFromSession(session, { profile: "default" }), + sandboxIdentity: { kind: "selected", value: { name: sandboxName, agent: "openclaw" } }, + gatewayAuthority: { kind: "selected", value: gateway }, + }; + fs.writeFileSync(path.join(stateDir, "onboard-session.json"), `${JSON.stringify(session)}\n`, { + mode: 0o600, + }); fs.writeFileSync( path.join(stateDir, "sandboxes.json"), - JSON.stringify({ defaultSandbox: null, sandboxes: {} }), + `${JSON.stringify({ + defaultSandbox: sandboxName, + sandboxes: { + [sandboxName]: { + name: sandboxName, + agent: null, + dashboardPort: null, + gatewayName: gateway.gatewayName, + gatewayPort: gateway.gatewayPort, + lifecycleGeneration: "ordinary-authority-generation", + openshellDriver: "docker", + }, + }, + })}\n`, + { mode: 0o600 }, ); return stateDir; } @@ -169,6 +219,7 @@ exit 0 it("skips the confirmation prompt and completes successfully for --yes", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-uninstall-yes-")); writeFakeTools(path.join(tmp, "bin")); + seedCompletedDefaultAuthority(tmp); try { const result = runUninstall(tmp, ["--yes"]); const output = `${result.stdout}${result.stderr}`; @@ -269,6 +320,7 @@ exit 0 it("uses NemoHermes branding for --yes when Hermes is active", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "nemohermes-uninstall-yes-")); writeFakeTools(path.join(tmp, "bin")); + seedCompletedDefaultAuthority(tmp); try { const result = runUninstall(tmp, ["--yes"], { NEMOCLAW_AGENT: "hermes" });