diff --git a/.github/workflows/podman-cpu-proof.yaml b/.github/workflows/podman-cpu-proof.yaml index 03800f6b361..dd45c85d2d0 100644 --- a/.github/workflows/podman-cpu-proof.yaml +++ b/.github/workflows/podman-cpu-proof.yaml @@ -14,17 +14,21 @@ on: - "src/lib/adapters/podman/**" - "src/lib/onboard/docker-driver-gateway-*.ts" - "src/lib/onboard/managed-bootstrap/podman-*.ts" + - "src/lib/onboard/experimental/portable-cpu-delegation-preflight*.ts" - "src/lib/onboard/experimental/portable-demo-lifecycle.ts" - "src/lib/onboard/runtime-provider/container-state-mutation.ts" - "src/lib/onboard/runtime-provider/docker-state-mutation.ts" + - "src/lib/onboard/experimental/portable-host-preparation*.ts" - "src/lib/onboard/runtime-provider/podman*.ts" - "scripts/install-openshell.sh" + - "scripts/checks/run-portable-cpu-delegation-proof.mts" - "test/e2e/live/podman-cpu-lifecycle-artifacts.ts" - "test/e2e/live/podman-cpu-lifecycle-helpers.ts" - "test/e2e/live/podman-cpu-lifecycle-policy.yaml" - "test/e2e/live/podman-cpu-lifecycle.test.ts" - "test/e2e/registry/native-runtime-qualification.ts" - "test/e2e/support/native-runtime-qualification.test.ts" + - "test/e2e/live/portable-cpu-delegation-proof.test.ts" - "src/lib/onboard/experimental/portable-demo-lifecycle.test.ts" - "test/e2e/support/podman-cpu-proof-workflow.test.ts" @@ -36,6 +40,67 @@ concurrency: cancel-in-progress: true jobs: + portable-cpu-delegation: + name: Portable CPU delegation admission on Ubuntu 22.04 + runs-on: ubuntu-22.04 + timeout-minutes: 15 + env: + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/portable-cpu-delegation + E2E_CPU_DELEGATION_USER: nemoclaw-e2e + E2E_TARGET_ID: portable-cpu-delegation + E2E_SOURCE_REVISION: ${{ github.event.pull_request.head.sha }} + NEMOCLAW_RUN_LIVE_E2E: "1" + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + ref: ${{ github.event.pull_request.head.sha }} + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 22.19.0 + cache: npm + + - name: Install locked test dependencies + run: npm ci --ignore-scripts + + - name: Build shared sandbox-name contract + run: npm run build:policy-boundary + + - name: Prepare system and app slice CPU settings without service delegation + shell: bash + run: node --experimental-strip-types scripts/checks/run-portable-cpu-delegation-proof.mts prepare + + - name: Verify missing delegation blocks portable configuration and service activation + shell: bash + run: node --experimental-strip-types scripts/checks/run-portable-cpu-delegation-proof.mts reject + + - name: Apply administrator delegation and prove admission + shell: bash + run: node --experimental-strip-types scripts/checks/run-portable-cpu-delegation-proof.mts admit + + - name: Capture CPU delegation failure diagnostics + if: failure() + shell: bash + run: node --experimental-strip-types scripts/checks/run-portable-cpu-delegation-proof.mts diagnostics + + - name: Restore the user manager boundary + if: always() + shell: bash + run: node --experimental-strip-types scripts/checks/run-portable-cpu-delegation-proof.mts cleanup + + - name: Upload CPU delegation evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: portable-cpu-delegation-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }} + path: e2e-artifacts/portable-cpu-delegation/ + include-hidden-files: false + if-no-files-found: error + retention-days: 14 + podman-cpu-lifecycle: name: Rootless Podman CPU lifecycle with Docker disabled runs-on: ubuntu-26.04 diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 522e617ace9..813eb4f7f74 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -3334,6 +3334,1276 @@ Then rerun portable onboarding: $$nemoclaw onboard --experimental-profile portable ``` +### Portable CPU Delegation Preflight Fails + +The portable experimental profile requires the current user's systemd hierarchy to expose the `cpu` cgroup controller to `app.slice`. NemoClaw checks this requirement before it writes portable configuration, activates services, starts the registry, builds an image, or creates a sandbox. The credential-free preflight reads only `cgroup.controllers` files under `/sys/fs/cgroup`. + +The preflight distinguishes a missing controller file from a read failure. Classify the file state before you select a recovery action: + +```bash +uid="$(id -u)" +user_slice="/sys/fs/cgroup/user.slice/user-${uid}.slice" +user_manager="/sys/fs/cgroup/user.slice/user-${uid}.slice/user@${uid}.service" +export LC_ALL=C + +for controllers in \ + /sys/fs/cgroup/cgroup.controllers \ + "${user_slice}/cgroup.controllers" \ + "${user_manager}/cgroup.controllers" \ + "${user_manager}/app.slice/cgroup.controllers" +do + if [ ! -e "$controllers" ]; then + printf '%s: missing\n' "$controllers" + elif [ ! -r "$controllers" ]; then + printf '%s: unreadable\n' "$controllers" + elif evidence="$(node - "$controllers" <<'NODE' +const fs = require("node:fs"); + +const controllers = process.argv[2]; +let descriptor; +try { + descriptor = fs.openSync(controllers, "r"); + const buffer = Buffer.alloc(4097); + let bytesRead = 0; + while (bytesRead < buffer.length) { + const count = fs.readSync( + descriptor, + buffer, + bytesRead, + buffer.length - bytesRead, + null, + ); + if (count === 0) break; + bytesRead += count; + } + + const content = buffer.subarray(0, bytesRead); + if (content.length > 4096) { + process.exitCode = 2; + } else { + const body = content.at(-1) === 0x0a ? content.subarray(0, -1) : content; + const text = body.toString("utf8"); + const names = text === "" ? [] : text.split(" "); + if ( + body.includes(0x0a) || + new Set(names).size !== names.length || + names.some((name) => !/^[a-z][a-z0-9_]*$/u.test(name)) + ) { + process.exitCode = 2; + } else { + process.stdout.write(names.join(" ")); + } + } +} catch { + process.exitCode = 3; +} finally { + if (descriptor !== undefined) { + try { + fs.closeSync(descriptor); + } catch { + process.exitCode = 3; + } + } +} +NODE + )"; then + printf '%s: readable (%s)\n' "$controllers" "$evidence" + else + read_status="$?" + if [ "$read_status" = "2" ]; then + printf '%s: malformed\n' "$controllers" + else + printf '%s: read failed\n' "$controllers" + fi + fi +done +``` + +An access policy can prevent the current user from testing whether a path exists. For each `missing` or `read failed` result, ask an administrator to run `sudo stat -- `. If `stat` finds the file but the current-user read failed, classify the file as unreadable. If `stat` reports that the path does not exist, classify the file as missing. An empty readable file is valid evidence that exposes no controllers. A `malformed` result means the successful read did not contain the bounded, space-separated controller names supplied by the kernel. + +Do not print malformed bytes directly to a terminal. Ask an administrator to inspect only the reported path and its cgroups v2 mount: + +```bash +reported_path="" +uid="$(id -u)" +user_slice="/sys/fs/cgroup/user.slice/user-${uid}.slice" +user_manager="/sys/fs/cgroup/user.slice/user-${uid}.slice/user@${uid}.service" +reported_path_is_expected=0 +for expected_path in \ + /sys/fs/cgroup/cgroup.controllers \ + "${user_slice}/cgroup.controllers" \ + "${user_manager}/cgroup.controllers" \ + "${user_manager}/app.slice/cgroup.controllers" +do + if [ "$reported_path" = "$expected_path" ]; then + reported_path_is_expected=1 + fi +done +if [ "$reported_path_is_expected" != "1" ]; then + printf 'Refusing unexpected cgroup evidence path: %s\n' "$reported_path" >&2 + exit 1 +fi + +sudo stat -- "$reported_path" +sudo od -An -tx1 -N 256 -v -- "$reported_path" +findmnt --target "$reported_path" --output TARGET,FSTYPE,OPTIONS +``` + +The administrator must correct the cgroups v2 mount or kernel-provided evidence before you rerun the classification command. Do not change systemd delegation, stop or start the user manager, or reboot the host to repair malformed evidence. + +The error and file state identify the required recovery: + +| Reported condition and file state | Meaning | Recovery | +| --- | --- | --- | +| cgroups v2 is unavailable; the root file is missing | The host does not expose the cgroups v2 controller file at the expected mount. | Ask an administrator to boot or configure the Linux host with a cgroups v2 mount. | +| Any controller file exists but is unreadable, or its read fails | The current user cannot read the existing hierarchy boundary. | Stop before changing delegation. Ask an administrator to restore read and directory-traversal access or correct the AppArmor, SELinux, or other Linux Security Module (LSM) policy that denied the read. Then run the classification command again. | +| Any controller file is malformed | The file does not contain the bounded, space-separated controller names supplied by the kernel, so the hierarchy result is inconclusive. | Stop before changing delegation. Ask an administrator to inspect the cgroup filesystem and active security tooling for that exact file. Run the classification command again only after the file contains kernel-provided controller names. | +| The kernel hierarchy does not expose `cpu`; the root file is readable | The root `cgroup.controllers` file does not contain the `cpu` controller. | Ask an administrator to enable the `cpu` controller in the host's kernel cgroup hierarchy. | +| systemd did not expose `cpu`; the per-user-slice file is missing or does not contain `cpu` | The current user's `user-UID.slice` ancestor does not receive the `cpu` controller. | Apply all three named systemd drop-ins below. | +| systemd did not delegate `cpu`; the user-manager file is missing or does not contain `cpu` | The current user's manager does not receive the `cpu` controller. | Apply all three named systemd drop-ins below. | +| `cpu` is unavailable to `app.slice`; its file is missing | The current user's `app.slice` hierarchy does not exist for this boot. | Apply all three named systemd drop-ins below. Then use the stop, reload, and start sequence or reboot the host. | +| `cpu` is unavailable to `app.slice`; its file is readable but does not contain `cpu` | The user manager does not have a CPU resource setting that activates the delegated controller for `app.slice`. | Apply all three named systemd drop-ins below. Then use the stop, reload, and start sequence or reboot the host. | + +Do not use a boot, delegation, or service lifecycle action to correct an unreadable or malformed file. Those actions do not restore read access or valid kernel evidence. + +The systemd changes require administrator access. NemoClaw does not edit `/etc/systemd`, invoke `sudo`, remove the sandbox CPU limit, or continue with weaker resource isolation. + + + The `user@.service` template applies to every user manager on the host. The per-UID slice drop-in + applies to the affected user's ancestor, and the `app.slice` drop-in applies to every user manager + on the host. Applying or removing any of the three drop-ins requires the administrator to stop the + affected user's manager, reload systemd, and start the manager. The administrator can reboot the + host instead of running that sequence. Stopping the manager stops that user's systemd services, + including rootless Podman and other user services. Plan each interruption with the affected user + and host administrator. + + +Record the affected user's numeric ID: + +```bash +uid="$(id -u)" +printf 'Current user ID: %s\n' "$uid" +``` + +Use the three dedicated NemoClaw drop-in paths below. The service drop-in delegates `cpu` to user managers. The per-UID slice and `app.slice` drop-ins each request the kernel default CPU weight of `100`. Those explicit settings activate the controller at both slice boundaries. If any file exists, stop and ask the administrator to inspect its ownership and content. Do not replace any file. + +```bash +uid="$(id -u)" +delegation_drop_in="/etc/systemd/system/user@.service.d/90-nemoclaw-cpu-delegation.conf" +app_slice_drop_in="/etc/systemd/user/app.slice.d/90-nemoclaw-cpu-controller.conf" +user_slice_drop_in="/etc/systemd/system/user-${uid}.slice.d/90-nemoclaw-cpu-controller.conf" +delegation_drop_in_dir="$(dirname "$delegation_drop_in")" +app_slice_drop_in_dir="$(dirname "$app_slice_drop_in")" +user_slice_drop_in_dir="$(dirname "$user_slice_drop_in")" + +inspect_cpu_controller_path() { + sudo sh -c ' + if [ -L "$1" ]; then printf "symlink\n" + elif [ -f "$1" ]; then printf "file\n" + elif [ -d "$1" ]; then printf "directory\n" + elif [ -e "$1" ]; then printf "other\n" + else printf "absent\n" + fi + ' sh "$1" +} + +for drop_in in "$delegation_drop_in" "$app_slice_drop_in" "$user_slice_drop_in"; do + if ! path_kind="$(inspect_cpu_controller_path "$drop_in")"; then + printf 'CPU controller drop-in inspection failed: %s\n' "$drop_in" >&2 + exit 1 + fi + if [ "$path_kind" != "absent" ]; then + printf 'Refusing to replace existing file: %s\n' "$drop_in" >&2 + exit 1 + fi +done + +ensure_drop_in_dir() { + directory_name="$1" + drop_in_dir="$2" + + if ! path_kind="$(inspect_cpu_controller_path "$drop_in_dir")"; then + printf 'CPU controller drop-in directory inspection failed: %s\n' \ + "$drop_in_dir" >&2 + return 1 + fi + if [ "$path_kind" != "absent" ] && [ "$path_kind" != "directory" ]; then + printf 'Refusing unexpected CPU controller drop-in directory: %s\n' "$drop_in_dir" >&2 + return 1 + fi + + if [ "$path_kind" = "directory" ]; then + if [ "$(sudo stat -Lc '%U:%G %a' -- "$drop_in_dir")" != "root:root 755" ]; then + printf 'Refusing to change existing drop-in directory owner or mode: %s\n' \ + "$drop_in_dir" >&2 + return 1 + fi + printf 'Record for rollback: %s_drop_in_dir_created=0\n' "$directory_name" + return 0 + fi + + printf 'Record for rollback: %s_drop_in_dir_created=unrecorded\n' \ + "$directory_name" + if ! sudo mkdir -m 0755 -- "$drop_in_dir"; then + printf 'CPU controller drop-in directory creation failed: %s\n' "$drop_in_dir" >&2 + return 1 + fi + + if ! drop_in_dir_id="$(sudo stat -Lc '%d:%i' -- "$drop_in_dir")"; then + printf 'CPU controller drop-in directory identity recording failed: %s\n' \ + "$drop_in_dir" >&2 + return 1 + fi + printf 'Record for rollback: %s_drop_in_dir_id=%s\n' \ + "$directory_name" "$drop_in_dir_id" + printf 'Record for rollback: %s_drop_in_dir_created=1\n' "$directory_name" + + if [ "$(sudo stat -Lc '%U:%G %a' -- "$drop_in_dir")" != "root:root 755" ]; then + printf 'Unexpected owner or mode on new drop-in directory: %s\n' "$drop_in_dir" >&2 + return 1 + fi +} + +for drop_in_name in delegation app_slice user_slice; do + printf 'Record for rollback: %s_drop_in_created=0\n' "$drop_in_name" + printf 'Record for rollback: %s_drop_in_dir_created=0\n' "$drop_in_name" + printf 'Record for rollback: %s_staging_dir_path=\n' "$drop_in_name" + printf 'Record for rollback: %s_staging_dir_created=0\n' "$drop_in_name" +done +ensure_drop_in_dir delegation "$delegation_drop_in_dir" || exit 1 +ensure_drop_in_dir app_slice "$app_slice_drop_in_dir" || exit 1 +ensure_drop_in_dir user_slice "$user_slice_drop_in_dir" || exit 1 + +create_staging_dir() { + drop_in_name="$1" + drop_in_dir="$2" + + if ! staging_token="$(node -e \ + 'process.stdout.write(require("node:crypto").randomBytes(16).toString("hex"))')" \ + || [[ ! "$staging_token" =~ ^[0-9a-f]{32}$ ]]; then + printf 'CPU controller staging name generation failed: %s\n' "$drop_in_name" >&2 + return 1 + fi + staging_dir="${drop_in_dir}/.nemoclaw-cpu-controller.${staging_token}" + printf 'Record for rollback: %s_staging_dir_path=%s\n' \ + "$drop_in_name" "$staging_dir" + printf 'Record for rollback: %s_staging_dir_created=unrecorded\n' \ + "$drop_in_name" + if ! sudo mkdir -m 0700 -- "$staging_dir"; then + printf 'CPU controller staging directory creation failed: %s\n' "$staging_dir" >&2 + return 1 + fi + if ! staging_dir_id="$(sudo stat -Lc '%d:%i' -- "$staging_dir")"; then + printf 'CPU controller staging directory identity recording failed: %s\n' \ + "$staging_dir" >&2 + return 1 + fi + printf 'Record for rollback: %s_staging_dir_id=%s\n' \ + "$drop_in_name" "$staging_dir_id" + printf 'Record for rollback: %s_staging_dir_created=1\n' "$drop_in_name" +} + +cleanup_staging_dir() { + staging_dir="$1" + expected_id="$2" + staging_file="${staging_dir}/drop-in.conf" + + if ! path_kind="$(inspect_cpu_controller_path "$staging_dir")" \ + || [ "$path_kind" != "directory" ] \ + || ! current_id="$(sudo stat -Lc '%d:%i' -- "$staging_dir")" \ + || [ "$current_id" != "$expected_id" ]; then + printf 'Refusing CPU controller staging directory whose identity changed: %s\n' \ + "$staging_dir" >&2 + return 1 + fi + if ! child_kind="$(inspect_cpu_controller_path "$staging_file")"; then + printf 'CPU controller staging file inspection failed: %s\n' "$staging_file" >&2 + return 1 + fi + if [ "$child_kind" = "file" ]; then + if [ "$(sudo stat -Lc '%d:%i' -- "$staging_dir")" != "$expected_id" ] \ + || ! sudo rm -- "$staging_file"; then + return 1 + fi + elif [ "$child_kind" != "absent" ]; then + printf 'Refusing unexpected CPU controller staging file type: %s\n' \ + "$staging_file" >&2 + return 1 + fi + if [ "$(sudo stat -Lc '%d:%i' -- "$staging_dir")" != "$expected_id" ]; then + printf 'Refusing CPU controller staging directory whose identity changed: %s\n' \ + "$staging_dir" >&2 + return 1 + fi + sudo rmdir -- "$staging_dir" +} + +create_drop_in() { + drop_in_name="$1" + drop_in="$2" + shift 2 + drop_in_dir="$(dirname -- "$drop_in")" + + if ! create_staging_dir "$drop_in_name" "$drop_in_dir"; then + return 1 + fi + staging_file="${staging_dir}/drop-in.conf" + + if ! printf '%s\n' "$@" | sudo sh -c 'set -C; cat > "$1"' sh "$staging_file" \ + || ! sudo chown root:root -- "$staging_file" \ + || ! sudo chmod 0644 -- "$staging_file"; then + printf 'CPU controller drop-in creation failed: %s\n' "$drop_in" >&2 + cleanup_staging_dir "$staging_dir" "$staging_dir_id" || true + return 1 + fi + + if ! drop_in_id="$(sudo stat -Lc '%d:%i' -- "$staging_file")"; then + printf 'CPU controller drop-in creation failed: %s\n' "$drop_in" >&2 + cleanup_staging_dir "$staging_dir" "$staging_dir_id" || true + return 1 + fi + printf 'Record for rollback: %s_drop_in_id=%s\n' "$drop_in_name" "$drop_in_id" + + if ! sudo ln -T -- "$staging_file" "$drop_in"; then + printf 'CPU controller drop-in creation failed: %s\n' "$drop_in" >&2 + cleanup_staging_dir "$staging_dir" "$staging_dir_id" || true + return 1 + fi + printf 'Record for rollback: %s_drop_in_created=1\n' "$drop_in_name" + + if ! cleanup_staging_dir "$staging_dir" "$staging_dir_id"; then + printf 'CPU controller staging cleanup failed: %s\n' "$staging_dir" >&2 + return 1 + fi +} + +creation_failed=0 +if ! create_drop_in \ + delegation "$delegation_drop_in" '[Service]' 'Delegate=cpu memory pids'; then + creation_failed=1 +elif ! create_drop_in app_slice "$app_slice_drop_in" '[Slice]' 'CPUWeight=100'; then + creation_failed=1 +elif ! create_drop_in user_slice "$user_slice_drop_in" '[Slice]' 'CPUWeight=100'; then + creation_failed=1 +fi +if [ "$creation_failed" != "0" ]; then + exit 1 +fi +``` + +If a creation command fails, do not reload systemd. Ask the administrator to inspect all three drop-in paths and their directories. Use the final value printed for every `*_created` record and its matching `device:inode` record. The initial `0` records describe only the state before creation starts; always route recovery from the final printed values. Do not search for staging names. Use only an exact `*_staging_dir_path` printed before its atomic `mkdir` attempt. + +Choose the cleanup route that matches the final records: + +- If any final `*_created` value is `0` but the same command printed its matching `*_id` afterward, that file or directory receipt was interrupted between its identity and final commit records. Retain that identity and replace the matching final `*_created` value with `1`. The general cleanup will require the path type and identity to match that record before removal. +- If any final `*_drop_in_dir_created` or `*_staging_dir_created` value is `unrecorded`, do not treat a missing identity as proof of absence and do not enter the general partial cleanup. Complete [Recover an Unrecorded Drop-In Directory](#recover-an-unrecorded-drop-in-directory) for each exact recorded intent, replace each resolved final value with `0`, and leave its identity empty. Enter the general partial cleanup only after every final `*_created` value is `0` or `1`. +- Otherwise, when every final `*_created` value is `0` or `1` and every final `0` has an empty identity, go directly to [Clean Up a Partial Drop-In Creation](#clean-up-a-partial-drop-in-creation). + +#### Recover an Unrecorded Drop-In Directory + +If the final directory record is `created=unrecorded`, the procedure recorded an exact creation intent but did not commit a creation identity. The interruption can occur before `mkdir`, during it, or before its identity record. Do not treat the initial `created=0` record as proof that the directory was pre-existing. Do not use the general partial-creation cleanup until this state is resolved. + +Establish an exclusive host-configuration maintenance window with the administrator. Pause package operations and all other changes under `/etc/systemd/system` and `/etc/systemd/user` for the command's duration. Set `unrecorded_directory` to the exact recorded drop-in-directory path or unpredictable staging intent. The command accepts only those six exact paths, validates each staging name without scanning for basenames, requires the expected type and metadata, requires an empty directory, and binds removal to two matching identity reads. + +```bash +unrecorded_directory="" +delegation_drop_in_dir="/etc/systemd/system/user@.service.d" +app_slice_drop_in_dir="/etc/systemd/user/app.slice.d" +user_slice_drop_in_dir="/etc/systemd/system/user-.slice.d" +delegation_staging_dir_path="" +app_slice_staging_dir_path="" +user_slice_staging_dir_path="" + +validate_staging_intent() { + parent="$1" + intent="$2" + prefix="${parent}/.nemoclaw-cpu-controller." + token="${intent#"$prefix"}" + [ "$intent" != "$token" ] && [[ "$token" =~ ^[0-9a-f]{32}$ ]] +} + +expected_mode="" +if [ "$unrecorded_directory" = "$delegation_drop_in_dir" ] \ + || [ "$unrecorded_directory" = "$app_slice_drop_in_dir" ] \ + || [ "$unrecorded_directory" = "$user_slice_drop_in_dir" ]; then + expected_mode="755" +elif [ -n "$delegation_staging_dir_path" ] \ + && [ "$unrecorded_directory" = "$delegation_staging_dir_path" ] \ + && validate_staging_intent "$delegation_drop_in_dir" "$delegation_staging_dir_path"; then + expected_mode="700" +elif [ -n "$app_slice_staging_dir_path" ] \ + && [ "$unrecorded_directory" = "$app_slice_staging_dir_path" ] \ + && validate_staging_intent "$app_slice_drop_in_dir" "$app_slice_staging_dir_path"; then + expected_mode="700" +elif [ -n "$user_slice_staging_dir_path" ] \ + && [ "$unrecorded_directory" = "$user_slice_staging_dir_path" ] \ + && validate_staging_intent "$user_slice_drop_in_dir" "$user_slice_staging_dir_path"; then + expected_mode="700" +else + printf 'Refusing unexpected unrecorded drop-in directory: %s\n' \ + "$unrecorded_directory" >&2 + exit 1 +fi + +inspect_cpu_controller_path() { + sudo sh -c ' + if [ -L "$1" ]; then printf "symlink\n" + elif [ -f "$1" ]; then printf "file\n" + elif [ -d "$1" ]; then printf "directory\n" + elif [ -e "$1" ]; then printf "other\n" + else printf "absent\n" + fi + ' sh "$1" +} + +if ! path_kind="$(inspect_cpu_controller_path "$unrecorded_directory")"; then + printf 'Unrecorded drop-in directory inspection failed: %s\n' \ + "$unrecorded_directory" >&2 + exit 1 +fi +if [ "$path_kind" = "absent" ]; then + exit 0 +fi +if [ "$path_kind" != "directory" ]; then + printf 'Refusing unexpected unrecorded drop-in directory type: %s\n' \ + "$unrecorded_directory" >&2 + exit 1 +fi +if [ "$(sudo stat -Lc '%U:%G %a' -- "$unrecorded_directory")" \ + != "root:root ${expected_mode}" ]; then + printf 'Refusing unexpected unrecorded drop-in directory metadata: %s\n' \ + "$unrecorded_directory" >&2 + exit 1 +fi +if ! first_entry="$(sudo find \ + "$unrecorded_directory" -mindepth 1 -maxdepth 1 -print -quit)"; then + printf 'Unrecorded drop-in directory content inspection failed: %s\n' \ + "$unrecorded_directory" >&2 + exit 1 +fi +if [ -n "$first_entry" ]; then + printf 'Refusing nonempty unrecorded drop-in directory: %s\n' \ + "$unrecorded_directory" >&2 + exit 1 +fi +if ! first_id="$(sudo stat -Lc '%d:%i' -- "$unrecorded_directory")" \ + || [[ ! "$first_id" =~ ^[0-9]+:[0-9]+$ ]] \ + || ! second_id="$(sudo stat -Lc '%d:%i' -- "$unrecorded_directory")" \ + || [ "$second_id" != "$first_id" ]; then + printf 'Refusing unrecorded drop-in directory whose identity is unstable: %s\n' \ + "$unrecorded_directory" >&2 + exit 1 +fi +sudo rmdir -- "$unrecorded_directory" +``` + +If the command refuses the path, preserve it and inspect the reported condition in the same maintenance window. After a successful removal or an already-absent result, replace the final `created=unrecorded` value with `created=0` and leave its identity empty before running the general cleanup. + +#### Clean Up a Partial Drop-In Creation + +Run this command only when creation failed before any `systemctl daemon-reload`. Enter this procedure only when every final `*_created` value is `0` or `1`. Copy the final creation records into every placeholder. The command validates the complete record before it removes anything. It removes only objects recorded as created by this procedure and bound to their creation-time identity. It preserves pre-existing objects, accepts recorded objects that are already absent, and refuses identity or type drift. + +```bash +delegation_drop_in="/etc/systemd/system/user@.service.d/90-nemoclaw-cpu-delegation.conf" +app_slice_drop_in="/etc/systemd/user/app.slice.d/90-nemoclaw-cpu-controller.conf" +user_slice_drop_in="/etc/systemd/system/user-.slice.d/90-nemoclaw-cpu-controller.conf" +delegation_drop_in_created="" +delegation_drop_in_id="" +app_slice_drop_in_created="" +app_slice_drop_in_id="" +user_slice_drop_in_created="" +user_slice_drop_in_id="" +delegation_drop_in_dir_created="" +delegation_drop_in_dir_id="" +app_slice_drop_in_dir_created="" +app_slice_drop_in_dir_id="" +user_slice_drop_in_dir_created="" +user_slice_drop_in_dir_id="" +delegation_staging_dir_path="" +delegation_staging_dir_created="" +delegation_staging_dir_id="" +app_slice_staging_dir_path="" +app_slice_staging_dir_created="" +app_slice_staging_dir_id="" +user_slice_staging_dir_path="" +user_slice_staging_dir_created="" +user_slice_staging_dir_id="" + +inspect_cpu_controller_path() { + sudo sh -c ' + if [ -L "$1" ]; then printf "symlink\n" + elif [ -f "$1" ]; then printf "file\n" + elif [ -d "$1" ]; then printf "directory\n" + elif [ -e "$1" ]; then printf "other\n" + else printf "absent\n" + fi + ' sh "$1" +} + +validate_created_drop_in() { + drop_in="$1" + created="$2" + expected_id="$3" + + if [ "$created" = "0" ]; then + if [ -n "$expected_id" ]; then + printf 'Unexpected identity for unrecorded CPU controller drop-in: %s\n' \ + "$drop_in" >&2 + return 1 + fi + return 0 + fi + if [ "$created" != "1" ] || [[ ! "$expected_id" =~ ^[0-9]+:[0-9]+$ ]]; then + printf 'Invalid creation-time file record for: %s\n' "$drop_in" >&2 + return 1 + fi + if ! path_kind="$(inspect_cpu_controller_path "$drop_in")"; then + printf 'CPU controller drop-in inspection failed: %s\n' "$drop_in" >&2 + return 1 + fi + if [ "$path_kind" = "absent" ]; then + return 0 + fi + if [ "$path_kind" != "file" ]; then + printf 'Refusing unexpected CPU controller drop-in type: %s\n' "$drop_in" >&2 + return 1 + fi + if ! current_id="$(sudo stat -Lc '%d:%i' -- "$drop_in")" \ + || [ "$current_id" != "$expected_id" ]; then + printf 'Refusing CPU controller drop-in whose identity changed: %s\n' "$drop_in" >&2 + return 1 + fi +} + +validate_created_directory() { + drop_in_dir="$1" + created="$2" + expected_id="$3" + + if [ "$created" = "0" ]; then + if [ -n "$expected_id" ]; then + printf 'Unexpected identity for unrecorded CPU controller directory: %s\n' \ + "$drop_in_dir" >&2 + return 1 + fi + return 0 + fi + if [ "$created" != "1" ] || [[ ! "$expected_id" =~ ^[0-9]+:[0-9]+$ ]]; then + printf 'Invalid creation-time directory record for: %s\n' "$drop_in_dir" >&2 + return 1 + fi + if ! path_kind="$(inspect_cpu_controller_path "$drop_in_dir")"; then + printf 'CPU controller drop-in directory inspection failed: %s\n' \ + "$drop_in_dir" >&2 + return 1 + fi + if [ "$path_kind" = "absent" ]; then + return 0 + fi + if [ "$path_kind" != "directory" ]; then + printf 'Refusing unexpected CPU controller drop-in directory type: %s\n' \ + "$drop_in_dir" >&2 + return 1 + fi + if ! current_id="$(sudo stat -Lc '%d:%i' -- "$drop_in_dir")" \ + || [ "$current_id" != "$expected_id" ]; then + printf 'Refusing CPU controller drop-in directory whose identity changed: %s\n' \ + "$drop_in_dir" >&2 + return 1 + fi +} + +validate_staging_intent() { + parent="$1" + intent="$2" + prefix="${parent}/.nemoclaw-cpu-controller." + token="${intent#"$prefix"}" + [ "$intent" != "$token" ] && [[ "$token" =~ ^[0-9a-f]{32}$ ]] +} + +validate_created_staging_directory() { + parent="$1" + staging_dir="$2" + created="$3" + expected_id="$4" + + if [ "$created" = "0" ]; then + if [ -n "$expected_id" ]; then + printf 'Unexpected identity for unrecorded staging directory: %s\n' \ + "$staging_dir" >&2 + return 1 + fi + return 0 + fi + if [ "$created" != "1" ] \ + || ! validate_staging_intent "$parent" "$staging_dir"; then + printf 'Invalid creation-time staging record for: %s\n' "$staging_dir" >&2 + return 1 + fi + validate_created_directory "$staging_dir" "$created" "$expected_id" || return 1 + if ! path_kind="$(inspect_cpu_controller_path "$staging_dir")"; then + printf 'CPU controller staging directory inspection failed: %s\n' \ + "$staging_dir" >&2 + return 1 + fi + if [ "$path_kind" = "absent" ]; then return 0; fi + if [ "$(sudo stat -Lc '%U:%G %a' -- "$staging_dir")" != "root:root 700" ]; then + printf 'Refusing unexpected staging directory metadata: %s\n' "$staging_dir" >&2 + return 1 + fi + if ! child_kind="$(inspect_cpu_controller_path "${staging_dir}/drop-in.conf")" \ + || { [ "$child_kind" != "absent" ] && [ "$child_kind" != "file" ]; }; then + printf 'Refusing unexpected CPU controller staging file: %s\n' \ + "${staging_dir}/drop-in.conf" >&2 + return 1 + fi +} + +validation_failed=0 +validate_created_drop_in \ + "$delegation_drop_in" \ + "$delegation_drop_in_created" \ + "$delegation_drop_in_id" || validation_failed=1 +validate_created_drop_in \ + "$app_slice_drop_in" \ + "$app_slice_drop_in_created" \ + "$app_slice_drop_in_id" || validation_failed=1 +validate_created_drop_in \ + "$user_slice_drop_in" \ + "$user_slice_drop_in_created" \ + "$user_slice_drop_in_id" || validation_failed=1 +validate_created_directory \ + "$(dirname "$delegation_drop_in")" \ + "$delegation_drop_in_dir_created" \ + "$delegation_drop_in_dir_id" || validation_failed=1 +validate_created_directory \ + "$(dirname "$app_slice_drop_in")" \ + "$app_slice_drop_in_dir_created" \ + "$app_slice_drop_in_dir_id" || validation_failed=1 +validate_created_directory \ + "$(dirname "$user_slice_drop_in")" \ + "$user_slice_drop_in_dir_created" \ + "$user_slice_drop_in_dir_id" || validation_failed=1 +validate_created_staging_directory \ + "$(dirname "$delegation_drop_in")" \ + "$delegation_staging_dir_path" \ + "$delegation_staging_dir_created" \ + "$delegation_staging_dir_id" || validation_failed=1 +validate_created_staging_directory \ + "$(dirname "$app_slice_drop_in")" \ + "$app_slice_staging_dir_path" \ + "$app_slice_staging_dir_created" \ + "$app_slice_staging_dir_id" || validation_failed=1 +validate_created_staging_directory \ + "$(dirname "$user_slice_drop_in")" \ + "$user_slice_staging_dir_path" \ + "$user_slice_staging_dir_created" \ + "$user_slice_staging_dir_id" || validation_failed=1 +if [ "$validation_failed" != "0" ]; then + exit 1 +fi + +remove_created_drop_in() { + drop_in="$1" + created="$2" + expected_id="$3" + + if [ "$created" = "0" ]; then + return 0 + fi + if ! path_kind="$(inspect_cpu_controller_path "$drop_in")"; then + printf 'CPU controller drop-in inspection failed: %s\n' "$drop_in" >&2 + return 1 + fi + if [ "$path_kind" = "absent" ]; then return 0; fi + if [ "$path_kind" != "file" ]; then + printf 'Refusing unexpected CPU controller drop-in type: %s\n' "$drop_in" >&2 + return 1 + fi + validate_created_drop_in "$drop_in" "$created" "$expected_id" || return 1 + sudo rm -- "$drop_in" +} + +remove_created_directory() { + drop_in_dir="$1" + created="$2" + expected_id="$3" + + if [ "$created" = "0" ]; then + return 0 + fi + if ! path_kind="$(inspect_cpu_controller_path "$drop_in_dir")"; then + printf 'CPU controller drop-in directory inspection failed: %s\n' \ + "$drop_in_dir" >&2 + return 1 + fi + if [ "$path_kind" = "absent" ]; then return 0; fi + if [ "$path_kind" != "directory" ]; then + printf 'Refusing unexpected CPU controller drop-in directory type: %s\n' \ + "$drop_in_dir" >&2 + return 1 + fi + validate_created_directory "$drop_in_dir" "$created" "$expected_id" || return 1 + sudo rmdir -- "$drop_in_dir" +} + +remove_created_staging_directory() { + parent="$1" + staging_dir="$2" + created="$3" + expected_id="$4" + + if [ "$created" = "0" ]; then return 0; fi + validate_created_staging_directory \ + "$parent" "$staging_dir" "$created" "$expected_id" || return 1 + if ! path_kind="$(inspect_cpu_controller_path "$staging_dir")"; then return 1; fi + if [ "$path_kind" = "absent" ]; then return 0; fi + staging_file="${staging_dir}/drop-in.conf" + if ! child_kind="$(inspect_cpu_controller_path "$staging_file")"; then return 1; fi + if [ "$child_kind" = "file" ]; then + if [ "$(sudo stat -Lc '%d:%i' -- "$staging_dir")" != "$expected_id" ] \ + || ! sudo rm -- "$staging_file"; then + return 1 + fi + fi + if [ "$(sudo stat -Lc '%d:%i' -- "$staging_dir")" != "$expected_id" ]; then + return 1 + fi + sudo rmdir -- "$staging_dir" +} + +cleanup_failed=0 +remove_created_drop_in \ + "$user_slice_drop_in" "$user_slice_drop_in_created" "$user_slice_drop_in_id" || + cleanup_failed=1 +remove_created_drop_in \ + "$app_slice_drop_in" "$app_slice_drop_in_created" "$app_slice_drop_in_id" || + cleanup_failed=1 +remove_created_drop_in \ + "$delegation_drop_in" "$delegation_drop_in_created" "$delegation_drop_in_id" || + cleanup_failed=1 +remove_created_staging_directory \ + "$(dirname "$app_slice_drop_in")" \ + "$app_slice_staging_dir_path" \ + "$app_slice_staging_dir_created" \ + "$app_slice_staging_dir_id" || cleanup_failed=1 +remove_created_staging_directory \ + "$(dirname "$delegation_drop_in")" \ + "$delegation_staging_dir_path" \ + "$delegation_staging_dir_created" \ + "$delegation_staging_dir_id" || cleanup_failed=1 +remove_created_staging_directory \ + "$(dirname "$user_slice_drop_in")" \ + "$user_slice_staging_dir_path" \ + "$user_slice_staging_dir_created" \ + "$user_slice_staging_dir_id" || cleanup_failed=1 +remove_created_directory \ + "$(dirname "$user_slice_drop_in")" \ + "$user_slice_drop_in_dir_created" \ + "$user_slice_drop_in_dir_id" || cleanup_failed=1 +remove_created_directory \ + "$(dirname "$app_slice_drop_in")" \ + "$app_slice_drop_in_dir_created" \ + "$app_slice_drop_in_dir_id" || cleanup_failed=1 +remove_created_directory \ + "$(dirname "$delegation_drop_in")" \ + "$delegation_drop_in_dir_created" \ + "$delegation_drop_in_dir_id" || cleanup_failed=1 +exit "$cleanup_failed" +``` + +If the command exits nonzero, inspect every refusal and preserve the affected path. After correcting a transient cleanup failure, rerun the complete command with the same final creation records. Do not reload systemd after partial-creation cleanup. + +Verify that all three persistent paths are root-owned regular files with mode `0644`: + +```bash +for drop_in in "$delegation_drop_in" "$app_slice_drop_in" "$user_slice_drop_in"; do + if sudo test -L "$drop_in" \ + || ! sudo test -f "$drop_in" \ + || [ "$(sudo stat -Lc '%U:%G %a' -- "$drop_in")" != "root:root 644" ]; then + printf 'Unexpected CPU controller drop-in type, owner, or mode: %s\n' "$drop_in" >&2 + exit 1 + fi +done + +sudo stat -Lc '%n %d:%i' -- \ + "$delegation_drop_in" "$app_slice_drop_in" "$user_slice_drop_in" +sudo cat -- "$delegation_drop_in" +sudo cat -- "$app_slice_drop_in" +sudo cat -- "$user_slice_drop_in" +``` + +Continue only when the command exits with status `0`, all three `device:inode` values match the creation command's rollback records, the service file contains the service settings below, and both slice files contain the slice settings below: + +```ini +[Service] +Delegate=cpu memory pids +[Slice] +CPUWeight=100 +``` + +Record all rollback lines from the creation command in the administrator's change record, including each file's last `created` value, each published file's `device:inode` value, and all three directory creation records. They identify the exact files and any directories created by this procedure and contain no credentials. Rollback must use those recorded values. If a required value is lost, do not use the removal command below; ask the administrator to inspect and remove the configuration through the host's normal change-management process. + +The administrator's `sudo` policy can request authentication. NemoClaw never receives that credential. + + + Save the affected user's work before running the next command. Stopping and starting the user + manager interrupts that user's systemd services, including rootless Podman and other user + services. Run the command from an administrator session that does not depend on the affected + user's manager. + + +Run the stop, reload, and start sequence: + +```bash +uid="" +sudo systemctl stop "user@${uid}.service" || exit 1 +sudo systemctl daemon-reload || exit 1 + +start_output="" +if ! start_output="$(sudo systemctl start "user@${uid}.service" 2>&1)"; then + printf '%s\n' "$start_output" >&2 + manager_status="$( + sudo systemctl status "user@${uid}.service" --no-pager 2>&1 || true + )" + printf '%s\n' "$manager_status" >&2 + if [[ "${start_output}"$'\n'"${manager_status}" == *"219/CGROUP"* ]]; then + printf '%s\n' \ + 'Immediate user-manager start failed with 219/CGROUP; use later-login recovery.' >&2 + else + printf '%s\n' 'Immediate user-manager start failed; inspect the reported status.' >&2 + fi + exit 1 +fi +``` + +After the affected user signs in again, start `app.slice` in that user's session: + +```bash +systemctl --user start app.slice +``` + +The stop must complete before `daemon-reload` runs. Reloading while the instantiated unit is inactive lets systemd recalculate ancestor controller masks for the new delegation. The final start makes the user manager read the `app.slice` CPU weight and enable the delegated controller. Do not replace this sequence with `systemctl restart`, which can fail with `219/CGROUP` before systemd recalculates the masks. The sequence stops at the first failed command. If a command fails after the manager stops, correct the failure and rerun the failed command and each remaining command. + +On Ubuntu 22.04, the immediate `systemctl start user@${uid}.service` can still fail with `status=219/CGROUP` even after the inactive reload. That result leaves the current user manager stopped, but a later login can create it under the corrected cgroup hierarchy. Do not remove any of the three drop-ins or repeat the creation command. Save any remaining work in the affected user's sessions, sign out of all those sessions, and sign in again. Then start `app.slice` and run the verification below. For any other start failure, inspect `sudo systemctl status "user@${uid}.service" --no-pager` and `sudo journalctl -u "user@${uid}.service" --no-pager` before retrying the failed start. + +The administrator can reboot the host instead of running the stop, reload, and start sequence. + + + Save work for every host user before rebooting. A reboot interrupts all user services and host + workloads, not only the affected user's services. + + +Reboot only after the drop-in verification succeeds: + +```bash +sudo systemctl reboot +``` + +Verify the root hierarchy, current user manager, and `app.slice`: this command also verifies the per-user slice, for four boundaries in total. + +```bash +uid="$(id -u)" +user_slice="/sys/fs/cgroup/user.slice/user-${uid}.slice" +user_manager="/sys/fs/cgroup/user.slice/user-${uid}.slice/user@${uid}.service" +export LC_ALL=C + +classify_cpu_controller() { + controllers="$1" + if [ ! -e "$controllers" ]; then + printf 'missing\n' + return 1 + fi + if [ ! -r "$controllers" ]; then + printf 'unreadable\n' + return 1 + fi + + if evidence="$(node - "$controllers" <<'NODE' +const fs = require("node:fs"); + +const controllers = process.argv[2]; +let descriptor; +try { + descriptor = fs.openSync(controllers, "r"); + const buffer = Buffer.alloc(4097); + let bytesRead = 0; + while (bytesRead < buffer.length) { + const count = fs.readSync( + descriptor, + buffer, + bytesRead, + buffer.length - bytesRead, + null, + ); + if (count === 0) break; + bytesRead += count; + } + + const content = buffer.subarray(0, bytesRead); + if (content.length > 4096) { + process.exitCode = 2; + } else { + const body = content.at(-1) === 0x0a ? content.subarray(0, -1) : content; + const text = body.toString("utf8"); + const names = text === "" ? [] : text.split(" "); + if ( + body.includes(0x0a) || + new Set(names).size !== names.length || + names.some((name) => !/^[a-z][a-z0-9_]*$/u.test(name)) + ) { + process.exitCode = 2; + } else { + process.stdout.write(names.join(" ")); + } + } +} catch { + process.exitCode = 3; +} finally { + if (descriptor !== undefined) { + try { + fs.closeSync(descriptor); + } catch { + process.exitCode = 3; + } + } +} +NODE + )"; then + case " $evidence " in + *" cpu "*) printf 'cpu\n' ;; + *) + printf 'missing\n' + return 1 + ;; + esac + else + read_status="$?" + if [ "$read_status" = "2" ]; then + printf 'malformed\n' + else + printf 'read failed\n' + fi + return 1 + fi +} + +verification_failed=0 +for controllers in \ + /sys/fs/cgroup/cgroup.controllers \ + "${user_slice}/cgroup.controllers" \ + "${user_manager}/cgroup.controllers" \ + "${user_manager}/app.slice/cgroup.controllers" +do + printf '%s: ' "$controllers" + classify_cpu_controller "$controllers" || verification_failed=1 +done +exit "$verification_failed" +``` + +Continue only when all four lines end in `: cpu`. This verification reads no credentials. Then rerun portable onboarding: + +```bash +$$nemoclaw onboard --experimental-profile portable +``` + +#### Remove the CPU Controller Drop-Ins + +Remove only the three named files and any drop-in directory that the creation record marks as created by this procedure. Do not use `systemctl revert`. Do not remove another `user@.service`, per-user-slice, or `app.slice` drop-in or a pre-existing drop-in directory. + +Retrieve each file's final `created` value and every applicable `device:inode` value from the administrator's creation-time change record. Inspect only paths that the record marks as created: + +```bash +uid="" +delegation_drop_in="/etc/systemd/system/user@.service.d/90-nemoclaw-cpu-delegation.conf" +app_slice_drop_in="/etc/systemd/user/app.slice.d/90-nemoclaw-cpu-controller.conf" +user_slice_drop_in="/etc/systemd/system/user-${uid}.slice.d/90-nemoclaw-cpu-controller.conf" +delegation_drop_in_created="" +expected_delegation_drop_in_id="" +app_slice_drop_in_created="" +expected_app_slice_drop_in_id="" +user_slice_drop_in_created="" +expected_user_slice_drop_in_id="" +delegation_drop_in_dir_created="" +delegation_drop_in_dir_id="" +app_slice_drop_in_dir_created="" +app_slice_drop_in_dir_id="" +user_slice_drop_in_dir_created="" +user_slice_drop_in_dir_id="" + +if [ "$delegation_drop_in_created" = "1" ]; then + sudo cat -- "$delegation_drop_in" + sudo stat -Lc '%n %d:%i' -- "$delegation_drop_in" +fi +if [ "$app_slice_drop_in_created" = "1" ]; then + sudo cat -- "$app_slice_drop_in" + sudo stat -Lc '%n %d:%i' -- "$app_slice_drop_in" +fi +if [ "$user_slice_drop_in_created" = "1" ]; then + sudo cat -- "$user_slice_drop_in" + sudo stat -Lc '%n %d:%i' -- "$user_slice_drop_in" +fi +``` + +For each path marked as created, continue only when its content and current `device:inode` value match the creation-time record. Use an empty identity value for a file or directory whose final `created` value is `0`. The removal command validates the complete record before it stops the manager or removes a path. An already absent recorded path is accepted on retry, an identity mismatch preserves every remaining recorded path, and a path marked as not created is never removed. + + + Save the affected user's work before running the removal command. Stopping and starting the user + manager interrupts that user's systemd services, including rootless Podman and other user + services. Run the command from an administrator session that does not depend on the affected + user's manager. + + +```bash +uid="" +delegation_drop_in="/etc/systemd/system/user@.service.d/90-nemoclaw-cpu-delegation.conf" +app_slice_drop_in="/etc/systemd/user/app.slice.d/90-nemoclaw-cpu-controller.conf" +user_slice_drop_in="/etc/systemd/system/user-${uid}.slice.d/90-nemoclaw-cpu-controller.conf" +delegation_drop_in_created="" +expected_delegation_drop_in_id="" +app_slice_drop_in_created="" +expected_app_slice_drop_in_id="" +user_slice_drop_in_created="" +expected_user_slice_drop_in_id="" +delegation_drop_in_dir_created="" +delegation_drop_in_dir_id="" +app_slice_drop_in_dir_created="" +app_slice_drop_in_dir_id="" +user_slice_drop_in_dir_created="" +user_slice_drop_in_dir_id="" + +inspect_cpu_controller_path() { + sudo sh -c ' + if [ -L "$1" ]; then printf "symlink\n" + elif [ -f "$1" ]; then printf "file\n" + elif [ -d "$1" ]; then printf "directory\n" + elif [ -e "$1" ]; then printf "other\n" + else printf "absent\n" + fi + ' sh "$1" +} + +validate_recorded_drop_in() { + drop_in="$1" + created="$2" + expected_id="$3" + + if [ "$created" = "0" ]; then + if [ -n "$expected_id" ]; then + printf 'Unexpected identity for unrecorded CPU controller drop-in: %s\n' \ + "$drop_in" >&2 + return 1 + fi + return 0 + fi + if [ "$created" != "1" ] || [[ ! "$expected_id" =~ ^[0-9]+:[0-9]+$ ]]; then + printf 'Invalid creation-time identity for: %s\n' "$drop_in" >&2 + return 1 + fi + if ! path_kind="$(inspect_cpu_controller_path "$drop_in")"; then + printf 'CPU controller drop-in inspection failed: %s\n' "$drop_in" >&2 + return 1 + fi + if [ "$path_kind" = "absent" ]; then + return 0 + fi + if [ "$path_kind" != "file" ]; then + printf 'Refusing unexpected CPU controller drop-in type: %s\n' "$drop_in" >&2 + return 1 + fi + if ! current_id="$(sudo stat -Lc '%d:%i' -- "$drop_in")" \ + || [ "$current_id" != "$expected_id" ]; then + printf 'Refusing CPU controller drop-in whose identity changed: %s\n' "$drop_in" >&2 + return 1 + fi +} + +validate_recorded_directory() { + drop_in_dir="$1" + created="$2" + expected_id="$3" + + if [ "$created" = "0" ]; then + if [ -n "$expected_id" ]; then + printf 'Unexpected identity for unrecorded CPU controller directory: %s\n' \ + "$drop_in_dir" >&2 + return 1 + fi + return 0 + fi + if [ "$created" != "1" ] || [[ ! "$expected_id" =~ ^[0-9]+:[0-9]+$ ]]; then + printf 'Invalid creation-time directory record for: %s\n' "$drop_in_dir" >&2 + return 1 + fi + if ! path_kind="$(inspect_cpu_controller_path "$drop_in_dir")"; then + printf 'CPU controller drop-in directory inspection failed: %s\n' \ + "$drop_in_dir" >&2 + return 1 + fi + if [ "$path_kind" = "absent" ]; then + return 0 + fi + if [ "$path_kind" != "directory" ]; then + printf 'Refusing unexpected CPU controller drop-in directory type: %s\n' \ + "$drop_in_dir" >&2 + return 1 + fi + if ! current_id="$(sudo stat -Lc '%d:%i' -- "$drop_in_dir")" \ + || [ "$current_id" != "$expected_id" ]; then + printf 'Refusing CPU controller drop-in directory whose identity changed: %s\n' \ + "$drop_in_dir" >&2 + return 1 + fi +} + +validation_failed=0 +validate_recorded_drop_in \ + "$delegation_drop_in" \ + "$delegation_drop_in_created" \ + "$expected_delegation_drop_in_id" || validation_failed=1 +validate_recorded_drop_in \ + "$app_slice_drop_in" \ + "$app_slice_drop_in_created" \ + "$expected_app_slice_drop_in_id" || validation_failed=1 +validate_recorded_drop_in \ + "$user_slice_drop_in" \ + "$user_slice_drop_in_created" \ + "$expected_user_slice_drop_in_id" || validation_failed=1 +validate_recorded_directory \ + "$(dirname "$delegation_drop_in")" \ + "$delegation_drop_in_dir_created" \ + "$delegation_drop_in_dir_id" || validation_failed=1 +validate_recorded_directory \ + "$(dirname "$app_slice_drop_in")" \ + "$app_slice_drop_in_dir_created" \ + "$app_slice_drop_in_dir_id" || validation_failed=1 +validate_recorded_directory \ + "$(dirname "$user_slice_drop_in")" \ + "$user_slice_drop_in_dir_created" \ + "$user_slice_drop_in_dir_id" || validation_failed=1 +if [ "$validation_failed" != "0" ]; then + exit 1 +fi + +remove_validated_drop_in() { + drop_in="$1" + created="$2" + expected_id="$3" + + if [ "$created" = "0" ]; then + return 0 + fi + if ! path_kind="$(inspect_cpu_controller_path "$drop_in")"; then + printf 'CPU controller drop-in inspection failed: %s\n' "$drop_in" >&2 + return 1 + fi + if [ "$path_kind" = "absent" ]; then return 0; fi + if [ "$path_kind" != "file" ]; then + printf 'Refusing unexpected CPU controller drop-in type: %s\n' "$drop_in" >&2 + return 1 + fi + validate_recorded_drop_in "$drop_in" "$created" "$expected_id" || return 1 + sudo rm -- "$drop_in" +} + +remove_validated_directory() { + drop_in_dir="$1" + created="$2" + expected_id="$3" + + if [ "$created" = "0" ]; then + return 0 + fi + if ! path_kind="$(inspect_cpu_controller_path "$drop_in_dir")"; then + printf 'CPU controller drop-in directory inspection failed: %s\n' \ + "$drop_in_dir" >&2 + return 1 + fi + if [ "$path_kind" = "absent" ]; then return 0; fi + if [ "$path_kind" != "directory" ]; then + printf 'Refusing unexpected CPU controller drop-in directory type: %s\n' \ + "$drop_in_dir" >&2 + return 1 + fi + validate_recorded_directory "$drop_in_dir" "$created" "$expected_id" || return 1 + sudo rmdir -- "$drop_in_dir" +} + +if ! sudo systemctl stop "user@${uid}.service"; then + exit 1 +fi + +cleanup_failed=0 +remove_validated_drop_in \ + "$user_slice_drop_in" \ + "$user_slice_drop_in_created" \ + "$expected_user_slice_drop_in_id" || cleanup_failed=1 +remove_validated_drop_in \ + "$delegation_drop_in" \ + "$delegation_drop_in_created" \ + "$expected_delegation_drop_in_id" || cleanup_failed=1 +remove_validated_drop_in \ + "$app_slice_drop_in" \ + "$app_slice_drop_in_created" \ + "$expected_app_slice_drop_in_id" || cleanup_failed=1 +remove_validated_directory \ + "$(dirname "$delegation_drop_in")" \ + "$delegation_drop_in_dir_created" \ + "$delegation_drop_in_dir_id" || cleanup_failed=1 +remove_validated_directory \ + "$(dirname "$app_slice_drop_in")" \ + "$app_slice_drop_in_dir_created" \ + "$app_slice_drop_in_dir_id" || cleanup_failed=1 +remove_validated_directory \ + "$(dirname "$user_slice_drop_in")" \ + "$user_slice_drop_in_dir_created" \ + "$user_slice_drop_in_dir_id" || cleanup_failed=1 +sudo systemctl daemon-reload || cleanup_failed=1 + +start_output="" +if ! start_output="$(sudo systemctl start "user@${uid}.service" 2>&1)"; then + printf '%s\n' "$start_output" >&2 + manager_status="$( + sudo systemctl status "user@${uid}.service" --no-pager 2>&1 || true + )" + printf '%s\n' "$manager_status" >&2 + if [[ "${start_output}"$'\n'"${manager_status}" == *"219/CGROUP"* ]]; then + printf '%s\n' \ + 'Immediate user-manager start failed with 219/CGROUP; use later-login recovery.' >&2 + else + printf '%s\n' 'Immediate user-manager start failed; inspect the reported status.' >&2 + fi + cleanup_failed=1 +fi +exit "$cleanup_failed" +``` + +The inactive reload removes all three NemoClaw drop-ins from the instantiated units and recalculates ancestor controller masks before the manager starts. Do not replace this sequence with `systemctl restart`. If the command exits nonzero, inspect every reported failure. After correcting it, rerun the complete removal command with the same creation-time identities; paths already removed by the earlier attempt are accepted. + +If the immediate start reports `219/CGROUP`, the recorded drop-ins have already been removed, but the current user manager remains stopped. Save any remaining work in the affected user's sessions, sign out of all those sessions, and sign in again so systemd creates the user manager under the restored hierarchy. Do not restore the drop-ins only because the immediate start returned `219/CGROUP`. Then run the removal verification below. For another start failure, use the reported `systemctl status` output and `sudo journalctl -u "user@${uid}.service" --no-pager` to correct the service failure before retrying the removal command. + +After the affected user signs in again, verify that systemd no longer loads any of the three drop-ins: + +```bash +uid="" +delegation_drop_in="/etc/systemd/system/user@.service.d/90-nemoclaw-cpu-delegation.conf" +app_slice_drop_in="/etc/systemd/user/app.slice.d/90-nemoclaw-cpu-controller.conf" +user_slice_drop_in="/etc/systemd/system/user-${uid}.slice.d/90-nemoclaw-cpu-controller.conf" + +for drop_in in "$delegation_drop_in" "$app_slice_drop_in" "$user_slice_drop_in"; do + if sudo test -e "$drop_in" || sudo test -L "$drop_in"; then + printf 'CPU controller drop-in remains after removal: %s\n' "$drop_in" >&2 + exit 1 + fi +done + +systemctl cat user@.service +systemctl cat "user-${uid}.slice" +systemctl --user cat app.slice +``` + +The output must not list any NemoClaw CPU-controller drop-in. Another administrator-owned drop-in can still configure `cpu`; do not remove it as part of this rollback. Removing the NemoClaw drop-ins can make the portable CPU delegation preflight fail again. + ### Portable Podman Readiness Fails Portable commands use the current user's rootless Podman socket authority recorded in NemoClaw state. diff --git a/scripts/checks/run-portable-cpu-delegation-proof.mts b/scripts/checks/run-portable-cpu-delegation-proof.mts new file mode 100644 index 00000000000..28b78adac1a --- /dev/null +++ b/scripts/checks/run-portable-cpu-delegation-proof.mts @@ -0,0 +1,1487 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +export type PortableCpuDelegationProofMode = + | "prepare" + | "reject" + | "admit" + | "diagnostics" + | "cleanup"; + +export const PORTABLE_CPU_DELEGATION_PROOF_CONTRACT = Object.freeze({ + proofUser: "nemoclaw-e2e", + targetId: "portable-cpu-delegation", + delegationDropIn: "/etc/systemd/system/user@.service.d/90-nemoclaw-cpu-delegation.conf", + delegationDropInContent: "[Service]\nDelegate=cpu memory pids\n", + missingDelegationDropInContent: "[Service]\nDelegate=memory pids\n", + userSliceDropInName: "90-nemoclaw-cpu-controller.conf", + userSliceDropInContent: "[Slice]\nCPUWeight=100\n", + appSliceDropIn: "/etc/systemd/user/app.slice.d/90-nemoclaw-cpu-controller.conf", + appSliceDropInContent: "[Slice]\nCPUWeight=100\n", + controllerEvidenceReadBytes: 4097, + immediateStartFailure: "219/CGROUP", +}); + +export type CommandResult = { + readonly status: number; + readonly stdout: string; + readonly stderr: string; +}; + +export type CommandOptions = { + readonly cwd?: string; + readonly input?: string; +}; + +export interface HostCommandRunner { + run(executable: string, argv: readonly string[], options?: CommandOptions): CommandResult; +} + +export type HostPathStat = { + readonly dev: number; + readonly ino: number; + isDirectory(): boolean; + isSymbolicLink(): boolean; +}; + +export interface HostFilesystem { + appendText(target: string, content: string): void; + exists(target: string): boolean; + lstat(target: string): HostPathStat; + makeDirectory( + target: string, + options: { readonly mode: number; readonly recursive: boolean }, + ): void; + readText(target: string): string; + removeDirectory(target: string): void; + removeFile(target: string): void; + writeExclusive(target: string, content: string): void; +} + +export type PortableCpuDelegationProofDeps = { + readonly env?: NodeJS.ProcessEnv; + readonly filesystem?: HostFilesystem; + readonly runner?: HostCommandRunner; + readonly randomId?: () => string; + readonly sleep?: (milliseconds: number) => void; + readonly stderr?: Pick; +}; + +type ProofContext = { + readonly artifactDir: string; + readonly delegationDropIn: string; + readonly delegationDropInDir: string; + readonly delegationDropInDirMarker: string; + readonly delegationDropInMarker: string; + readonly env: NodeJS.ProcessEnv; + readonly filesystem: HostFilesystem; + readonly appSliceDropIn: string; + readonly appSliceDropInDir: string; + readonly appSliceDropInDirMarker: string; + readonly appSliceDropInMarker: string; + readonly proofUser: string; + readonly randomId: () => string; + readonly runner: HostCommandRunner; + readonly runnerTemp: string; + readonly sleep: (milliseconds: number) => void; + readonly sourceCacheDir: string; + readonly sourceCacheMarker: string; + readonly sourceCacheParent: string; + readonly sourceCacheParentMarker: string; + readonly stderr: Pick; + readonly userSliceDropInDirMarker: string; + readonly userSliceDropInMarker: string; + readonly workspace: string; + readonly workspaceTraverseMarker: string; +}; + +type PreparedDropIn = { + dirCreated: boolean; + dirId: string; + id: string; + temp: string; + tempId: string; +}; + +type PrepareState = { + readonly appSlice: PreparedDropIn; + createdUser: boolean; + readonly delegation: PreparedDropIn; + sourceCacheId: string; + sourceCacheParentId: string; + readonly userSlice: PreparedDropIn; + uid: string; +}; + +const MODES: readonly PortableCpuDelegationProofMode[] = [ + "prepare", + "reject", + "admit", + "diagnostics", + "cleanup", +]; + +const MARKER_NAMES = Object.freeze({ + appSliceDropIn: "nemoclaw-app-slice-drop-in-created", + appSliceDropInDir: "nemoclaw-app-slice-drop-in-dir-created", + delegationDropIn: "nemoclaw-cpu-delegation-drop-in-created", + delegationDropInDir: "nemoclaw-cpu-delegation-drop-in-dir-created", + sourceCache: "nemoclaw-source-require-cache-created", + sourceCacheParent: "nemoclaw-source-require-cache-parent-created", + userSliceDropIn: "nemoclaw-user-slice-drop-in-created", + userSliceDropInDir: "nemoclaw-user-slice-drop-in-dir-created", + workspaceModes: "nemoclaw-workspace-traverse-modes", +}); + +class ProofError extends Error {} + +function defaultRunner(): HostCommandRunner { + return { + run(executable, argv, options = {}) { + const result = spawnSync(executable, [...argv], { + cwd: options.cwd, + encoding: "utf8", + input: options.input, + shell: false, + }); + if (result.error) throw result.error; + if (result.status === null) { + throw new ProofError( + `${executable} terminated by ${result.signal ?? "an unknown signal"}.`, + ); + } + return { + status: result.status, + stdout: result.stdout ?? "", + stderr: result.stderr ?? "", + }; + }, + }; +} + +function defaultFilesystem(): HostFilesystem { + return { + appendText(target, content) { + fs.appendFileSync(target, content, { encoding: "utf8" }); + }, + exists(target) { + return fs.existsSync(target); + }, + lstat(target) { + return fs.lstatSync(target); + }, + makeDirectory(target, options) { + fs.mkdirSync(target, options); + }, + readText(target) { + return fs.readFileSync(target, "utf8"); + }, + removeDirectory(target) { + fs.rmdirSync(target); + }, + removeFile(target) { + fs.unlinkSync(target); + }, + writeExclusive(target, content) { + fs.writeFileSync(target, content, { encoding: "utf8", flag: "wx", mode: 0o600 }); + }, + }; +} + +function blockingSleep(milliseconds: number): void { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, milliseconds); +} + +function requireAbsolutePath(value: string | undefined, name: string): string { + if (!value || !path.isAbsolute(value) || value.includes("\0") || value.includes("\n")) { + throw new ProofError(`${name} must be an absolute path without control bytes.`); + } + return path.normalize(value); +} + +function requireExact(value: string | undefined, expected: string, name: string): string { + if (value !== expected) throw new ProofError(`${name} must be ${expected}.`); + return value; +} + +function requireSha(value: string | undefined): string { + if (!value || !/^[a-f0-9]{40}$/u.test(value)) { + throw new ProofError("E2E_SOURCE_REVISION must be a 40-character lowercase commit SHA."); + } + return value; +} + +function requireNumeric(value: string | undefined, name: string): string { + if (!value || !/^[0-9]+$/u.test(value)) throw new ProofError(`${name} must be numeric.`); + return value; +} + +function userSliceDropIn(uid: string): string { + return `/etc/systemd/system/user-${uid}.slice.d/${PORTABLE_CPU_DELEGATION_PROOF_CONTRACT.userSliceDropInName}`; +} + +function context(deps: PortableCpuDelegationProofDeps): ProofContext { + const env = deps.env ?? process.env; + const workspace = requireAbsolutePath(env.GITHUB_WORKSPACE, "GITHUB_WORKSPACE"); + const runnerTemp = requireAbsolutePath(env.RUNNER_TEMP, "RUNNER_TEMP"); + const artifactDir = requireAbsolutePath(env.E2E_ARTIFACT_DIR, "E2E_ARTIFACT_DIR"); + const expectedArtifactDir = path.join(workspace, "e2e-artifacts", "portable-cpu-delegation"); + if (artifactDir !== expectedArtifactDir) { + throw new ProofError(`E2E_ARTIFACT_DIR must be ${expectedArtifactDir}.`); + } + requireExact( + env.E2E_CPU_DELEGATION_USER, + PORTABLE_CPU_DELEGATION_PROOF_CONTRACT.proofUser, + "E2E_CPU_DELEGATION_USER", + ); + requireExact(env.E2E_TARGET_ID, PORTABLE_CPU_DELEGATION_PROOF_CONTRACT.targetId, "E2E_TARGET_ID"); + requireSha(env.E2E_SOURCE_REVISION); + const delegationDropIn = PORTABLE_CPU_DELEGATION_PROOF_CONTRACT.delegationDropIn; + const appSliceDropIn = PORTABLE_CPU_DELEGATION_PROOF_CONTRACT.appSliceDropIn; + return { + artifactDir, + delegationDropIn, + delegationDropInDir: path.dirname(delegationDropIn), + delegationDropInDirMarker: path.join(runnerTemp, MARKER_NAMES.delegationDropInDir), + delegationDropInMarker: path.join(runnerTemp, MARKER_NAMES.delegationDropIn), + env, + filesystem: deps.filesystem ?? defaultFilesystem(), + appSliceDropIn, + appSliceDropInDir: path.dirname(appSliceDropIn), + appSliceDropInDirMarker: path.join(runnerTemp, MARKER_NAMES.appSliceDropInDir), + appSliceDropInMarker: path.join(runnerTemp, MARKER_NAMES.appSliceDropIn), + proofUser: PORTABLE_CPU_DELEGATION_PROOF_CONTRACT.proofUser, + randomId: deps.randomId ?? randomUUID, + runner: deps.runner ?? defaultRunner(), + runnerTemp, + sleep: deps.sleep ?? blockingSleep, + sourceCacheDir: path.join(workspace, "node_modules", ".cache", "nemoclaw-source-require"), + sourceCacheMarker: path.join(runnerTemp, MARKER_NAMES.sourceCache), + sourceCacheParent: path.join(workspace, "node_modules", ".cache"), + sourceCacheParentMarker: path.join(runnerTemp, MARKER_NAMES.sourceCacheParent), + stderr: deps.stderr ?? process.stderr, + userSliceDropInDirMarker: path.join(runnerTemp, MARKER_NAMES.userSliceDropInDir), + userSliceDropInMarker: path.join(runnerTemp, MARKER_NAMES.userSliceDropIn), + workspace, + workspaceTraverseMarker: path.join(runnerTemp, MARKER_NAMES.workspaceModes), + }; +} + +function run( + ctx: ProofContext, + executable: string, + argv: readonly string[], + options: CommandOptions = {}, +): CommandResult { + return ctx.runner.run(executable, argv, options); +} + +function checked( + ctx: ProofContext, + executable: string, + argv: readonly string[], + options: CommandOptions = {}, +): CommandResult { + const result = run(ctx, executable, argv, options); + if (result.status !== 0) { + throw new ProofError( + `${executable} ${argv.join(" ")} failed (${String(result.status)}): ${result.stderr.trim()}`, + ); + } + return result; +} + +function sudo( + ctx: ProofContext, + argv: readonly string[], + options: CommandOptions = {}, +): CommandResult { + return run(ctx, "sudo", argv, options); +} + +function checkedSudo( + ctx: ProofContext, + argv: readonly string[], + options: CommandOptions = {}, +): CommandResult { + return checked(ctx, "sudo", argv, options); +} + +function sudoTest(ctx: ProofContext, argv: readonly string[]): boolean { + const result = sudo(ctx, ["test", ...argv]); + if (result.status === 0) return true; + if (result.status === 1 && result.stderr.trim() === "") return false; + throw new ProofError( + `sudo test ${argv.join(" ")} failed (${String(result.status)}): ${result.stderr.trim()}`, + ); +} + +function sudoExists(ctx: ProofContext, target: string): boolean { + return sudoTest(ctx, ["-e", target]) || sudoTest(ctx, ["-L", target]); +} + +function ensureDirectAbsent(ctx: ProofContext, target: string, label: string): void { + try { + ctx.filesystem.lstat(target); + throw new ProofError(`${label} already exists: ${target}`); + } catch (error) { + if (error instanceof ProofError) throw error; + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } +} + +function writeMarker(ctx: ProofContext, target: string, value: string): void { + ctx.filesystem.writeExclusive(target, `${value}\n`); +} + +function readMarker(ctx: ProofContext, target: string): string { + return ctx.filesystem.readText(target).trim(); +} + +function appendGithubEnv(ctx: ProofContext, name: string, value: string): void { + const githubEnv = requireAbsolutePath(ctx.env.GITHUB_ENV, "GITHUB_ENV"); + ctx.filesystem.appendText(githubEnv, `${name}=${value}\n`); +} + +function identity(ctx: ProofContext, target: string): string { + const value = checkedSudo(ctx, ["stat", "-Lc", "%d:%i", "--", target]).stdout.trim(); + if (!/^[0-9]+:[0-9]+$/u.test(value)) { + throw new ProofError(`Invalid device:inode identity for ${target}.`); + } + return value; +} + +function directIdentity(ctx: ProofContext, target: string): string { + const value = ctx.filesystem.lstat(target); + if (!Number.isSafeInteger(value.dev) || !Number.isSafeInteger(value.ino)) { + throw new ProofError(`Invalid device:inode identity for ${target}.`); + } + return `${String(value.dev)}:${String(value.ino)}`; +} + +function ownerMode(ctx: ProofContext, target: string): string { + return checkedSudo(ctx, ["stat", "-Lc", "%U:%G %a", "--", target]).stdout.trim(); +} + +function userExists(ctx: ProofContext): boolean { + const result = run(ctx, "getent", ["passwd", ctx.proofUser]); + if (result.status === 0) return true; + if (result.status === 2) return false; + throw new ProofError( + `getent passwd ${ctx.proofUser} failed (${String(result.status)}): ${result.stderr.trim()}`, + ); +} + +function userComment(ctx: ProofContext): string { + const passwd = checked(ctx, "getent", ["passwd", ctx.proofUser]).stdout.trim(); + return passwd.split(":")[4] ?? ""; +} + +function userHome(ctx: ProofContext): string { + const passwd = checked(ctx, "getent", ["passwd", ctx.proofUser]).stdout.trim(); + const home = passwd.split(":")[5]; + return requireAbsolutePath(home, "proof user home"); +} + +function uidOf(ctx: ProofContext): string { + return requireNumeric(checked(ctx, "id", ["-u", ctx.proofUser]).stdout.trim(), "proof UID"); +} + +function ensurePrivilegedDirectory( + ctx: ProofContext, + target: string, + marker: string, + createdEnv: string, + idEnv: string, + label: string, + recordCreatedIdentity: (id: string) => void, +): { readonly created: boolean; readonly id: string } { + if ( + sudoTest(ctx, ["-L", target]) || + (sudoTest(ctx, ["-e", target]) && !sudoTest(ctx, ["-d", target])) + ) { + throw new ProofError(`${label} has an unexpected type.`); + } + if (sudoTest(ctx, ["-d", target])) { + if (ownerMode(ctx, target) !== "root:root 755") { + throw new ProofError(`${label} has unexpected owner or mode.`); + } + return { created: false, id: "" }; + } + + appendGithubEnv(ctx, createdEnv, "unrecorded"); + checkedSudo(ctx, ["mkdir", "-m", "0755", "--", target]); + let id = ""; + let markerPublished = false; + try { + id = identity(ctx, target); + recordCreatedIdentity(id); + appendGithubEnv(ctx, idEnv, id); + writeMarker(ctx, marker, id); + markerPublished = true; + appendGithubEnv(ctx, createdEnv, "1"); + if (ownerMode(ctx, target) !== "root:root 755") { + throw new ProofError(`${label} has unexpected owner or mode.`); + } + return { created: true, id }; + } catch (error) { + const removed = id + ? removeOwnedPath(ctx, target, id, "directory") + : recoverUnrecordedPrivilegedDirectory(ctx, target, "root:root 755"); + if (removed) { + if (markerPublished) safeUnlink(ctx, marker); + } else { + ctx.stderr.write( + `${label} creation failed before a complete receipt could be published; inspect ${target} before retrying.\n`, + ); + } + throw error; + } +} + +function createOwnedDropIn( + ctx: ProofContext, + target: string, + marker: string, + template: string, + content: string, + idEnv: string, + createdEnv: string, + stagingEnv: string, + stagingIdEnv: string, + stagingCreatedEnv: string, + recordState: (id: string, staging: string, stagingId: string) => void, +): void { + if (sudoExists(ctx, target)) throw new ProofError(`Proof drop-in already exists: ${target}`); + const targetDir = path.dirname(target); + const nonce = ctx.randomId(); + if (!/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u.test(nonce)) { + throw new ProofError("Random staging identifier has an unexpected format."); + } + const staging = path.join(targetDir, `${template.replace(/X+$/u, "")}${nonce}`); + const temporary = path.join(staging, "drop-in.conf"); + appendGithubEnv(ctx, stagingEnv, staging); + appendGithubEnv(ctx, stagingCreatedEnv, "unrecorded"); + let stagingId = ""; + let id = ""; + let markerPublished = false; + let published = false; + try { + checkedSudo(ctx, ["mkdir", "-m", "0700", "--", staging]); + stagingId = identity(ctx, staging); + recordState(id, staging, stagingId); + appendGithubEnv(ctx, stagingIdEnv, stagingId); + appendGithubEnv(ctx, stagingCreatedEnv, "1"); + checkedSudo(ctx, ["tee", temporary], { input: content }); + checkedSudo(ctx, ["chown", "root:root", "--", temporary]); + checkedSudo(ctx, ["chmod", "0644", "--", temporary]); + id = identity(ctx, temporary); + recordState(id, staging, stagingId); + appendGithubEnv(ctx, createdEnv, "unrecorded"); + appendGithubEnv(ctx, idEnv, id); + checkedSudo(ctx, ["mv", "--no-clobber", "--", temporary, target]); + if ( + sudoTest(ctx, ["-L", target]) || + !sudoTest(ctx, ["-f", target]) || + identity(ctx, target) !== id + ) { + throw new ProofError(`Proof drop-in publication did not create the expected file: ${target}`); + } + published = true; + writeMarker(ctx, marker, id); + markerPublished = true; + appendGithubEnv(ctx, createdEnv, "1"); + checkedSudo(ctx, ["rmdir", "--", staging]); + return; + } catch (error) { + const targetRemoved = !published || (id !== "" && removeOwnedPath(ctx, target, id, "file")); + const stagingRemoved = stagingId + ? removeOwnedTree(ctx, staging, stagingId) + : recoverUnrecordedPrivilegedDirectory(ctx, staging, "root:root 700"); + if (targetRemoved && stagingRemoved) { + if (markerPublished) safeUnlink(ctx, marker); + } else { + ctx.stderr.write( + `Proof drop-in publication rollback was incomplete for ${target}; ownership receipts were preserved.\n`, + ); + } + throw error; + } +} + +function emptyPreparedDropIn(): PreparedDropIn { + return { dirCreated: false, dirId: "", id: "", temp: "", tempId: "" }; +} + +function prepareOwnedDropIn( + ctx: ProofContext, + state: PreparedDropIn, + target: string, + marker: string, + directoryMarker: string, + environmentPrefix: string, + template: string, + content: string, +): void { + const directory = ensurePrivilegedDirectory( + ctx, + path.dirname(target), + directoryMarker, + `${environmentPrefix}_DROP_IN_DIR_CREATED`, + `${environmentPrefix}_DROP_IN_DIR_ID`, + `Proof drop-in directory ${path.dirname(target)}`, + (id) => { + state.dirCreated = true; + state.dirId = id; + }, + ); + state.dirCreated = directory.created; + state.dirId = directory.id; + createOwnedDropIn( + ctx, + target, + marker, + template, + content, + `${environmentPrefix}_DROP_IN_ID`, + `${environmentPrefix}_DROP_IN_CREATED`, + `${environmentPrefix}_DROP_IN_TEMP`, + `${environmentPrefix}_DROP_IN_TEMP_ID`, + `${environmentPrefix}_DROP_IN_TEMP_CREATED`, + (id, temporary, temporaryId) => { + state.id = id; + state.temp = temporary; + state.tempId = temporaryId; + }, + ); +} + +function redact(ctx: ProofContext, content: string): string { + return checked(ctx, "python3", ["test/e2e/lib/redact-text.py"], { + cwd: ctx.workspace, + input: content, + }).stdout; +} + +function persistDiagnostics(ctx: ProofContext, target: string, content: string): void { + checkedSudo(ctx, ["tee", target], { input: redact(ctx, content) }); +} + +function startUserManagerWithLaterLogin( + ctx: ProofContext, + uid: string, + user: string, + diagnostics: string, +): void { + const unit = `user@${uid}.service`; + if (sudo(ctx, ["systemctl", "start", unit]).status === 0) return; + const status = sudo(ctx, ["systemctl", "--no-pager", "--full", "status", unit]); + const journal = sudo(ctx, ["journalctl", "--no-pager", "--unit", unit, "--lines", "200"]); + persistDiagnostics( + ctx, + diagnostics, + `${status.stdout}${status.stderr}${journal.stdout}${journal.stderr}`, + ); + if ( + sudo(ctx, [ + "grep", + "-Fq", + PORTABLE_CPU_DELEGATION_PROOF_CONTRACT.immediateStartFailure, + diagnostics, + ]).status !== 0 + ) { + const captured = sudo(ctx, ["cat", "--", diagnostics]); + ctx.stderr.write(`${captured.stdout}${captured.stderr}`); + throw new ProofError(`Immediate start of ${unit} failed without 219/CGROUP.`); + } + sudo(ctx, ["loginctl", "terminate-user", user]); + checkedSudo(ctx, ["--login", "--user", user, "/bin/true"]); + for (let attempt = 0; attempt < 30; attempt += 1) { + if (sudo(ctx, ["systemctl", "is-active", "--quiet", unit]).status === 0) return; + ctx.sleep(1_000); + } + const captured = sudo(ctx, ["cat", "--", diagnostics]); + ctx.stderr.write(`${captured.stdout}${captured.stderr}`); + throw new ProofError(`Later login did not activate ${unit}.`); +} + +function appendInitialReceipts(ctx: ProofContext, userCommentValue: string): void { + const values: Readonly> = { + E2E_CPU_DELEGATION_USER_CLAIMED: "1", + E2E_CPU_DELEGATION_USER_COMMENT: userCommentValue, + E2E_CPU_DELEGATION_USER_CREATED: "0", + E2E_CPU_DELEGATION_DROP_IN_DIR: ctx.delegationDropInDir, + E2E_CPU_DELEGATION_DROP_IN_DIR_CREATED: "0", + E2E_CPU_DELEGATION_DROP_IN_DIR_ID: "", + E2E_CPU_DELEGATION_DROP_IN_DIR_MARKER: ctx.delegationDropInDirMarker, + E2E_CPU_DELEGATION_DROP_IN_MARKER: ctx.delegationDropInMarker, + E2E_CPU_DELEGATION_DROP_IN_CREATED: "0", + E2E_CPU_DELEGATION_DROP_IN_ID: "", + E2E_CPU_DELEGATION_DROP_IN_TEMP: "", + E2E_CPU_DELEGATION_DROP_IN_TEMP_CREATED: "0", + E2E_CPU_DELEGATION_DROP_IN_TEMP_ID: "", + E2E_APP_SLICE_DROP_IN: ctx.appSliceDropIn, + E2E_APP_SLICE_DROP_IN_DIR: ctx.appSliceDropInDir, + E2E_APP_SLICE_DROP_IN_DIR_CREATED: "0", + E2E_APP_SLICE_DROP_IN_DIR_ID: "", + E2E_APP_SLICE_DROP_IN_DIR_MARKER: ctx.appSliceDropInDirMarker, + E2E_APP_SLICE_DROP_IN_MARKER: ctx.appSliceDropInMarker, + E2E_APP_SLICE_DROP_IN_CREATED: "0", + E2E_APP_SLICE_DROP_IN_ID: "", + E2E_APP_SLICE_DROP_IN_TEMP: "", + E2E_APP_SLICE_DROP_IN_TEMP_CREATED: "0", + E2E_APP_SLICE_DROP_IN_TEMP_ID: "", + E2E_USER_SLICE_DROP_IN: "", + E2E_USER_SLICE_DROP_IN_DIR: "", + E2E_USER_SLICE_DROP_IN_DIR_CREATED: "0", + E2E_USER_SLICE_DROP_IN_DIR_ID: "", + E2E_USER_SLICE_DROP_IN_DIR_MARKER: ctx.userSliceDropInDirMarker, + E2E_USER_SLICE_DROP_IN_MARKER: ctx.userSliceDropInMarker, + E2E_USER_SLICE_DROP_IN_CREATED: "0", + E2E_USER_SLICE_DROP_IN_ID: "", + E2E_USER_SLICE_DROP_IN_TEMP: "", + E2E_USER_SLICE_DROP_IN_TEMP_CREATED: "0", + E2E_USER_SLICE_DROP_IN_TEMP_ID: "", + E2E_SOURCE_CACHE_DIR: ctx.sourceCacheDir, + E2E_SOURCE_CACHE_CREATED: "0", + E2E_SOURCE_CACHE_ID: "", + E2E_SOURCE_CACHE_MARKER: ctx.sourceCacheMarker, + E2E_SOURCE_CACHE_PARENT: ctx.sourceCacheParent, + E2E_SOURCE_CACHE_PARENT_CREATED: "0", + E2E_SOURCE_CACHE_PARENT_ID: "", + E2E_SOURCE_CACHE_PARENT_MARKER: ctx.sourceCacheParentMarker, + E2E_WORKSPACE_TRAVERSE_MARKER: ctx.workspaceTraverseMarker, + }; + for (const [name, value] of Object.entries(values)) appendGithubEnv(ctx, name, value); +} + +function prepareWorkspaceTraversal(ctx: ProofContext): void { + const ancestors: string[] = []; + let current = ctx.workspace; + while (current !== path.dirname(current)) { + ancestors.push(current); + current = path.dirname(current); + } + ctx.filesystem.writeExclusive(ctx.workspaceTraverseMarker, ""); + for (const workspacePath of ancestors.reverse()) { + if (sudo(ctx, ["--user", ctx.proofUser, "test", "-x", workspacePath]).status === 0) continue; + const originalMode = checked(ctx, "stat", ["-c", "%a", "--", workspacePath]).stdout.trim(); + if (!/^[0-7]{3,4}$/u.test(originalMode)) { + throw new ProofError(`Workspace mode is invalid for ${workspacePath}.`); + } + ctx.filesystem.appendText(ctx.workspaceTraverseMarker, `${originalMode}\t${workspacePath}\n`); + checkedSudo(ctx, ["chmod", "o+x", "--", workspacePath]); + } + if ( + sudo(ctx, [ + "--user", + ctx.proofUser, + "test", + "-x", + path.join(ctx.workspace, "node_modules", ".bin", "vitest"), + ]).status !== 0 + ) { + throw new ProofError("Dedicated proof user cannot execute the Vitest entrypoint."); + } +} + +function prepareSourceCacheParent(ctx: ProofContext, state: PrepareState): void { + try { + const parentStat = ctx.filesystem.lstat(ctx.sourceCacheParent); + if (parentStat.isSymbolicLink() || !parentStat.isDirectory()) { + throw new ProofError("Source-loader cache parent has an unexpected type."); + } + return; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + appendGithubEnv(ctx, "E2E_SOURCE_CACHE_PARENT_CREATED", "unrecorded"); + ctx.filesystem.makeDirectory(ctx.sourceCacheParent, { recursive: false, mode: 0o755 }); + try { + state.sourceCacheParentId = directIdentity(ctx, ctx.sourceCacheParent); + appendGithubEnv(ctx, "E2E_SOURCE_CACHE_PARENT_ID", state.sourceCacheParentId); + writeMarker(ctx, ctx.sourceCacheParentMarker, state.sourceCacheParentId); + appendGithubEnv(ctx, "E2E_SOURCE_CACHE_PARENT_CREATED", "1"); + } catch (error) { + const removed = state.sourceCacheParentId + ? removeDirectOwnedDirectory(ctx, ctx.sourceCacheParent, state.sourceCacheParentId) + : removeDirectUnrecordedDirectory(ctx, ctx.sourceCacheParent); + if (!removed) { + ctx.stderr.write( + `Source-loader cache parent creation failed before a complete receipt was published; inspect ${ctx.sourceCacheParent} before retrying.\n`, + ); + } + throw error; + } +} + +function prepareSourceCache(ctx: ProofContext, state: PrepareState): void { + prepareSourceCacheParent(ctx, state); + if (sudo(ctx, ["--user", ctx.proofUser, "test", "-x", ctx.sourceCacheParent]).status !== 0) { + throw new ProofError("Dedicated proof user cannot traverse the source-loader cache parent."); + } + if (sudoExists(ctx, ctx.sourceCacheDir)) { + throw new ProofError("Source-loader cache already exists."); + } + appendGithubEnv(ctx, "E2E_SOURCE_CACHE_CREATED", "unrecorded"); + checkedSudo(ctx, ["mkdir", "-m", "0700", "--", ctx.sourceCacheDir]); + let markerPublished = false; + try { + state.sourceCacheId = identity(ctx, ctx.sourceCacheDir); + appendGithubEnv(ctx, "E2E_SOURCE_CACHE_ID", state.sourceCacheId); + writeMarker(ctx, ctx.sourceCacheMarker, state.sourceCacheId); + markerPublished = true; + appendGithubEnv(ctx, "E2E_SOURCE_CACHE_CREATED", "1"); + checkedSudo(ctx, ["chown", `${state.uid}:${state.uid}`, "--", ctx.sourceCacheDir]); + } catch (error) { + const removed = state.sourceCacheId + ? removeOwnedPath(ctx, ctx.sourceCacheDir, state.sourceCacheId, "directory") + : recoverUnrecordedPrivilegedDirectory(ctx, ctx.sourceCacheDir, "root:root 700"); + if (removed) { + if (markerPublished) safeUnlink(ctx, ctx.sourceCacheMarker); + } else { + ctx.stderr.write( + `Source-loader cache creation failed before a complete receipt could be published; inspect ${ctx.sourceCacheDir} before retrying.\n`, + ); + } + throw error; + } +} + +function restoreWorkspaceModes(ctx: ProofContext): boolean { + if (!ctx.filesystem.exists(ctx.workspaceTraverseMarker)) return true; + let complete = true; + const records = ctx.filesystem.readText(ctx.workspaceTraverseMarker).split("\n").filter(Boolean); + for (const record of records) { + const tab = record.indexOf("\t"); + const mode = record.slice(0, tab); + const workspacePath = record.slice(tab + 1); + if (tab < 0 || !/^[0-7]{3,4}$/u.test(mode) || !path.isAbsolute(workspacePath)) { + complete = false; + continue; + } + if (sudo(ctx, ["chmod", mode, "--", workspacePath]).status !== 0) complete = false; + } + if (complete) ctx.filesystem.removeFile(ctx.workspaceTraverseMarker); + return complete; +} + +function removeOwnedPath( + ctx: ProofContext, + target: string, + expectedId: string, + kind: "file" | "directory", +): boolean { + if (!sudoExists(ctx, target)) return true; + const expectedType = kind === "file" ? "-f" : "-d"; + if (sudoTest(ctx, ["-L", target]) || !sudoTest(ctx, [expectedType, target])) return false; + if (identity(ctx, target) !== expectedId) return false; + const operation = kind === "file" ? ["rm", "-f", "--", target] : ["rmdir", "--", target]; + return sudo(ctx, operation).status === 0 && !sudoExists(ctx, target); +} + +function removeOwnedTree(ctx: ProofContext, target: string, expectedId: string): boolean { + if (!sudoExists(ctx, target)) return true; + if (sudoTest(ctx, ["-L", target]) || !sudoTest(ctx, ["-d", target])) return false; + if (identity(ctx, target) !== expectedId) return false; + return ( + sudo(ctx, ["rm", "-rf", "--one-file-system", "--", target]).status === 0 && + !sudoExists(ctx, target) + ); +} + +function recoverUnrecordedPrivilegedDirectory( + ctx: ProofContext, + target: string, + expectedOwnerMode: string, +): boolean { + if (!sudoExists(ctx, target)) return true; + if (sudoTest(ctx, ["-L", target]) || !sudoTest(ctx, ["-d", target])) return false; + if (ownerMode(ctx, target) !== expectedOwnerMode) return false; + const content = sudo(ctx, [ + "find", + target, + "-mindepth", + "1", + "-maxdepth", + "1", + "-print", + "-quit", + ]); + if (content.status !== 0 || content.stdout !== "") return false; + const firstId = identity(ctx, target); + if (identity(ctx, target) !== firstId) return false; + return sudo(ctx, ["rmdir", "--", target]).status === 0 && !sudoExists(ctx, target); +} + +function removeDirectOwnedDirectory( + ctx: ProofContext, + target: string, + expectedId: string, +): boolean { + try { + const value = ctx.filesystem.lstat(target); + if ( + value.isSymbolicLink() || + !value.isDirectory() || + directIdentity(ctx, target) !== expectedId + ) + return false; + ctx.filesystem.removeDirectory(target); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "ENOENT"; + } +} + +function removeDirectUnrecordedDirectory(ctx: ProofContext, target: string): boolean { + try { + const value = ctx.filesystem.lstat(target); + if (value.isSymbolicLink() || !value.isDirectory()) return false; + const firstId = directIdentity(ctx, target); + if (directIdentity(ctx, target) !== firstId) return false; + ctx.filesystem.removeDirectory(target); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "ENOENT"; + } +} + +function cleanupFailedPrepare( + ctx: ProofContext, + state: PrepareState, + expectedComment: string, +): void { + let complete = true; + if (state.createdUser && userExists(ctx) && state.uid) { + if (sudo(ctx, ["systemctl", "stop", `user@${state.uid}.service`]).status !== 0) + complete = false; + } + for (const dropIn of [state.delegation, state.appSlice, state.userSlice]) + if (dropIn.temp && (!dropIn.tempId || !removeOwnedTree(ctx, dropIn.temp, dropIn.tempId))) + complete = false; + if ( + state.sourceCacheId && + !removeOwnedPath(ctx, ctx.sourceCacheDir, state.sourceCacheId, "directory") + ) + complete = false; + if ( + state.sourceCacheParentId && + !removeDirectOwnedDirectory(ctx, ctx.sourceCacheParent, state.sourceCacheParentId) + ) + complete = false; + const userSlicePath = state.uid ? userSliceDropIn(state.uid) : ""; + for (const [dropIn, target] of [ + [state.delegation, ctx.delegationDropIn], + [state.appSlice, ctx.appSliceDropIn], + [state.userSlice, userSlicePath], + ] as const) + if (dropIn.id && !removeOwnedPath(ctx, target, dropIn.id, "file")) complete = false; + if ( + (state.delegation.id || state.appSlice.id || state.userSlice.id) && + sudo(ctx, ["systemctl", "daemon-reload"]).status !== 0 + ) + complete = false; + for (const [dropIn, target] of [ + [state.appSlice, ctx.appSliceDropInDir], + [state.delegation, ctx.delegationDropInDir], + [state.userSlice, userSlicePath ? path.dirname(userSlicePath) : ""], + ] as const) + if ( + dropIn.dirCreated && + dropIn.dirId && + !removeOwnedPath(ctx, target, dropIn.dirId, "directory") + ) + complete = false; + if (state.createdUser && userExists(ctx)) { + if (userComment(ctx) === expectedComment) { + if (sudo(ctx, ["loginctl", "disable-linger", ctx.proofUser]).status !== 0) complete = false; + sudo(ctx, ["loginctl", "terminate-user", ctx.proofUser]); + if (sudo(ctx, ["userdel", "--remove", ctx.proofUser]).status !== 0) complete = false; + } else { + complete = false; + } + } + if (!restoreWorkspaceModes(ctx)) complete = false; + if (!complete) + ctx.stderr.write( + "Preparation rollback was incomplete; final cleanup will retry recorded resources.\n", + ); +} + +function prepare(ctx: ProofContext): void { + requireNumeric(ctx.env.GITHUB_RUN_ID, "GITHUB_RUN_ID"); + requireNumeric(ctx.env.GITHUB_RUN_ATTEMPT, "GITHUB_RUN_ATTEMPT"); + const expectedComment = `nemoclaw-cpu-proof-${ctx.env.GITHUB_RUN_ID}-${ctx.env.GITHUB_RUN_ATTEMPT}`; + const state: PrepareState = { + appSlice: emptyPreparedDropIn(), + createdUser: false, + delegation: emptyPreparedDropIn(), + sourceCacheId: "", + sourceCacheParentId: "", + userSlice: emptyPreparedDropIn(), + uid: "", + }; + let complete = false; + try { + appendInitialReceipts(ctx, expectedComment); + if (userExists(ctx)) throw new ProofError("CPU delegation proof user already exists."); + for (const [target, label] of [ + [ctx.delegationDropIn, "CPU delegation proof drop-in"], + [ctx.appSliceDropIn, "app.slice proof drop-in"], + ] as const) { + if (sudoExists(ctx, target)) throw new ProofError(`${label} already exists.`); + } + for (const [marker, label] of [ + [ctx.delegationDropInMarker, "CPU delegation proof ownership marker"], + [ctx.delegationDropInDirMarker, "CPU delegation directory marker"], + [ctx.appSliceDropInMarker, "app.slice proof ownership marker"], + [ctx.appSliceDropInDirMarker, "app.slice directory marker"], + [ctx.userSliceDropInMarker, "per-user slice proof ownership marker"], + [ctx.userSliceDropInDirMarker, "per-user slice directory marker"], + [ctx.sourceCacheMarker, "source-cache ownership marker"], + [ctx.sourceCacheParentMarker, "source-cache parent ownership marker"], + [ctx.workspaceTraverseMarker, "workspace traversal receipt"], + ] as const) + ensureDirectAbsent(ctx, marker, label); + checkedSudo(ctx, [ + "useradd", + "--create-home", + "--shell", + "/bin/bash", + "--comment", + expectedComment, + ctx.proofUser, + ]); + state.createdUser = true; + appendGithubEnv(ctx, "E2E_CPU_DELEGATION_USER_CREATED", "1"); + state.uid = uidOf(ctx); + appendGithubEnv(ctx, "E2E_CPU_DELEGATION_HOME", userHome(ctx)); + appendGithubEnv(ctx, "E2E_CPU_DELEGATION_UID", state.uid); + appendGithubEnv(ctx, "E2E_CPU_DELEGATION_RUNTIME_DIR", `/run/user/${state.uid}`); + const userSlice = userSliceDropIn(state.uid); + const userSliceDir = path.dirname(userSlice); + appendGithubEnv(ctx, "E2E_USER_SLICE_DROP_IN", userSlice); + appendGithubEnv(ctx, "E2E_USER_SLICE_DROP_IN_DIR", userSliceDir); + if (sudoExists(ctx, userSlice)) throw new ProofError("Per-user slice proof drop-in exists."); + prepareWorkspaceTraversal(ctx); + prepareSourceCache(ctx, state); + checkedSudo(ctx, [ + "install", + "-d", + "-o", + state.uid, + "-g", + state.uid, + "-m", + "0700", + ctx.artifactDir, + ]); + prepareOwnedDropIn( + ctx, + state.userSlice, + userSlice, + ctx.userSliceDropInMarker, + ctx.userSliceDropInDirMarker, + "E2E_USER_SLICE", + ".nemoclaw-cpu-controller.XXXXXX", + PORTABLE_CPU_DELEGATION_PROOF_CONTRACT.userSliceDropInContent, + ); + prepareOwnedDropIn( + ctx, + state.appSlice, + ctx.appSliceDropIn, + ctx.appSliceDropInMarker, + ctx.appSliceDropInDirMarker, + "E2E_APP_SLICE", + ".nemoclaw-cpu-controller.XXXXXX", + PORTABLE_CPU_DELEGATION_PROOF_CONTRACT.appSliceDropInContent, + ); + prepareOwnedDropIn( + ctx, + state.delegation, + ctx.delegationDropIn, + ctx.delegationDropInMarker, + ctx.delegationDropInDirMarker, + "E2E_CPU_DELEGATION", + ".nemoclaw-cpu-delegation.XXXXXX", + PORTABLE_CPU_DELEGATION_PROOF_CONTRACT.missingDelegationDropInContent, + ); + checkedSudo(ctx, ["systemctl", "daemon-reload"]); + checkedSudo(ctx, ["loginctl", "enable-linger", ctx.proofUser]); + startUserManagerWithLaterLogin( + ctx, + state.uid, + ctx.proofUser, + path.join(ctx.artifactDir, "prepare-user-manager-diagnostics.txt"), + ); + complete = true; + } finally { + if (!complete) cleanupFailedPrepare(ctx, state, expectedComment); + } +} + +function proofArguments(ctx: ProofContext, state: "missing" | "delegated"): readonly string[] { + const uid = requireNumeric(ctx.env.E2E_CPU_DELEGATION_UID, "E2E_CPU_DELEGATION_UID"); + const home = requireAbsolutePath(ctx.env.E2E_CPU_DELEGATION_HOME, "E2E_CPU_DELEGATION_HOME"); + const runtimeDir = requireAbsolutePath( + ctx.env.E2E_CPU_DELEGATION_RUNTIME_DIR, + "E2E_CPU_DELEGATION_RUNTIME_DIR", + ); + return [ + "--user", + ctx.proofUser, + "env", + "-i", + `E2E_ARTIFACT_DIR=${ctx.artifactDir}`, + `E2E_CPU_DELEGATION_STATE=${state}`, + `E2E_CPU_DELEGATION_UID=${uid}`, + `E2E_SOURCE_REVISION=${requireSha(ctx.env.E2E_SOURCE_REVISION)}`, + `E2E_TARGET_ID=${PORTABLE_CPU_DELEGATION_PROOF_CONTRACT.targetId}`, + `HOME=${home}`, + `NEMOCLAW_RUN_LIVE_E2E=${requireExact(ctx.env.NEMOCLAW_RUN_LIVE_E2E, "1", "NEMOCLAW_RUN_LIVE_E2E")}`, + `PATH=${ctx.env.PATH ?? ""}`, + `XDG_RUNTIME_DIR=${runtimeDir}`, + "./node_modules/.bin/vitest", + "run", + "--no-cache", + "--project", + "e2e-live", + "test/e2e/live/portable-cpu-delegation-proof.test.ts", + ]; +} + +function runProof(ctx: ProofContext, state: "missing" | "delegated"): void { + checkedSudo(ctx, proofArguments(ctx, state), { cwd: ctx.workspace }); +} + +function expectedMarkerPath(ctx: ProofContext, envName: string, expected: string): void { + if (ctx.env[envName] !== expected) + throw new ProofError(`${envName} does not match its fixed receipt path.`); +} + +function userSliceReceiptPaths(ctx: ProofContext) { + const dropIn = userSliceDropIn( + requireNumeric(ctx.env.E2E_CPU_DELEGATION_UID, "E2E_CPU_DELEGATION_UID"), + ); + const dropInDir = path.dirname(dropIn); + for (const [name, expected] of [ + ["E2E_USER_SLICE_DROP_IN", dropIn], + ["E2E_USER_SLICE_DROP_IN_DIR", dropInDir], + ["E2E_USER_SLICE_DROP_IN_MARKER", ctx.userSliceDropInMarker], + ["E2E_USER_SLICE_DROP_IN_DIR_MARKER", ctx.userSliceDropInDirMarker], + ] as const) + expectedMarkerPath(ctx, name, expected); + return { dropIn, dropInDir }; +} + +function validateOwnedDropIn( + ctx: ProofContext, + target: string, + marker: string, + label: string, +): void { + const expectedId = readMarker(ctx, marker); + if (!/^[0-9]+:[0-9]+$/u.test(expectedId)) + throw new ProofError(`${label} ownership marker is invalid.`); + if ( + sudoTest(ctx, ["-L", target]) || + !sudoTest(ctx, ["-f", target]) || + identity(ctx, target) !== expectedId + ) { + throw new ProofError(`${label} identity changed before admission.`); + } +} + +function admit(ctx: ProofContext): void { + const userSlice = userSliceReceiptPaths(ctx); + expectedMarkerPath(ctx, "E2E_CPU_DELEGATION_DROP_IN_MARKER", ctx.delegationDropInMarker); + expectedMarkerPath(ctx, "E2E_APP_SLICE_DROP_IN", ctx.appSliceDropIn); + expectedMarkerPath(ctx, "E2E_APP_SLICE_DROP_IN_MARKER", ctx.appSliceDropInMarker); + validateOwnedDropIn( + ctx, + ctx.delegationDropIn, + ctx.delegationDropInMarker, + "CPU delegation proof drop-in", + ); + validateOwnedDropIn(ctx, ctx.appSliceDropIn, ctx.appSliceDropInMarker, "app.slice proof drop-in"); + validateOwnedDropIn( + ctx, + userSlice.dropIn, + ctx.userSliceDropInMarker, + "per-user slice proof drop-in", + ); + const uid = requireNumeric(ctx.env.E2E_CPU_DELEGATION_UID, "E2E_CPU_DELEGATION_UID"); + checkedSudo(ctx, ["systemctl", "stop", `user@${uid}.service`]); + checkedSudo(ctx, ["tee", ctx.delegationDropIn], { + input: PORTABLE_CPU_DELEGATION_PROOF_CONTRACT.delegationDropInContent, + }); + checkedSudo(ctx, ["systemctl", "daemon-reload"]); + startUserManagerWithLaterLogin( + ctx, + uid, + ctx.proofUser, + path.join(ctx.artifactDir, "admission-user-manager-diagnostics.txt"), + ); + runProof(ctx, "delegated"); +} + +function diagnostics(ctx: ProofContext): void { + checkedSudo(ctx, ["install", "-d", "-m", "0700", ctx.artifactDir]); + let content = `source_revision=${requireSha(ctx.env.E2E_SOURCE_REVISION)}\nproof_uid=${ctx.env.E2E_CPU_DELEGATION_UID ?? "unset"}\n`; + const uid = ctx.env.E2E_CPU_DELEGATION_UID; + if (uid && /^[0-9]+$/u.test(uid)) { + const unit = `user@${uid}.service`; + for (const [executable, argv] of [ + ["sudo", ["systemctl", "--no-pager", "--full", "status", unit]], + ["sudo", ["journalctl", "--no-pager", "--unit", unit, "--lines", "200"]], + [ + "sudo", + [ + "find", + `/sys/fs/cgroup/user.slice/user-${uid}.slice`, + "-maxdepth", + "3", + "(", + "-name", + "cgroup.controllers", + "-o", + "-name", + "cgroup.subtree_control", + ")", + "-print", + "-exec", + "cat", + "{}", + ";", + ], + ], + ] as const) { + const result = run(ctx, executable, argv); + content += `${result.stdout}${result.stderr}`; + } + } + const target = path.join(ctx.artifactDir, "user-manager-diagnostics.txt"); + persistDiagnostics(ctx, target, content); + checkedSudo(ctx, ["chmod", "0600", target]); +} + +function safeUnlink(ctx: ProofContext, target: string): boolean { + try { + ctx.filesystem.removeFile(target); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "ENOENT"; + } +} + +type OwnershipReceipt = { + readonly complete: boolean; + readonly id: string; + readonly markerOwned: boolean; +}; + +type CreationState = "0" | "1" | "unrecorded"; + +function creationState(ctx: ProofContext, name: string): CreationState | "invalid" { + const value = ctx.env[name] ?? "0"; + return value === "0" || value === "1" || value === "unrecorded" ? value : "invalid"; +} + +function cleanupUserSliceReceiptPaths(ctx: ProofContext) { + const noCreatedObjects = [ + "E2E_USER_SLICE_DROP_IN_CREATED", + "E2E_USER_SLICE_DROP_IN_DIR_CREATED", + "E2E_USER_SLICE_DROP_IN_TEMP_CREATED", + ].every((name) => creationState(ctx, name) === "0"); + if (noCreatedObjects && !ctx.env.E2E_USER_SLICE_DROP_IN && !ctx.env.E2E_USER_SLICE_DROP_IN_DIR) + return undefined; + return userSliceReceiptPaths(ctx); +} + +function ownershipReceipt(ctx: ProofContext, marker: string, idEnv: string): OwnershipReceipt { + const environmentId = ctx.env[idEnv] ?? ""; + const environmentIdValid = /^[0-9]+:[0-9]+$/u.test(environmentId); + if (!ctx.filesystem.exists(marker)) { + return environmentId === "" || environmentIdValid + ? { complete: true, id: environmentId, markerOwned: false } + : { complete: false, id: "", markerOwned: false }; + } + const markerId = readMarker(ctx, marker); + const markerIdValid = /^[0-9]+:[0-9]+$/u.test(markerId); + if (markerIdValid && (environmentId === "" || markerId === environmentId)) { + return { complete: true, id: markerId, markerOwned: true }; + } + return environmentIdValid + ? { complete: false, id: environmentId, markerOwned: false } + : { complete: false, id: "", markerOwned: false }; +} + +function removeFromReceipt( + ctx: ProofContext, + target: string, + marker: string, + idEnv: string, + createdEnv: string, + kind: "file" | "directory", + unrecordedOwnerMode = "", +): boolean { + const state = creationState(ctx, createdEnv); + const receipt = ownershipReceipt(ctx, marker, idEnv); + if (state === "invalid") return false; + if (state === "0") return receipt.id === "" && receipt.complete; + if (receipt.id === "") { + return state === "unrecorded" + ? kind === "file" + ? !sudoExists(ctx, target) + : recoverUnrecordedPrivilegedDirectory(ctx, target, unrecordedOwnerMode) + : false; + } + if (!removeOwnedPath(ctx, target, receipt.id, kind)) return false; + if (receipt.markerOwned && !safeUnlink(ctx, marker)) return false; + return receipt.complete; +} + +function cleanupSourceCache(ctx: ProofContext): boolean { + const state = creationState(ctx, "E2E_SOURCE_CACHE_CREATED"); + const receipt = ownershipReceipt(ctx, ctx.sourceCacheMarker, "E2E_SOURCE_CACHE_ID"); + if (state === "invalid") return false; + if (state === "0") return receipt.id === "" && receipt.complete; + if (receipt.id === "") { + return state === "unrecorded" + ? recoverUnrecordedPrivilegedDirectory(ctx, ctx.sourceCacheDir, "root:root 700") + : false; + } + if (!sudoExists(ctx, ctx.sourceCacheDir)) { + if (receipt.markerOwned && !safeUnlink(ctx, ctx.sourceCacheMarker)) return false; + return receipt.complete; + } + if (sudoTest(ctx, ["-L", ctx.sourceCacheDir]) || !sudoTest(ctx, ["-d", ctx.sourceCacheDir])) + return false; + if (identity(ctx, ctx.sourceCacheDir) !== receipt.id) return false; + if ( + sudo(ctx, ["rm", "-rf", "--one-file-system", "--", ctx.sourceCacheDir]).status !== 0 || + sudoExists(ctx, ctx.sourceCacheDir) + ) + return false; + if (receipt.markerOwned && !safeUnlink(ctx, ctx.sourceCacheMarker)) return false; + return receipt.complete; +} + +function cleanupSourceCacheParent(ctx: ProofContext): boolean { + const state = creationState(ctx, "E2E_SOURCE_CACHE_PARENT_CREATED"); + const receipt = ownershipReceipt(ctx, ctx.sourceCacheParentMarker, "E2E_SOURCE_CACHE_PARENT_ID"); + if (state === "invalid") return false; + if (state === "0") return receipt.id === "" && receipt.complete; + if (receipt.id === "") { + return state === "unrecorded" + ? removeDirectUnrecordedDirectory(ctx, ctx.sourceCacheParent) + : false; + } + if (!removeDirectOwnedDirectory(ctx, ctx.sourceCacheParent, receipt.id)) return false; + if (receipt.markerOwned && !safeUnlink(ctx, ctx.sourceCacheParentMarker)) return false; + return receipt.complete; +} + +function cleanupTemporary( + ctx: ProofContext, + pathEnv: string, + idEnv: string, + createdEnv: string, + targetDirectory: string, + namePrefix: string, +): boolean { + const state = creationState(ctx, createdEnv); + const temporary = ctx.env[pathEnv] ?? ""; + const expectedId = ctx.env[idEnv] ?? ""; + if (state === "invalid") return false; + if (state === "0") return temporary === "" && expectedId === ""; + if ( + !path.isAbsolute(temporary) || + path.dirname(temporary) !== targetDirectory || + !path.basename(temporary).startsWith(namePrefix) || + !/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u.test( + path.basename(temporary).slice(namePrefix.length), + ) + ) + return false; + if (expectedId === "") { + return state === "unrecorded" + ? recoverUnrecordedPrivilegedDirectory(ctx, temporary, "root:root 700") + : false; + } + if (!/^[0-9]+:[0-9]+$/u.test(expectedId)) return false; + return removeOwnedTree(ctx, temporary, expectedId); +} + +function cleanup(ctx: ProofContext): void { + let complete = true; + let userSlice: ReturnType; + try { + userSlice = cleanupUserSliceReceiptPaths(ctx); + } catch (error) { + ctx.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + complete = false; + } + const claimed = ctx.env.E2E_CPU_DELEGATION_USER_CLAIMED === "1"; + const expectedComment = ctx.env.E2E_CPU_DELEGATION_USER_COMMENT ?? ""; + let ownedUser = false; + let uid = ""; + if (claimed && userExists(ctx)) { + if (expectedComment && userComment(ctx) === expectedComment) { + ownedUser = true; + uid = uidOf(ctx); + if (sudo(ctx, ["systemctl", "stop", `user@${uid}.service`]).status !== 0) complete = false; + } else { + complete = false; + } + } + const temporaries: [string, string, string, string, string][] = [ + [ + "E2E_CPU_DELEGATION_DROP_IN_TEMP", + "E2E_CPU_DELEGATION_DROP_IN_TEMP_ID", + "E2E_CPU_DELEGATION_DROP_IN_TEMP_CREATED", + ctx.delegationDropInDir, + ".nemoclaw-cpu-delegation.", + ], + [ + "E2E_APP_SLICE_DROP_IN_TEMP", + "E2E_APP_SLICE_DROP_IN_TEMP_ID", + "E2E_APP_SLICE_DROP_IN_TEMP_CREATED", + ctx.appSliceDropInDir, + ".nemoclaw-cpu-controller.", + ], + ]; + if (userSlice) + temporaries.push([ + "E2E_USER_SLICE_DROP_IN_TEMP", + "E2E_USER_SLICE_DROP_IN_TEMP_ID", + "E2E_USER_SLICE_DROP_IN_TEMP_CREATED", + userSlice.dropInDir, + ".nemoclaw-cpu-controller.", + ]); + for (const temporary of temporaries) if (!cleanupTemporary(ctx, ...temporary)) complete = false; + const dropIns: [string, string, string, string][] = [ + [ + ctx.delegationDropIn, + ctx.delegationDropInMarker, + "E2E_CPU_DELEGATION_DROP_IN_ID", + "E2E_CPU_DELEGATION_DROP_IN_CREATED", + ], + [ + ctx.appSliceDropIn, + ctx.appSliceDropInMarker, + "E2E_APP_SLICE_DROP_IN_ID", + "E2E_APP_SLICE_DROP_IN_CREATED", + ], + ]; + if (userSlice) + dropIns.push([ + userSlice.dropIn, + ctx.userSliceDropInMarker, + "E2E_USER_SLICE_DROP_IN_ID", + "E2E_USER_SLICE_DROP_IN_CREATED", + ]); + for (const dropIn of dropIns) if (!removeFromReceipt(ctx, ...dropIn, "file")) complete = false; + if (!cleanupSourceCache(ctx)) complete = false; + if (!cleanupSourceCacheParent(ctx)) complete = false; + if (sudo(ctx, ["systemctl", "daemon-reload"]).status !== 0) complete = false; + const directories: [string, string, string, string][] = [ + [ + ctx.appSliceDropInDir, + ctx.appSliceDropInDirMarker, + "E2E_APP_SLICE_DROP_IN_DIR_ID", + "E2E_APP_SLICE_DROP_IN_DIR_CREATED", + ], + [ + ctx.delegationDropInDir, + ctx.delegationDropInDirMarker, + "E2E_CPU_DELEGATION_DROP_IN_DIR_ID", + "E2E_CPU_DELEGATION_DROP_IN_DIR_CREATED", + ], + ]; + if (userSlice) + directories.push([ + userSlice.dropInDir, + ctx.userSliceDropInDirMarker, + "E2E_USER_SLICE_DROP_IN_DIR_ID", + "E2E_USER_SLICE_DROP_IN_DIR_CREATED", + ]); + for (const directory of directories) + if (!removeFromReceipt(ctx, ...directory, "directory", "root:root 755")) complete = false; + if (ownedUser && userExists(ctx)) { + if (sudo(ctx, ["loginctl", "disable-linger", ctx.proofUser]).status !== 0) complete = false; + sudo(ctx, ["loginctl", "terminate-user", ctx.proofUser]); + if (sudo(ctx, ["userdel", "--remove", ctx.proofUser]).status !== 0 || userExists(ctx)) + complete = false; + } + if ( + ctx.filesystem.exists(ctx.artifactDir) && + sudo(ctx, [ + "chown", + "-R", + `${String(process.getuid?.() ?? 0)}:${String(process.getgid?.() ?? 0)}`, + ctx.artifactDir, + ]).status !== 0 + ) + complete = false; + if (!restoreWorkspaceModes(ctx)) complete = false; + if (!complete) + throw new ProofError("CPU delegation proof cleanup was incomplete; receipts were preserved."); +} + +export function parsePortableCpuDelegationProofMode( + argv: readonly string[], +): PortableCpuDelegationProofMode { + if (argv.length !== 1 || !MODES.includes(argv[0] as PortableCpuDelegationProofMode)) { + throw new ProofError(`Expected exactly one mode: ${MODES.join(" | ")}.`); + } + return argv[0] as PortableCpuDelegationProofMode; +} + +export function runPortableCpuDelegationProofMode( + mode: PortableCpuDelegationProofMode, + deps: PortableCpuDelegationProofDeps = {}, +): void { + const ctx = context(deps); + const modes: Record void> = { + prepare, + reject: (value) => runProof(value, "missing"), + admit, + diagnostics, + cleanup, + }; + modes[mode](ctx); +} + +export function portableCpuDelegationProofCli( + argv: readonly string[] = process.argv.slice(2), + deps: PortableCpuDelegationProofDeps = {}, +): void { + runPortableCpuDelegationProofMode(parsePortableCpuDelegationProofMode(argv), deps); +} + +function installSupplementalSignalHandlers(): void { + for (const signal of ["SIGINT", "SIGTERM"] as const) { + process.once(signal, () => { + process.exitCode = signal === "SIGINT" ? 130 : 143; + }); + } +} + +const invokedPath = process.argv[1] ? pathToFileURL(path.resolve(process.argv[1])).href : ""; +if (import.meta.url === invokedPath) { + installSupplementalSignalHandlers(); + try { + portableCpuDelegationProofCli(); + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 1; + } +} diff --git a/src/lib/onboard/experimental/portable-cpu-delegation-preflight.test.ts b/src/lib/onboard/experimental/portable-cpu-delegation-preflight.test.ts new file mode 100644 index 00000000000..05c98288dfc --- /dev/null +++ b/src/lib/onboard/experimental/portable-cpu-delegation-preflight.test.ts @@ -0,0 +1,375 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; +import { + cpuDelegationControllerPaths, + inspectPortableCpuDelegation, + portableCpuDelegationError, +} from "./portable-cpu-delegation-preflight"; + +function files( + contents: Record, +): (file: string, maxBytes: number) => Buffer { + return (file: string, maxBytes: number) => { + const value = + contents[file] ?? + (() => { + throw Object.assign(new Error(`ENOENT: no such file or directory, open '${file}'`), { + code: "ENOENT", + }); + })(); + const bytes = Buffer.isBuffer(value) ? value : Buffer.from(value, "utf8"); + return bytes.subarray(0, maxBytes); + }; +} + +function unreadableAt( + unreadableFile: string, + contents: Record, +): (file: string, maxBytes: number) => Buffer { + const readFile = files(contents); + const throwUnreadable = (file: string): never => { + throw Object.assign(new Error(`EACCES: permission denied, open '${file}'`), { + code: "EACCES", + }); + }; + return (file: string, maxBytes: number) => + file === unreadableFile ? throwUnreadable(file) : readFile(file, maxBytes); +} + +const UID = 1001; +const PATHS = cpuDelegationControllerPaths(UID); + +const CPU_FULL = "cpuset cpu io memory pids"; +const NO_CPU = "cpuset io memory pids"; +const MALFORMED = "cpu memory\nDelegate=cpu"; + +function expectManagerInterruptionGuidance(detail: string): void { + const saveWork = detail.indexOf("Save the current user's work"); + const stopManager = detail.indexOf("stops the user manager"); + const startFailure = detail.indexOf("start fails with 219/CGROUP"); + const laterLogin = detail.indexOf("start a later login session"); + const saveHostWork = detail.indexOf("save every user's work first"); + const reboot = detail.indexOf("reboot the host"); + + expect(saveWork).toBeGreaterThanOrEqual(0); + expect(saveWork).toBeLessThan(stopManager); + expect(startFailure).toBeGreaterThan(stopManager); + expect(laterLogin).toBeGreaterThan(startFailure); + expect(saveHostWork).toBeGreaterThan(laterLogin); + expect(saveHostWork).toBeLessThan(reboot); +} + +function expectThreeCpuControllerSettings(detail: string): void { + expect(detail).toContain("`CPUWeight=100` for `user-1001.slice`"); + expect(detail).toContain("`Delegate=cpu memory pids` for `user@.service`"); + expect(detail).toContain("`CPUWeight=100` for `app.slice`"); +} + +describe("inspectPortableCpuDelegation", () => { + it("skips the check on non-Linux platforms", () => { + const preflight = inspectPortableCpuDelegation({ + platform: "darwin", + uid: UID, + }); + expect(preflight.ok).toBe(true); + }); + + it("reports cgroups v2 unavailable when the root controllers file is missing", () => { + const preflight = inspectPortableCpuDelegation({ + platform: "linux", + uid: UID, + readControllerFileSync: files({}), + }); + expect(preflight.ok).toBe(false); + expect(preflight.failure).toBe("cgroups-v2-unavailable"); + expect(preflight.detail).toContain("cgroups v2"); + expect(preflight.detail.indexOf("save every user's work")).toBeLessThan( + preflight.detail.indexOf("Boot a cgroups v2 host"), + ); + }); + + it("reports access recovery when the root controllers file is unreadable", () => { + const preflight = inspectPortableCpuDelegation({ + platform: "linux", + uid: UID, + readControllerFileSync: unreadableAt(PATHS.root, {}), + }); + expect(preflight.ok).toBe(false); + expect(preflight.failure).toBe("cgroup-controllers-unreadable"); + expect(preflight.detail).toContain("EACCES"); + expect(preflight.detail).toContain("mount permissions"); + expect(preflight.detail).toContain("security policy"); + expect(preflight.detail).not.toContain("Boot a cgroups v2 host"); + }); + + it("reports when the kernel hierarchy does not expose the cpu controller", () => { + const preflight = inspectPortableCpuDelegation({ + platform: "linux", + uid: UID, + readControllerFileSync: files({ + [PATHS.root]: NO_CPU, + [PATHS.userManager]: CPU_FULL, + [PATHS.appSlice]: CPU_FULL, + }), + }); + expect(preflight.ok).toBe(false); + expect(preflight.failure).toBe("cpu-controller-unavailable"); + expect(preflight.detail).toContain('no "cpu"'); + expect(preflight.detail.indexOf("save every user's work")).toBeLessThan( + preflight.detail.indexOf("Enable the cpu controller"), + ); + }); + + it("reports when systemd did not delegate cpu to the user manager (missing file)", () => { + const preflight = inspectPortableCpuDelegation({ + platform: "linux", + uid: UID, + readControllerFileSync: files({ + [PATHS.root]: CPU_FULL, + [PATHS.userSlice]: CPU_FULL, + }), + }); + expect(preflight.ok).toBe(false); + expect(preflight.failure).toBe("systemd-user-delegation-missing"); + expectThreeCpuControllerSettings(preflight.detail); + expectManagerInterruptionGuidance(preflight.detail); + }); + + it.each([ + ["is missing", files({ [PATHS.root]: CPU_FULL })], + ["does not expose cpu", files({ [PATHS.root]: CPU_FULL, [PATHS.userSlice]: NO_CPU })], + ] as const)( + "reports when the per-user systemd slice %s (#9188)", + (_condition, readControllerFile) => { + const readControllerFileSync = vi.fn(readControllerFile); + const preflight = inspectPortableCpuDelegation({ + platform: "linux", + uid: UID, + readControllerFileSync, + }); + + expect(preflight.ok).toBe(false); + expect(preflight.failure).toBe("systemd-user-slice-cpu-unavailable"); + expect(preflight.detail).toContain(PATHS.userSlice); + expectThreeCpuControllerSettings(preflight.detail); + expectManagerInterruptionGuidance(preflight.detail); + expect(readControllerFileSync.mock.calls).toEqual([ + [PATHS.root, 4097], + [PATHS.userSlice, 4097], + ]); + }, + ); + + it("reports access recovery when the per-user slice evidence is unreadable (#9188)", () => { + const readControllerFileSync = vi.fn(unreadableAt(PATHS.userSlice, { [PATHS.root]: CPU_FULL })); + const preflight = inspectPortableCpuDelegation({ + platform: "linux", + uid: UID, + readControllerFileSync, + }); + + expect(preflight.ok).toBe(false); + expect(preflight.failure).toBe("cgroup-controllers-unreadable"); + expect(preflight.detail).toContain(PATHS.userSlice); + expect(preflight.detail).toContain("Do not change systemd delegation"); + expect(preflight.detail).not.toContain("CPUWeight=100"); + expect(readControllerFileSync.mock.calls).toEqual([ + [PATHS.root, 4097], + [PATHS.userSlice, 4097], + ]); + }); + + it("reports access recovery when the user manager controllers file is unreadable", () => { + const preflight = inspectPortableCpuDelegation({ + platform: "linux", + uid: UID, + readControllerFileSync: unreadableAt(PATHS.userManager, { + [PATHS.root]: CPU_FULL, + [PATHS.userSlice]: CPU_FULL, + }), + }); + expect(preflight.ok).toBe(false); + expect(preflight.failure).toBe("cgroup-controllers-unreadable"); + expect(preflight.detail).toContain("EACCES"); + expect(preflight.detail).toContain("Do not change systemd delegation"); + expect(preflight.detail).not.toContain("restart the user manager"); + }); + + it("reports when systemd did not delegate cpu to the user manager (no cpu token)", () => { + const preflight = inspectPortableCpuDelegation({ + platform: "linux", + uid: UID, + readControllerFileSync: files({ + [PATHS.root]: CPU_FULL, + [PATHS.userSlice]: CPU_FULL, + [PATHS.userManager]: NO_CPU, + [PATHS.appSlice]: CPU_FULL, + }), + }); + expect(preflight.ok).toBe(false); + expect(preflight.failure).toBe("systemd-user-delegation-missing"); + expectThreeCpuControllerSettings(preflight.detail); + expectManagerInterruptionGuidance(preflight.detail); + }); + + it("reports when the cpu controller is not available to app.slice for this boot", () => { + const preflight = inspectPortableCpuDelegation({ + platform: "linux", + uid: UID, + readControllerFileSync: files({ + [PATHS.root]: CPU_FULL, + [PATHS.userSlice]: CPU_FULL, + [PATHS.userManager]: CPU_FULL, + [PATHS.appSlice]: NO_CPU, + }), + }); + expect(preflight.ok).toBe(false); + expect(preflight.failure).toBe("app-slice-cpu-unavailable"); + expect(preflight.detail).toContain("app.slice"); + expect(preflight.detail).toContain("CPU controller setting"); + expectManagerInterruptionGuidance(preflight.detail); + }); + + it("reports when the app.slice controllers file is missing", () => { + const preflight = inspectPortableCpuDelegation({ + platform: "linux", + uid: UID, + readControllerFileSync: files({ + [PATHS.root]: CPU_FULL, + [PATHS.userSlice]: CPU_FULL, + [PATHS.userManager]: CPU_FULL, + }), + }); + expect(preflight.ok).toBe(false); + expect(preflight.failure).toBe("app-slice-cpu-unavailable"); + expect(preflight.detail).toContain("CPU controller setting"); + expectManagerInterruptionGuidance(preflight.detail); + }); + + it("reports access recovery when the app.slice controllers file is unreadable", () => { + const preflight = inspectPortableCpuDelegation({ + platform: "linux", + uid: UID, + readControllerFileSync: unreadableAt(PATHS.appSlice, { + [PATHS.root]: CPU_FULL, + [PATHS.userSlice]: CPU_FULL, + [PATHS.userManager]: CPU_FULL, + }), + }); + expect(preflight.ok).toBe(false); + expect(preflight.failure).toBe("cgroup-controllers-unreadable"); + expect(preflight.detail).toContain("EACCES"); + expect(preflight.detail).toContain("Do not change systemd delegation"); + expect(preflight.detail).not.toContain("Restart the user manager"); + }); + + it.each([ + ["root", PATHS.root, {}], + ["per-user slice", PATHS.userSlice, { [PATHS.root]: CPU_FULL }], + ["user manager", PATHS.userManager, { [PATHS.root]: CPU_FULL, [PATHS.userSlice]: CPU_FULL }], + [ + "app.slice", + PATHS.appSlice, + { + [PATHS.root]: CPU_FULL, + [PATHS.userSlice]: CPU_FULL, + [PATHS.userManager]: CPU_FULL, + }, + ], + ])( + "rejects malformed %s controller evidence before remediation (#9188)", + (_name, path, prefix) => { + const preflight = inspectPortableCpuDelegation({ + platform: "linux", + uid: UID, + readControllerFileSync: files({ ...prefix, [path]: MALFORMED }), + }); + + expect(preflight.ok).toBe(false); + expect(preflight.failure).toBe("cgroup-controllers-malformed"); + expect(preflight.detail).toContain(path); + expect(preflight.detail).toContain("evidence is malformed"); + expect(preflight.detail).not.toContain(MALFORMED); + expect(preflight.detail).not.toContain("Delegate=cpu memory pids"); + expect(preflight.detail).not.toContain("CPUWeight=100"); + expect(preflight.detail).not.toContain("stop and start"); + }, + ); + + it.each([ + ["NUL bytes", Buffer.from("cpu\0memory", "utf8")], + ["oversized content", Buffer.alloc(4097, 0x61)], + ["duplicate controller names", "cpu cpu memory"], + ])("rejects %s as malformed controller evidence (#9188)", (_case, content) => { + const preflight = inspectPortableCpuDelegation({ + platform: "linux", + uid: UID, + readControllerFileSync: files({ [PATHS.root]: content }), + }); + + expect(preflight.ok).toBe(false); + expect(preflight.failure).toBe("cgroup-controllers-malformed"); + expect(preflight.detail).toContain(PATHS.root); + expect(preflight.detail).not.toContain("Delegate=cpu memory pids"); + expect(preflight.detail).not.toContain("CPUWeight=100"); + }); + + it("caps each controller read at the evidence limit plus one sentinel byte (#9188)", () => { + const readControllerFileSync = vi.fn( + files({ + [PATHS.root]: CPU_FULL, + [PATHS.userSlice]: CPU_FULL, + [PATHS.userManager]: CPU_FULL, + [PATHS.appSlice]: CPU_FULL, + }), + ); + + expect( + inspectPortableCpuDelegation({ platform: "linux", uid: UID, readControllerFileSync }).ok, + ).toBe(true); + expect(readControllerFileSync.mock.calls).toEqual([ + [PATHS.root, 4097], + [PATHS.userSlice, 4097], + [PATHS.userManager, 4097], + [PATHS.appSlice, 4097], + ]); + }); + + it("passes when cpu is delegated through the whole current-user hierarchy", () => { + const preflight = inspectPortableCpuDelegation({ + platform: "linux", + uid: UID, + readControllerFileSync: files({ + [PATHS.root]: CPU_FULL, + [PATHS.userSlice]: CPU_FULL, + [PATHS.userManager]: CPU_FULL, + [PATHS.appSlice]: CPU_FULL, + }), + }); + expect(preflight.ok).toBe(true); + expect(preflight.failure).toBeUndefined(); + expect(preflight.detail).toContain("cpu controller"); + }); + + it("skips when the user id cannot be resolved", () => { + const preflight = inspectPortableCpuDelegation({ + platform: "linux", + uid: Number.NaN, + readControllerFileSync: files({}), + }); + expect(preflight.ok).toBe(true); + }); + + it("formats a throwable error from a failed inspection", () => { + const preflight = inspectPortableCpuDelegation({ + platform: "linux", + uid: UID, + readControllerFileSync: files({}), + }); + const error = portableCpuDelegationError(preflight); + expect(error.message).toContain("Portable CPU-delegation preflight failed"); + expect(error.message).toContain("cgroups v2"); + }); +}); diff --git a/src/lib/onboard/experimental/portable-cpu-delegation-preflight.ts b/src/lib/onboard/experimental/portable-cpu-delegation-preflight.ts new file mode 100644 index 00000000000..7718493f621 --- /dev/null +++ b/src/lib/onboard/experimental/portable-cpu-delegation-preflight.ts @@ -0,0 +1,383 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Portable admission must know whether the current user's systemd/cgroup +// hierarchy can actually enforce the sandbox CPU limit before any sandbox +// build or creation. OpenShell applies the limit through rootless Podman, +// which needs the `cpu` controller delegated down to the current user's +// `app.slice`. The stock systemd `user@.service` delegates only `pids memory`, +// so a host can pass the generic rootless-Podman checks and still fail at +// sandbox creation (gh #9188). +// +// The check is deliberately credential-free and read-only: it reads +// `cgroup.controllers` files under /sys/fs/cgroup and never edits systemd +// units, never uses sudo, and never weakens resource isolation. When the +// hierarchy cannot enforce the CPU limit, the caller must fail early with a +// diagnostic that distinguishes hierarchy and read failure modes and states +// the exact administrator remediation, then require the user to rerun the +// preflight. + +import fs from "node:fs"; + +export type CpuDelegationFailureReason = + | "cgroups-v2-unavailable" + | "cgroup-controllers-unreadable" + | "cgroup-controllers-malformed" + | "cpu-controller-unavailable" + | "systemd-user-slice-cpu-unavailable" + | "systemd-user-delegation-missing" + | "app-slice-cpu-unavailable"; + +export interface CpuDelegationPreflight { + readonly ok: boolean; + readonly failure?: CpuDelegationFailureReason; + readonly detail: string; +} + +export interface CpuDelegationPreflightDeps { + readonly platform?: NodeJS.Platform; + readonly uid?: number; + readonly readControllerFileSync?: (file: string, maxBytes: number) => Buffer; +} + +const CGROUP_ROOT = "/sys/fs/cgroup"; +const MAX_CONTROLLER_EVIDENCE_BYTES = 4096; +const CONTROLLER_NAME = /^[a-z][a-z0-9_]*$/u; + +export function cpuDelegationControllerPaths(uid: number): { + readonly root: string; + readonly userSlice: string; + readonly userManager: string; + readonly appSlice: string; +} { + return { + root: `${CGROUP_ROOT}/cgroup.controllers`, + userSlice: `${CGROUP_ROOT}/user.slice/user-${uid}.slice/cgroup.controllers`, + userManager: `${CGROUP_ROOT}/user.slice/user-${uid}.slice/user@${uid}.service/cgroup.controllers`, + appSlice: `${CGROUP_ROOT}/user.slice/user-${uid}.slice/user@${uid}.service/app.slice/cgroup.controllers`, + }; +} + +function cpuControllerSettingsGuidance(uid: number): string { + return ( + "Have an administrator apply all three CPU controller settings described in the " + + `troubleshooting guide: \`CPUWeight=100\` for \`user-${uid}.slice\`, ` + + "`Delegate=cpu memory pids` for `user@.service`, and `CPUWeight=100` for " + + "`app.slice`. " + ); +} + +type ControllerEvidence = + | { readonly ok: true; readonly content: string; readonly names: ReadonlySet } + | { readonly ok: false }; + +function parseControllerEvidence(content: Buffer): ControllerEvidence { + if (content.length > MAX_CONTROLLER_EVIDENCE_BYTES) { + return { ok: false }; + } + + const body = content.at(-1) === 0x0a ? content.subarray(0, content.length - 1) : content; + if (body.length === 0) { + return { ok: true, content: "", names: new Set() }; + } + if (body.includes(0x0a)) { + return { ok: false }; + } + + const text = body.toString("utf8"); + const names = text.split(" "); + const uniqueNames = new Set(names); + if (uniqueNames.size !== names.length || names.some((name) => !CONTROLLER_NAME.test(name))) { + return { ok: false }; + } + return { ok: true, content: text, names: uniqueNames }; +} + +type ControllerRead = + | { readonly ok: true; readonly content: Buffer } + | { + readonly ok: false; + readonly condition: "missing" | "unreadable"; + readonly errorCode?: string; + }; + +function readControllerFileSync(file: string, maxBytes: number): Buffer { + const fileDescriptor = fs.openSync(file, "r"); + try { + const buffer = Buffer.alloc(maxBytes); + let offset = 0; + while (offset < buffer.length) { + const bytesRead = fs.readSync(fileDescriptor, buffer, offset, buffer.length - offset, null); + if (bytesRead === 0) break; + offset += bytesRead; + } + return buffer.subarray(0, offset); + } finally { + fs.closeSync(fileDescriptor); + } +} + +function readControllers( + file: string, + readControllerFile: (file: string, maxBytes: number) => Buffer, +): ControllerRead { + try { + return { + ok: true, + content: readControllerFile(file, MAX_CONTROLLER_EVIDENCE_BYTES + 1), + }; + } catch (error) { + const errorCode = + typeof error === "object" && error !== null && "code" in error + ? String(error.code) + : undefined; + return { + ok: false, + condition: errorCode === "ENOENT" || errorCode === "ENOTDIR" ? "missing" : "unreadable", + ...(errorCode ? { errorCode } : {}), + }; + } +} + +function unreadableControllersDetail(file: string, errorCode?: string): string { + const code = errorCode ? ` (${errorCode})` : ""; + return ( + `NemoClaw could not read the cgroup controllers file ${file}${code}. ` + + "Have an administrator inspect the cgroup mount permissions and active Linux " + + "security policy, then make this exact file readable to the current user. Do not " + + "change systemd delegation until the preflight can inspect the file." + ); +} + +function malformedControllersDetail(file: string): string { + return ( + `NemoClaw could not classify the cgroup controller evidence because ${file} ` + + "does not contain the bounded, space-separated controller names supplied by the " + + "kernel. Have an administrator inspect the cgroup filesystem and active security " + + "tooling for this exact file. Do not change systemd configuration, stop the user " + + "manager, or reboot the host while the evidence is malformed. Rerun the portable " + + "preflight only after the file contains kernel-provided controller names." + ); +} + +function managerRecoveryGuidance(): string { + return ( + "Save the current user's work before the administrator stops the user manager: " + + "the stop interrupts that user's systemd-managed services. Apply the documented " + + "stop, daemon-reload, and start sequence. If the start fails with 219/CGROUP, do " + + "not continue onboarding; have the affected user sign out and start a later login " + + "session before rerunning the preflight. If host reboot is the alternative, save " + + "every user's work first: reboot interrupts host workloads. Only then may the " + + "administrator reboot the host." + ); +} + +type ParsedControllers = + | { readonly ok: true; readonly content: string; readonly names: ReadonlySet } + | { readonly ok: false; readonly preflight: CpuDelegationPreflight }; + +function parseControllers(file: string, content: Buffer): ParsedControllers { + const evidence = parseControllerEvidence(content); + if (!evidence.ok) { + return { + ok: false, + preflight: { + ok: false, + failure: "cgroup-controllers-malformed", + detail: malformedControllersDetail(file), + }, + }; + } + return { ok: true, content: evidence.content, names: evidence.names }; +} + +export function inspectPortableCpuDelegation( + deps: CpuDelegationPreflightDeps = {}, +): CpuDelegationPreflight { + if ((deps.platform ?? process.platform) !== "linux") { + return { + ok: true, + detail: "CPU-delegation preflight only applies on Linux; skipping.", + }; + } + const uid = deps.uid ?? process.geteuid?.() ?? process.getuid?.(); + if (!Number.isInteger(uid) || Number(uid) < 0) { + return { + ok: true, + detail: "Could not resolve the current user ID; CPU-delegation preflight skipped.", + }; + } + const readControllerFile = deps.readControllerFileSync ?? readControllerFileSync; + const numericUid = Number(uid); + const { root, userSlice, userManager, appSlice } = cpuDelegationControllerPaths(numericUid); + + const rootRead = readControllers(root, readControllerFile); + if (!rootRead.ok) { + if (rootRead.condition === "unreadable") { + return { + ok: false, + failure: "cgroup-controllers-unreadable", + detail: unreadableControllersDetail(root, rootRead.errorCode), + }; + } + return { + ok: false, + failure: "cgroups-v2-unavailable", + detail: + `cgroups v2 is not available: ${root} is missing. ` + + "Rootless Podman cannot enforce the sandbox CPU limit without a cgroups v2 " + + "kernel and mount. Before rebooting this host, save every user's work: reboot " + + "interrupts host workloads. Boot a cgroups v2 host and rerun the portable " + + "preflight.", + }; + } + const parsedRoot = parseControllers(root, rootRead.content); + if (!parsedRoot.ok) { + return parsedRoot.preflight; + } + const rootControllers = parsedRoot.names; + const rootContent = parsedRoot.content; + if (!rootControllers.has("cpu")) { + return { + ok: false, + failure: "cpu-controller-unavailable", + detail: + `The kernel cgroup hierarchy does not expose the cpu controller: ${root} ` + + `is "${rootContent.trim()}" (no "cpu"). Rootless Podman cannot enforce the ` + + "sandbox CPU limit. Before rebooting this host, save every user's work: reboot " + + "interrupts host workloads. Enable the cpu controller in the kernel cgroup " + + "hierarchy and rerun the portable preflight.", + }; + } + + const userSliceRead = readControllers(userSlice, readControllerFile); + if (!userSliceRead.ok) { + if (userSliceRead.condition === "unreadable") { + return { + ok: false, + failure: "cgroup-controllers-unreadable", + detail: unreadableControllersDetail(userSlice, userSliceRead.errorCode), + }; + } + return { + ok: false, + failure: "systemd-user-slice-cpu-unavailable", + detail: + `The current user's systemd slice has no cgroup controllers file ` + + `(${userSlice} is missing), so the cpu controller is not available at the ` + + `user-${numericUid}.slice ancestor. ` + + cpuControllerSettingsGuidance(numericUid) + + managerRecoveryGuidance() + + " Then rerun the portable preflight.", + }; + } + const parsedUserSlice = parseControllers(userSlice, userSliceRead.content); + if (!parsedUserSlice.ok) { + return parsedUserSlice.preflight; + } + if (!parsedUserSlice.names.has("cpu")) { + return { + ok: false, + failure: "systemd-user-slice-cpu-unavailable", + detail: + `The cpu controller is not available at the current user's systemd slice: ` + + `${userSlice} is "${parsedUserSlice.content.trim()}" (no "cpu"). ` + + `The user manager and app.slice cannot activate a controller that their ` + + `user-${numericUid}.slice ancestor did not receive. ` + + cpuControllerSettingsGuidance(numericUid) + + managerRecoveryGuidance() + + " Then rerun the portable preflight.", + }; + } + + const userManagerRead = readControllers(userManager, readControllerFile); + if (!userManagerRead.ok) { + if (userManagerRead.condition === "unreadable") { + return { + ok: false, + failure: "cgroup-controllers-unreadable", + detail: unreadableControllersDetail(userManager, userManagerRead.errorCode), + }; + } + return { + ok: false, + failure: "systemd-user-delegation-missing", + detail: + `The current user's systemd manager has no cgroup controllers file ` + + `(${userManager} is missing), so systemd has not exposed controllers to it. ` + + cpuControllerSettingsGuidance(numericUid) + + managerRecoveryGuidance() + + " Then rerun the portable preflight.", + }; + } + const parsedUserManager = parseControllers(userManager, userManagerRead.content); + if (!parsedUserManager.ok) { + return parsedUserManager.preflight; + } + const userManagerControllers = parsedUserManager.names; + const userManagerContent = parsedUserManager.content; + if (!userManagerControllers.has("cpu")) { + return { + ok: false, + failure: "systemd-user-delegation-missing", + detail: + `systemd did not delegate the cpu controller to the current user's manager: ` + + `${userManager} is "${userManagerContent.trim()}" (no "cpu"). The stock ` + + "user@.service delegates only `pids memory`. " + + cpuControllerSettingsGuidance(numericUid) + + managerRecoveryGuidance() + + " Then rerun the portable preflight.", + }; + } + + const appSliceRead = readControllers(appSlice, readControllerFile); + if (!appSliceRead.ok) { + if (appSliceRead.condition === "unreadable") { + return { + ok: false, + failure: "cgroup-controllers-unreadable", + detail: unreadableControllersDetail(appSlice, appSliceRead.errorCode), + }; + } + return { + ok: false, + failure: "app-slice-cpu-unavailable", + detail: + `The current user's app.slice has no cgroup controllers file (${appSlice} is ` + + "missing), so the cpu controller is not available to it for this boot. " + + cpuControllerSettingsGuidance(numericUid) + + managerRecoveryGuidance() + + " Then rerun the portable preflight.", + }; + } + const parsedAppSlice = parseControllers(appSlice, appSliceRead.content); + if (!parsedAppSlice.ok) { + return parsedAppSlice.preflight; + } + const appSliceControllers = parsedAppSlice.names; + const appSliceContent = parsedAppSlice.content; + if (!appSliceControllers.has("cpu")) { + return { + ok: false, + failure: "app-slice-cpu-unavailable", + detail: + `The cpu controller is not available to the current user's app.slice for ` + + `this boot: ${appSlice} is "${appSliceContent.trim()}" (no "cpu"). ` + + cpuControllerSettingsGuidance(numericUid) + + managerRecoveryGuidance() + + " Then rerun the portable preflight.", + }; + } + + return { + ok: true, + detail: + "The current user's systemd/cgroup hierarchy can enforce the sandbox CPU " + + "limit: the cpu controller is exposed at the per-user system slice, delegated " + + "to the user manager, and available to app.slice.", + }; +} + +export function portableCpuDelegationError(preflight: CpuDelegationPreflight): Error { + return new Error(`Portable CPU-delegation preflight failed: ${preflight.detail}`); +} diff --git a/src/lib/onboard/experimental/portable-host-preparation.test.ts b/src/lib/onboard/experimental/portable-host-preparation.test.ts index 007b319f0bc..0a8f6e418f3 100644 --- a/src/lib/onboard/experimental/portable-host-preparation.test.ts +++ b/src/lib/onboard/experimental/portable-host-preparation.test.ts @@ -10,6 +10,10 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { PodmanSocketAuthority } from "../../adapters/podman"; import type { CheckpointPortableRuntimeAuthority } from "../../state/onboard-checkpoint-types"; import { createPortableOnboardEnvironmentScope } from "../session-bootstrap"; +import { + cpuDelegationControllerPaths, + inspectPortableCpuDelegation, +} from "./portable-cpu-delegation-preflight"; import { portableHostPreparationInternals, preparePortableExperimentalHost as preparePortableExperimentalHostUnchecked, @@ -72,6 +76,11 @@ function preparePortableExperimentalHost( env, { ...deps, + // Tests run on hosts without the /sys/fs/cgroup hierarchy the portable + // CPU-delegation preflight reads; default to a passing stub and inject + // explicit results for the preflight wiring tests below. + cpuDelegationPreflight: + deps.cpuDelegationPreflight ?? (() => ({ ok: true, detail: "stubbed in tests" })), runtimeReadiness: deps.runtimeReadiness ?? successfulReadiness(deps.home ?? expectedAuthority?.homeDir ?? os.userInfo().homedir), @@ -100,6 +109,109 @@ describe("preparePortableExperimentalHost", () => { expect(docker).not.toHaveBeenCalled(); }); + it("fails the portable preflight when the user hierarchy cannot enforce the CPU limit (#9188)", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-portable-")); + tempDirs.push(home); + const systemctl = vi.fn< + (args: readonly string[], env: NodeJS.ProcessEnv, timeoutMs?: number) => SpawnResult + >(() => result()); + const docker = vi.fn(); + const env: NodeJS.ProcessEnv = { + HOME: home, + NEMOCLAW_EXPERIMENTAL_PROFILE: "portable", + }; + const cpuDelegationPreflight = () => ({ + ok: false, + failure: "systemd-user-delegation-missing" as const, + detail: "systemd did not delegate the cpu controller to the current user's manager.", + }); + + expect(() => + preparePortableExperimentalHost(env, { + platform: "linux", + home, + uid: 1001, + systemctl, + docker, + cpuDelegationPreflight, + }), + ).toThrow(/Portable CPU-delegation preflight failed/); + + // The gate must fire before any config write or service activation. + expect(systemctl).not.toHaveBeenCalled(); + expect(docker).not.toHaveBeenCalled(); + }); + + it("rejects malformed controller evidence before portable host effects (#9188)", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-portable-")); + tempDirs.push(home); + const systemctl = vi.fn(); + const docker = vi.fn(); + const validateConfigAuthority = vi.fn(); + const paths = cpuDelegationControllerPaths(1001); + const readControllerFileSync = vi.fn(() => Buffer.from("cpu memory\0Delegate=cpu")); + + expect(() => + preparePortableExperimentalHost( + { HOME: home, NEMOCLAW_EXPERIMENTAL_PROFILE: "portable" }, + { + platform: "linux", + home, + uid: 1001, + systemctl, + docker, + validateConfigAuthority, + cpuDelegationPreflight: (deps) => + inspectPortableCpuDelegation({ ...deps, readControllerFileSync }), + }, + ), + ).toThrow(/controller evidence.*malformed/u); + + expect(validateConfigAuthority).not.toHaveBeenCalled(); + expect(readControllerFileSync).toHaveBeenCalledOnce(); + expect(readControllerFileSync).toHaveBeenCalledWith(paths.root, 4097); + expect(systemctl).not.toHaveBeenCalled(); + expect(docker).not.toHaveBeenCalled(); + expect(fs.existsSync(path.join(home, ".config"))).toBe(false); + }); + + it("passes portable host preparation when the CPU-delegation preflight succeeds (#9188)", () => { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-portable-")); + tempDirs.push(home); + const systemctl = vi.fn< + (args: readonly string[], env: NodeJS.ProcessEnv, timeoutMs?: number) => SpawnResult + >(() => result()); + const docker = vi + .fn<(args: readonly string[], env: NodeJS.ProcessEnv) => SpawnResult>() + .mockReturnValueOnce(result()) // --version probe + .mockReturnValueOnce(result(1)) // inspect: registry not present + .mockReturnValueOnce(result()); // run + const podman = vi.fn<(args: readonly string[], env: NodeJS.ProcessEnv) => SpawnResult>(() => + result(0, "/run/user/1001/custom/podman.sock\n"), + ); + const hardenSocketDirectory = vi.fn(); + const env: NodeJS.ProcessEnv = { + HOME: home, + NEMOCLAW_EXPERIMENTAL_PROFILE: "portable", + }; + const cpuDelegationPreflight = () => ({ ok: true, detail: "cpu delegated" }); + + const prepared = preparePortableExperimentalHost(env, { + platform: "linux", + home, + uid: 1001, + systemctl, + podman, + docker, + hardenSocketDirectory, + validateConfigAuthority: vi.fn(), + cpuDelegationPreflight, + }); + + expect(prepared).not.toBeNull(); + expect(prepared?.authority.uid).toBe(1001); + }); + it("prepares the rootless socket and managed loopback registry deterministically", () => { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-portable-")); tempDirs.push(home); diff --git a/src/lib/onboard/experimental/portable-host-preparation.ts b/src/lib/onboard/experimental/portable-host-preparation.ts index c6b8a8950b9..362e25273e0 100644 --- a/src/lib/onboard/experimental/portable-host-preparation.ts +++ b/src/lib/onboard/experimental/portable-host-preparation.ts @@ -23,6 +23,12 @@ import { portablePodmanReadinessError, type PortablePodmanReadinessDeps, } from "./portable-runtime-readiness"; +import { + inspectPortableCpuDelegation, + portableCpuDelegationError, + type CpuDelegationPreflight, + type CpuDelegationPreflightDeps, +} from "./portable-cpu-delegation-preflight"; const REGISTRY_CONTAINER = "nemoclaw-portable-registry"; const REGISTRY_LABEL = "com.nvidia.nemoclaw.portable=1"; @@ -53,16 +59,8 @@ export interface PortableHostPreparationDeps { platform?: NodeJS.Platform; home?: string; uid?: number; - systemctl?: ( - args: readonly string[], - env: NodeJS.ProcessEnv, - timeoutMs?: number, - ) => SpawnResult; - podman?: ( - args: readonly string[], - env: NodeJS.ProcessEnv, - timeoutMs?: number, - ) => SpawnResult; + systemctl?: (args: readonly string[], env: NodeJS.ProcessEnv, timeoutMs?: number) => SpawnResult; + podman?: (args: readonly string[], env: NodeJS.ProcessEnv, timeoutMs?: number) => SpawnResult; docker?: (args: readonly string[], env: NodeJS.ProcessEnv) => SpawnResult; hardenSocketDirectory?: (socketPath: string, uid: number) => void; captureSocketAuthority?: (socketPath: string, uid: number) => PodmanSocketAuthority; @@ -76,6 +74,7 @@ export interface PortableHostPreparationDeps { socketPath: string | null; uid: number; }) => void; + cpuDelegationPreflight?: (deps: CpuDelegationPreflightDeps) => CpuDelegationPreflight; } export interface PortableHostPreparationResult { @@ -355,6 +354,13 @@ export function preparePortableExperimentalHost( if (!Number.isInteger(uid) || Number(uid) < 0) { throw new Error("The portable experimental profile could not resolve the current user ID."); } + // Fail early, before any config write or service activation, when the + // current user's systemd/cgroup hierarchy cannot enforce the sandbox CPU + // limit (gh #9188). The diagnostic is credential-free and never edits + // systemd units or weakens isolation. + const cpuDelegation = deps.cpuDelegationPreflight ?? inspectPortableCpuDelegation; + const cpuPreflight = cpuDelegation({ platform: deps.platform, uid: Number(uid) }); + if (!cpuPreflight.ok) throw portableCpuDelegationError(cpuPreflight); const currentHome = canonicalAbsolute(deps.home ?? os.userInfo().homedir, "home directory"); const home = canonicalAbsolute(expectedAuthority?.homeDir ?? currentHome, "home directory"); const configHome = path.join(home, ".config"); diff --git a/test/e2e/live/portable-cpu-delegation-proof.test.ts b/test/e2e/live/portable-cpu-delegation-proof.test.ts new file mode 100644 index 00000000000..a05ff1fe88f --- /dev/null +++ b/test/e2e/live/portable-cpu-delegation-proof.test.ts @@ -0,0 +1,224 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { execFileSync, spawnSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; + +import type { PodmanSocketAuthority } from "../../../src/lib/adapters/podman/index.ts"; +import { + inspectPortableCpuDelegation, + type CpuDelegationPreflight, +} from "../../../src/lib/onboard/experimental/portable-cpu-delegation-preflight.ts"; +import { preparePortableExperimentalHost } from "../../../src/lib/onboard/experimental/portable-host-preparation.ts"; +import { test } from "../fixtures/e2e-test.ts"; + +type ExpectedState = "missing" | "delegated"; +type SpawnResult = ReturnType; + +interface AdmissionEvidence { + readonly admissionCompleted: boolean; + readonly admissionEffects: readonly string[]; + readonly effectsBeforeAdmission: number; +} + +function expectedState(value: string | undefined): ExpectedState { + assert.ok( + value === "missing" || value === "delegated", + "E2E_CPU_DELEGATION_STATE must be missing or delegated.", + ); + return value; +} + +function expectedUid(): number { + const value = Number(process.env.E2E_CPU_DELEGATION_UID); + assert.ok(Number.isInteger(value) && value >= 0, "E2E_CPU_DELEGATION_UID must be a user ID"); + return value; +} + +function sourceRevision(): string { + const value = process.env.E2E_SOURCE_REVISION; + assert.match(value ?? "", /^[a-f0-9]{40}$/u, "E2E_SOURCE_REVISION must be a commit SHA"); + const checkoutRevision = execFileSync( + "git", + ["-c", `safe.directory=${process.cwd()}`, "rev-parse", "HEAD"], + { encoding: "utf8", killSignal: "SIGKILL", timeout: 10_000 }, + ).trim(); + assert.equal(value, checkoutRevision, "CPU delegation proof must run the requested commit"); + return value!; +} + +function commandResult(status = 0, stdout = ""): SpawnResult { + return { status, stdout, stderr: "" } as SpawnResult; +} + +function socketAuthority(uid: number, socketPath: string): PodmanSocketAuthority { + return { + directoryChain: [], + device: "1", + inode: "2", + mode: String(0o140660), + ownerUid: String(uid), + socketPath, + }; +} + +function proveFailureBeforeEffects( + preflight: CpuDelegationPreflight, + uid: number, + artifactRoot: string, +): AdmissionEvidence { + assert.equal(preflight.ok, false); + assert.equal(preflight.failure, "systemd-user-delegation-missing"); + const effects: string[] = []; + const home = fs.mkdtempSync(path.join(artifactRoot, "rejected-home-")); + const effect = (name: string): never => { + effects.push(name); + throw new Error(`Portable host preparation reached ${name} after failed CPU delegation.`); + }; + try { + assert.throws( + () => + preparePortableExperimentalHost( + { NEMOCLAW_EXPERIMENTAL_PROFILE: "portable" }, + { + platform: "linux", + home, + uid, + cpuDelegationPreflight: () => preflight, + validateConfigAuthority: () => effect("config authority validation"), + systemctl: () => effect("systemd mutation"), + podman: () => effect("Podman mutation"), + docker: () => effect("Docker-compatible mutation"), + }, + ), + /Portable CPU-delegation preflight failed/u, + ); + assert.deepEqual(fs.readdirSync(home), []); + assert.equal(effects.length, 0); + return { + admissionCompleted: false, + admissionEffects: [], + effectsBeforeAdmission: effects.length, + }; + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } +} + +function proveAdmission( + preflight: CpuDelegationPreflight, + uid: number, + artifactRoot: string, +): AdmissionEvidence { + assert.equal(preflight.ok, true, preflight.detail); + assert.equal(preflight.failure, undefined); + const effects: string[] = []; + const effectsBeforeAdmission = effects.length; + const home = fs.mkdtempSync(path.join(artifactRoot, "admitted-home-")); + const socketPath = `/run/user/${String(uid)}/podman/podman.sock`; + const authority = socketAuthority(uid, socketPath); + const dockerResults: Record = { + "--version": commandResult(), + inspect: commandResult(0, "1 true"), + }; + try { + const prepared = preparePortableExperimentalHost( + { NEMOCLAW_EXPERIMENTAL_PROFILE: "portable" }, + { + platform: "linux", + home, + uid, + cpuDelegationPreflight: () => preflight, + validateConfigAuthority: () => effects.push("config authority validation"), + systemctl: (args) => { + effects.push(`systemctl ${args.join(" ")}`); + return commandResult(); + }, + hardenSocketDirectory: () => effects.push("socket directory hardening"), + captureSocketAuthority: () => { + effects.push("socket authority capture"); + return authority; + }, + assertSocketAuthority: () => effects.push("socket authority assertion"), + runtimeReadiness: { + podmanCapture: () => { + effects.push("Podman API health probe"); + return { + status: 0, + stdout: JSON.stringify({ Server: { Version: "proof" } }), + stderr: "", + }; + }, + }, + docker: (args) => { + effects.push(`docker-compatible ${args.join(" ")}`); + const result = dockerResults[args[0] ?? ""]; + assert.ok(result, `Unexpected Docker-compatible proof command: ${args.join(" ")}`); + return result; + }, + }, + ); + assert.ok(prepared, "Delegated CPU hierarchy must complete portable host admission."); + assert.equal(prepared.authority.uid, uid); + assert.equal(prepared.authority.socketPath, socketPath); + assert.ok(effects.length > effectsBeforeAdmission); + return { + admissionCompleted: true, + admissionEffects: effects, + effectsBeforeAdmission, + }; + } finally { + fs.rmSync(home, { recursive: true, force: true }); + } +} + +const proveState: Record< + ExpectedState, + (preflight: CpuDelegationPreflight, uid: number, artifactRoot: string) => AdmissionEvidence +> = { + missing: proveFailureBeforeEffects, + delegated: proveAdmission, +}; + +const proof = process.env.E2E_TARGET_ID === "portable-cpu-delegation" ? test : test.skip; + +proof( + "records portable CPU delegation evidence for the configured hierarchy (#9188)", + { + timeout: 30_000, + meta: { + e2ePhases: [ + "inspect the configured CPU delegation hierarchy", + "record CPU delegation admission evidence", + ], + }, + }, + async ({ artifacts, progress }) => { + assert.equal(process.platform, "linux", "CPU delegation proof requires Linux"); + const uid = expectedUid(); + assert.notEqual(uid, 0, "CPU delegation proof requires a non-root user"); + assert.equal( + process.getuid?.(), + uid, + "CPU delegation proof must run as the dedicated non-root user", + ); + const state = expectedState(process.env.E2E_CPU_DELEGATION_STATE); + const revision = sourceRevision(); + progress.phase("inspect the configured CPU delegation hierarchy"); + const preflight = inspectPortableCpuDelegation({ uid }); + const evidence = proveState[state](preflight, uid, artifacts.rootDir); + progress.phase("record CPU delegation admission evidence"); + await artifacts.writeJson(`${state}.json`, { + schemaVersion: 1, + sourceRevision: revision, + state, + uid, + ok: preflight.ok, + failure: preflight.failure ?? null, + detail: preflight.detail, + ...evidence, + }); + }, +); diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 41b93ad979c..bbfec6b1fad 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -2,6 +2,14 @@ "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.\nSPDX-License-Identifier: Apache-2.0", "version": 1, "entries": [ + { + "live": "test/e2e/live/portable-cpu-delegation-proof.test.ts", + "fast": [ + "src/lib/onboard/experimental/portable-cpu-delegation-preflight.test.ts", + "src/lib/onboard/experimental/portable-host-preparation.test.ts", + "test/e2e/support/podman-cpu-proof-workflow.test.ts" + ] + }, { "live": "test/e2e/live/portable-profile-rootless-linux.test.ts", "fast": [ diff --git a/test/e2e/support/podman-cpu-proof-workflow.test.ts b/test/e2e/support/podman-cpu-proof-workflow.test.ts index 13b8a4e1c6b..b9f972f9214 100644 --- a/test/e2e/support/podman-cpu-proof-workflow.test.ts +++ b/test/e2e/support/podman-cpu-proof-workflow.test.ts @@ -1,8 +1,25 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; import { describe, expect, it } from "vitest"; +import { + parsePortableCpuDelegationProofMode, + PORTABLE_CPU_DELEGATION_PROOF_CONTRACT, + portableCpuDelegationProofCli, + type CommandOptions, + type CommandResult, + type HostCommandRunner, + type HostFilesystem, + type HostPathStat, + type PortableCpuDelegationProofMode, + runPortableCpuDelegationProofMode, +} from "../../../scripts/checks/run-portable-cpu-delegation-proof.mts"; import { readRepoText, readYaml, @@ -11,6 +28,10 @@ import { type WorkflowStep, } from "../../helpers/e2e-workflow-contract"; +const APP_DROP_IN = PORTABLE_CPU_DELEGATION_PROOF_CONTRACT.appSliceDropIn; +const DELEGATION_DROP_IN = PORTABLE_CPU_DELEGATION_PROOF_CONTRACT.delegationDropIn; +const USER_SLICE_DROP_IN = `/etc/systemd/system/user-1001.slice.d/${PORTABLE_CPU_DELEGATION_PROOF_CONTRACT.userSliceDropInName}`; + type PodmanProofWorkflow = Workflow & { on: { pull_request: { paths: string[]; types: string[] } }; permissions: Record; @@ -26,42 +47,342 @@ function proofJob(): WorkflowJob { return job!; } +function delegationJob(): WorkflowJob { + const job = workflow().jobs["portable-cpu-delegation"]; + expect(job).toBeDefined(); + return job!; +} + function namedStep(name: string): WorkflowStep { const step = proofJob().steps?.find((candidate) => candidate.name === name); expect(step, `missing Podman CPU proof step '${name}'`).toBeDefined(); return step!; } - +function namedDelegationStep(name: string): WorkflowStep { + const step = delegationJob().steps?.find((candidate) => candidate.name === name); + expect(step, `missing CPU delegation proof step '${name}'`).toBeDefined(); + return step!; +} +type RecordedCommand = { + readonly argv: readonly string[]; + readonly executable: string; + readonly options: CommandOptions; +}; +class ProofFixtureRunner implements HostCommandRunner { + readonly calls: RecordedCommand[] = []; + readonly contents = new Map(); + readonly directories = new Set([path.dirname(DELEGATION_DROP_IN), path.dirname(APP_DROP_IN)]); + readonly envAtCreate = new Map(); + readonly files = new Set(); + readonly identities = new Map(); + readonly ownerModes = new Map(); + failIdentityFor = ""; + failMoveFor = ""; + failRemovalFor = ""; + failRemovalPrefix = ""; + failTestFor = ""; + failTee = false; + failUserLookup = false; + failWorkspaceModeRestore = false; + managerStartDiagnostic = ""; + managerStartFailures = 0; + userCreated = false; + constructor( + readonly home: string, + readonly githubEnv: string, + ) {} + seedFile(target: string, id = "1:1"): void { + this.files.add(target); + this.identities.set(target, id); + } + hasResource(target: string): boolean { + return this.files.has(target) || this.directories.has(target); + } + private ok(stdout = ""): CommandResult { + return { status: 0, stdout, stderr: "" }; + } + private failed(stderr = "fixture failure"): CommandResult { + return { status: 1, stdout: "", stderr }; + } + private identityFor(target: string): string { + const value = this.identities.get(target) ?? `1:${String(this.identities.size + 1)}`; + this.identities.set(target, value); + return value; + } + private stat(argv: readonly string[], target: string): CommandResult { + const fail = target === this.failIdentityFor; + this.failIdentityFor = fail ? "" : this.failIdentityFor; + return fail + ? this.failed("identity fixture failure") + : argv[2] === "%U:%G %a" + ? this.ok(`${this.ownerModes.get(target) ?? "root:root 755"}\n`) + : this.ok(`${this.identityFor(target)}\n`); + } + private move(source: string, target: string): CommandResult { + this.seedFile(target, this.identityFor(source)); + this.files.delete(source); + this.contents.set(target, this.contents.get(source) ?? ""); + this.contents.delete(source); + return this.ok(); + } + private tee(target: string, options: CommandOptions): CommandResult { + this.seedFile(target); + this.contents.set(target, options.input ?? ""); + return this.ok(options.input ?? ""); + } + private remove(target: string): CommandResult { + for (const file of this.files) file.startsWith(`${target}/`) ? this.files.delete(file) : false; + this.files.delete(target); + this.directories.delete(target); + this.ownerModes.delete(target); + return this.ok(); + } + private test(argv: readonly string[]): CommandResult { + const flag = argv[0]; + const target = argv.at(-1) ?? ""; + const present = this.files.has(target) || this.directories.has(target); + const result = + flag === "-L" + ? false + : flag === "-d" + ? this.directories.has(target) + : flag === "-f" + ? this.files.has(target) + : flag === "-e" + ? present + : true; + return target === this.failTestFor + ? this.failed("sudo test fixture failure") + : result + ? this.ok() + : this.failed(""); + } + private sudo(argv: readonly string[], options: CommandOptions): CommandResult { + const operation = argv[0]; + const target = argv.at(-1) ?? ""; + switch (operation) { + case "test": + return this.test(argv.slice(1)); + case "stat": + return this.stat(argv, target); + case "mkdir": + this.directories.add(target); + this.envAtCreate.set(target, fs.readFileSync(this.githubEnv, "utf8")); + this.ownerModes.set( + target, + `root:root ${Number.parseInt(argv[2] ?? "755", 8).toString(8)}`, + ); + return this.ok(); + case "mv": { + const source = argv.at(-2) ?? ""; + return target === this.failMoveFor + ? this.failed("move fixture failure") + : this.move(source, target); + } + case "rm": + return target === this.failRemovalFor || + (this.failRemovalPrefix !== "" && target.startsWith(this.failRemovalPrefix)) + ? this.failed("removal fixture failure") + : this.remove(target); + case "rmdir": + return [...this.files, ...this.directories].some( + (entry) => entry !== target && entry.startsWith(`${target}/`), + ) + ? this.failed("directory not empty") + : target === this.failRemovalFor + ? this.failed("removal fixture failure") + : this.remove(target); + case "tee": + return this.failTee ? this.failed("tee fixture failure") : this.tee(target, options); + case "cat": + return this.ok(this.contents.get(target) ?? ""); + case "grep": + return (this.contents.get(target) ?? "").includes(argv[2] ?? "") + ? this.ok() + : this.failed(); + case "useradd": + this.userCreated = true; + return this.ok(); + case "userdel": + this.userCreated = false; + return this.ok(); + case "systemctl": + switch (argv[1]) { + case "start": { + const failStart = this.managerStartFailures > 0; + this.managerStartFailures -= Number(failStart); + return failStart ? this.failed(this.managerStartDiagnostic) : this.ok(); + } + case "status": + return this.failed(this.managerStartDiagnostic); + default: + return this.ok(); + } + case "journalctl": + return this.ok(this.managerStartDiagnostic); + case "chmod": + return this.failWorkspaceModeRestore && argv[1] !== "0600" && argv[1] !== "0644" + ? this.failed() + : this.ok(); + default: + return this.ok(); + } + } + run(executable: string, argv: readonly string[], options: CommandOptions = {}): CommandResult { + this.calls.push({ executable, argv: [...argv], options }); + switch (executable) { + case "sudo": + return argv[0] === "--user" ? this.ok() : this.sudo(argv, options); + case "id": + return argv[0] === "-u" ? this.ok("1001\n") : this.userCreated ? this.ok() : this.failed(); + case "getent": + return this.failUserLookup + ? this.failed("getent fixture failure") + : this.userCreated + ? this.ok(`nemoclaw-e2e:x:1001:1001:nemoclaw-cpu-proof-7-1:${this.home}:/bin/bash\n`) + : { status: 2, stdout: "", stderr: "" }; + case "stat": + return this.ok("755\n"); + case "python3": + return this.ok(options.input ?? ""); + default: + return this.ok(); + } + } +} +type RecordedFilesystemCall = { + readonly operation: string; + readonly target: string; +}; +class ProofFixtureFilesystem implements HostFilesystem { + readonly calls: RecordedFilesystemCall[] = []; + readonly envAtCreate = new Map(); + failLstatOnceFor = ""; + lstatSuccessesBeforeFailure = 0; + failWriteTarget = ""; + constructor(readonly githubEnv: string) {} + private lstatFailure(target: string): never { + throw Object.assign(new Error(`lstat fixture failure: ${target}`), { code: "EIO" }); + } + private publishConcurrentMarker(target: string): never { + fs.writeFileSync(target, "concurrent\n", { encoding: "utf8", flag: "wx", mode: 0o600 }); + throw new Error("filesystem fixture write failure"); + } + appendText(target: string, content: string): void { + this.calls.push({ operation: "appendText", target }); + fs.appendFileSync(target, content, { encoding: "utf8" }); + } + exists(target: string): boolean { + this.calls.push({ operation: "exists", target }); + return fs.existsSync(target); + } + lstat(target: string): HostPathStat { + this.calls.push({ operation: "lstat", target }); + const matches = target === this.failLstatOnceFor; + const fail = matches && this.lstatSuccessesBeforeFailure === 0; + this.failLstatOnceFor = fail ? "" : this.failLstatOnceFor; + this.lstatSuccessesBeforeFailure = matches + ? Math.max(0, this.lstatSuccessesBeforeFailure - 1) + : this.lstatSuccessesBeforeFailure; + return fail ? this.lstatFailure(target) : fs.lstatSync(target); + } + makeDirectory( + target: string, + options: { readonly mode: number; readonly recursive: boolean }, + ): void { + this.calls.push({ operation: "makeDirectory", target }); + fs.mkdirSync(target, options); + this.envAtCreate.set(target, fs.readFileSync(this.githubEnv, "utf8")); + } + readText(target: string): string { + this.calls.push({ operation: "readText", target }); + return fs.readFileSync(target, "utf8"); + } + removeDirectory(target: string): void { + this.calls.push({ operation: "removeDirectory", target }); + fs.rmdirSync(target); + } + removeFile(target: string): void { + this.calls.push({ operation: "removeFile", target }); + fs.unlinkSync(target); + } + writeExclusive(target: string, content: string): void { + this.calls.push({ operation: "writeExclusive", target }); + return target === this.failWriteTarget + ? this.publishConcurrentMarker(target) + : fs.writeFileSync(target, content, { encoding: "utf8", flag: "wx", mode: 0o600 }); + } +} +function createProofFixture() { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cpu-proof-script-")); + const workspace = path.join(directory, "workspace"); + const runnerTemp = path.join(directory, "runner-temp"); + const home = path.join(directory, "home"); + fs.mkdirSync(path.join(workspace, "node_modules"), { recursive: true }); + fs.mkdirSync(runnerTemp); + fs.mkdirSync(home); + const env: NodeJS.ProcessEnv = { + E2E_ARTIFACT_DIR: path.join(workspace, "e2e-artifacts", "portable-cpu-delegation"), + E2E_CPU_DELEGATION_USER: "nemoclaw-e2e", + E2E_SOURCE_REVISION: "a".repeat(40), + E2E_TARGET_ID: "portable-cpu-delegation", + GITHUB_ENV: path.join(directory, "github-env"), + GITHUB_RUN_ATTEMPT: "1", + GITHUB_RUN_ID: "7", + GITHUB_WORKSPACE: workspace, + NEMOCLAW_RUN_LIVE_E2E: "1", + PATH: process.env.PATH, + RUNNER_TEMP: runnerTemp, + }; + fs.writeFileSync(env.GITHUB_ENV!, "", { mode: 0o600 }); + return { + directory, + env, + filesystem: new ProofFixtureFilesystem(env.GITHUB_ENV!), + randomId: () => "00000000-0000-4000-8000-000000000000", + runner: new ProofFixtureRunner(home, env.GITHUB_ENV!), + }; +} +type ProofFixture = ReturnType; +function loadGithubEnv(fixture: ProofFixture): void { + const records = fs.readFileSync(fixture.env.GITHUB_ENV!, "utf8").split("\n").filter(Boolean); + for (const record of records) { + const separator = record.indexOf("="); + fixture.env[record.slice(0, separator)] = record.slice(separator + 1); + } +} +function withProofFixture(run: (fixture: ProofFixture) => void): void { + const fixture = createProofFixture(); + try { + run(fixture); + } finally { + fs.rmSync(fixture.directory, { force: true, recursive: true }); + } +} describe("native Podman CPU proof workflow", () => { // source-shape-contract: security -- Checkout binding and package pins bind the credential-free Podman proof to the commit under review and its runtime bytes it("runs as a credential-free PR workflow bound to the commit under review", () => { const parsed = workflow(); const job = proofJob(); - expect(parsed.permissions).toEqual({ contents: "read" }); expect(parsed.on.pull_request.types).toEqual(["opened", "synchronize", "reopened"]); - expect(parsed.on.pull_request.paths).toContain("src/lib/adapters/podman/**"); - expect(parsed.on.pull_request.paths).toContain("src/lib/onboard/docker-driver-gateway-*.ts"); - expect(parsed.on.pull_request.paths).toContain("src/lib/onboard/managed-bootstrap/podman-*.ts"); - expect(parsed.on.pull_request.paths).toContain( - "src/lib/onboard/experimental/portable-demo-lifecycle.ts", - ); - expect(parsed.on.pull_request.paths).toContain( - "src/lib/onboard/runtime-provider/container-state-mutation.ts", - ); - expect(parsed.on.pull_request.paths).toContain( - "src/lib/onboard/runtime-provider/docker-state-mutation.ts", - ); - expect(parsed.on.pull_request.paths).toContain("scripts/install-openshell.sh"); - expect(parsed.on.pull_request.paths).toContain( - "test/e2e/live/podman-cpu-lifecycle-artifacts.ts", - ); - expect(parsed.on.pull_request.paths).toContain("test/e2e/live/podman-cpu-lifecycle-helpers.ts"); - expect(parsed.on.pull_request.paths).toContain( - "test/e2e/live/podman-cpu-lifecycle-policy.yaml", - ); - expect(parsed.on.pull_request.paths).toContain( - "test/e2e/registry/native-runtime-qualification.ts", + expect(parsed.on.pull_request.paths).toEqual( + expect.arrayContaining([ + "src/lib/adapters/podman/**", + "src/lib/onboard/docker-driver-gateway-*.ts", + "src/lib/onboard/managed-bootstrap/podman-*.ts", + "src/lib/onboard/experimental/portable-demo-lifecycle.ts", + "src/lib/onboard/runtime-provider/container-state-mutation.ts", + "src/lib/onboard/runtime-provider/docker-state-mutation.ts", + "src/lib/onboard/experimental/portable-cpu-delegation-preflight*.ts", + "src/lib/onboard/experimental/portable-host-preparation*.ts", + "scripts/install-openshell.sh", + "test/e2e/live/podman-cpu-lifecycle-artifacts.ts", + "test/e2e/live/podman-cpu-lifecycle-helpers.ts", + "test/e2e/live/podman-cpu-lifecycle-policy.yaml", + "test/e2e/registry/native-runtime-qualification.ts", + "test/e2e/live/portable-cpu-delegation-proof.test.ts", + ]), ); expect(job.name).toBe("Rootless Podman CPU lifecycle with Docker disabled"); expect(job["runs-on"]).toBe("ubuntu-26.04"); @@ -88,6 +409,485 @@ describe("native Podman CPU proof workflow", () => { expect(installOpenShell).toContain("bash scripts/install-openshell.sh"); expect(installOpenShell).toContain("$HOME/.local/bin"); expect(readRepoText(".github/workflows/podman-cpu-proof.yaml")).not.toContain("${{ secrets."); + const delegation = delegationJob(); + const modeCommand = (mode: PortableCpuDelegationProofMode) => + `node --experimental-strip-types scripts/checks/run-portable-cpu-delegation-proof.mts ${mode}`; + expect(parsed.on.pull_request.paths).toContain( + "scripts/checks/run-portable-cpu-delegation-proof.mts", + ); + const proofScript = readRepoText("scripts/checks/run-portable-cpu-delegation-proof.mts"); + expect(proofScript).toContain("shell: false"); + expect(proofScript).not.toContain("shell: true"); + expect(proofScript).not.toContain("execSync("); + expect(proofScript).not.toContain("eval("); + expect(delegation.name).toBe("Portable CPU delegation admission on Ubuntu 22.04"); + expect(delegation["runs-on"]).toBe("ubuntu-22.04"); + expect(delegation["timeout-minutes"]).toBe(15); + expect(delegation.env?.E2E_CPU_DELEGATION_USER).toBe("nemoclaw-e2e"); + expect(delegation.env?.E2E_TARGET_ID).toBe("portable-cpu-delegation"); + expect(delegation.env?.E2E_SOURCE_REVISION).toBe("${{ github.event.pull_request.head.sha }}"); + expect(delegation.env?.NEMOCLAW_RUN_LIVE_E2E).toBe("1"); + expect(namedDelegationStep("Checkout").with).toMatchObject({ + "persist-credentials": false, + ref: "${{ github.event.pull_request.head.sha }}", + }); + expect(namedDelegationStep("Build shared sandbox-name contract").run).toBe( + "npm run build:policy-boundary", + ); + expect( + namedDelegationStep("Prepare system and app slice CPU settings without service delegation") + .run, + ).toBe(modeCommand("prepare")); + expect( + namedDelegationStep( + "Verify missing delegation blocks portable configuration and service activation", + ).run, + ).toBe(modeCommand("reject")); + expect(namedDelegationStep("Apply administrator delegation and prove admission").run).toBe( + modeCommand("admit"), + ); + const diagnostics = namedDelegationStep("Capture CPU delegation failure diagnostics"); + expect(diagnostics.if).toBe("failure()"); + expect(diagnostics.run).toBe(modeCommand("diagnostics")); + const cleanup = namedDelegationStep("Restore the user manager boundary"); + expect(cleanup.if).toBe("always()"); + expect(cleanup.run).toBe(modeCommand("cleanup")); + const delegationProof = readRepoText("test/e2e/live/portable-cpu-delegation-proof.test.ts"); + expect(delegationProof).toContain('from "../fixtures/e2e-test.ts"'); + expect(delegationProof).not.toContain('from "vitest"'); + expect(delegationProof).toContain('"rev-parse", "HEAD"'); + expect(delegationProof).toContain("e2ePhases"); + expect(delegationProof).toContain("process.env.E2E_CPU_DELEGATION_STATE"); + expect(delegationProof).toContain("process.getuid?.()"); + expect(delegationProof).not.toContain("process.argv"); + expect(delegationProof).not.toContain("main();"); + }); + it("executes the five typed proof modes with exact argv and durable cleanup receipts (#9188)", () => { + withProofFixture((fixture) => { + const modes: readonly PortableCpuDelegationProofMode[] = [ + "prepare", + "reject", + "admit", + "diagnostics", + "cleanup", + ]; + expect(modes.map((mode) => parsePortableCpuDelegationProofMode([mode]))).toEqual(modes); + runPortableCpuDelegationProofMode("prepare", fixture); + loadGithubEnv(fixture); + runPortableCpuDelegationProofMode("reject", fixture); + runPortableCpuDelegationProofMode("admit", fixture); + runPortableCpuDelegationProofMode("diagnostics", fixture); + runPortableCpuDelegationProofMode("cleanup", fixture); + const proofStates = fixture.runner.calls + .flatMap((call) => call.argv) + .filter((argument) => argument.startsWith("E2E_CPU_DELEGATION_STATE=")); + expect(proofStates).toEqual([ + "E2E_CPU_DELEGATION_STATE=missing", + "E2E_CPU_DELEGATION_STATE=delegated", + ]); + expect(fixture.runner.calls).toContainEqual( + expect.objectContaining({ + executable: "sudo", + argv: expect.arrayContaining(["systemctl", "daemon-reload"]), + }), + ); + expect( + fixture.runner.calls.every( + ({ executable, argv, options }) => + !(["bash", "sh"].includes(executable) && argv.includes("-c")) && !("shell" in options), + ), + ).toBe(true); + expect(fixture.runner.files.has(DELEGATION_DROP_IN)).toBe(false); + expect(fixture.runner.files.has(APP_DROP_IN)).toBe(false); + expect(fixture.runner.files.has(USER_SLICE_DROP_IN)).toBe(false); + expect(fs.existsSync(fixture.env.E2E_WORKSPACE_TRAVERSE_MARKER!)).toBe(false); + expect(fixture.runner.userCreated).toBe(false); + expect(fixture.filesystem.calls.map(({ operation }) => operation)).toEqual( + expect.arrayContaining([ + "appendText", + "exists", + "lstat", + "readText", + "removeDirectory", + "removeFile", + "writeExclusive", + ]), + ); + expect( + fixture.filesystem.calls.every( + ({ target }) => + target === fixture.env.GITHUB_ENV || + target.startsWith(fixture.env.GITHUB_WORKSPACE!) || + target.startsWith(fixture.env.RUNNER_TEMP!), + ), + ).toBe(true); + }); + }); + it.each([ + ["219/CGROUP", false, 1], + ["status=1/FAILURE", true, 0], + ] as const)( + "opens a later login only for an immediate %s manager-start failure (#9188)", + (diagnostic, shouldThrow, expectedLoginCount) => { + withProofFixture((fixture) => { + runPortableCpuDelegationProofMode("prepare", fixture); + loadGithubEnv(fixture); + fixture.runner.managerStartDiagnostic = diagnostic; + fixture.runner.managerStartFailures = 1; + const outcome = (() => { + try { + runPortableCpuDelegationProofMode("admit", fixture); + return ""; + } catch (error) { + return error instanceof Error ? error.message : String(error); + } + })(); + expect(outcome === "").toBe(!shouldThrow); + expect(outcome).toMatch(shouldThrow ? /failed without 219\/CGROUP/u : /^$/u); + expect( + fixture.runner.calls.filter(({ argv }) => argv[0] === "--login" && argv[1] === "--user"), + ).toHaveLength(expectedLoginCount); + }); + }, + ); + it("preserves identity and workspace-mode receipts when cleanup is incomplete (#9188)", () => { + withProofFixture((fixture) => { + const delegationMarker = path.join( + fixture.env.RUNNER_TEMP!, + "nemoclaw-cpu-delegation-drop-in-created", + ); + const workspaceMarker = path.join( + fixture.env.RUNNER_TEMP!, + "nemoclaw-workspace-traverse-modes", + ); + fs.writeFileSync(delegationMarker, "1:1\n", { mode: 0o600 }); + fs.writeFileSync(workspaceMarker, "755\t/checkout-parent\n", { mode: 0o600 }); + fixture.runner.seedFile(DELEGATION_DROP_IN, "2:2"); + fixture.runner.failWorkspaceModeRestore = true; + expect(() => runPortableCpuDelegationProofMode("cleanup", fixture)).toThrow( + /cleanup was incomplete/u, + ); + expect(fs.readFileSync(delegationMarker, "utf8")).toBe("1:1\n"); + expect(fs.readFileSync(workspaceMarker, "utf8")).toBe("755\t/checkout-parent\n"); + expect(fixture.runner.files.has(DELEGATION_DROP_IN)).toBe(true); + }); + }); + it("preserves a retry receipt when a cleanup predicate command fails (#9188)", () => { + withProofFixture((fixture) => { + const target = DELEGATION_DROP_IN; + const marker = path.join(fixture.env.RUNNER_TEMP!, "nemoclaw-cpu-delegation-drop-in-created"); + fs.writeFileSync(marker, "1:1\n", { mode: 0o600 }); + fixture.runner.seedFile(target, "1:1"); + fixture.runner.failTestFor = target; + fixture.env.E2E_CPU_DELEGATION_DROP_IN_CREATED = "1"; + expect(() => runPortableCpuDelegationProofMode("cleanup", fixture)).toThrow( + /sudo test fixture failure/u, + ); + expect(fs.readFileSync(marker, "utf8")).toBe("1:1\n"); + expect(fixture.runner.files.has(target)).toBe(true); + }); + }); + it("preserves the user claim when account inspection fails during cleanup (#9188)", () => { + withProofFixture((fixture) => { + runPortableCpuDelegationProofMode("prepare", fixture); + loadGithubEnv(fixture); + fixture.runner.failUserLookup = true; + expect(() => runPortableCpuDelegationProofMode("cleanup", fixture)).toThrow( + /getent passwd nemoclaw-e2e failed/u, + ); + expect(fixture.env.E2E_CPU_DELEGATION_USER_CLAIMED).toBe("1"); + expect(fixture.runner.userCreated).toBe(true); + }); + }); + it("records every unrecorded intent before its exact create command (#9188)", () => { + withProofFixture((fixture) => { + fixture.runner.directories.delete(path.dirname(APP_DROP_IN)); + fixture.runner.directories.delete(path.dirname(DELEGATION_DROP_IN)); + runPortableCpuDelegationProofMode("prepare", fixture); + loadGithubEnv(fixture); + const parent = path.join(fixture.env.GITHUB_WORKSPACE!, "node_modules/.cache"); + const cache = path.join(parent, "nemoclaw-source-require"); + const appTemp = fixture.env.E2E_APP_SLICE_DROP_IN_TEMP!; + const delegationTemp = fixture.env.E2E_CPU_DELEGATION_DROP_IN_TEMP!; + const userSliceTemp = fixture.env.E2E_USER_SLICE_DROP_IN_TEMP!; + const records = [ + [fixture.filesystem.envAtCreate.get(parent), "E2E_SOURCE_CACHE_PARENT_CREATED"], + [fixture.runner.envAtCreate.get(cache), "E2E_SOURCE_CACHE_CREATED"], + [ + fixture.runner.envAtCreate.get(path.dirname(APP_DROP_IN)), + "E2E_APP_SLICE_DROP_IN_DIR_CREATED", + ], + [ + fixture.runner.envAtCreate.get(path.dirname(DELEGATION_DROP_IN)), + "E2E_CPU_DELEGATION_DROP_IN_DIR_CREATED", + ], + [ + fixture.runner.envAtCreate.get(path.dirname(USER_SLICE_DROP_IN)), + "E2E_USER_SLICE_DROP_IN_DIR_CREATED", + ], + [fixture.runner.envAtCreate.get(appTemp), "E2E_APP_SLICE_DROP_IN_TEMP_CREATED"], + [fixture.runner.envAtCreate.get(delegationTemp), "E2E_CPU_DELEGATION_DROP_IN_TEMP_CREATED"], + [fixture.runner.envAtCreate.get(userSliceTemp), "E2E_USER_SLICE_DROP_IN_TEMP_CREATED"], + ] as const; + for (const [environment, name] of records) + expect(environment).toContain(`${name}=unrecorded\n`); + expect(fixture.runner.envAtCreate.get(appTemp)).toContain( + `E2E_APP_SLICE_DROP_IN_TEMP=${appTemp}\n`, + ); + expect(fixture.runner.envAtCreate.get(delegationTemp)).toContain( + `E2E_CPU_DELEGATION_DROP_IN_TEMP=${delegationTemp}\n`, + ); + expect(fixture.runner.envAtCreate.get(userSliceTemp)).toContain( + `E2E_USER_SLICE_DROP_IN_TEMP=${userSliceTemp}\n`, + ); + }); + }); + it.each(["app", "delegation", "userSlice", "cache", "staging"] as const)( + "rolls back an ordinary caught %s identity failure (#9188)", + (targetName) => { + withProofFixture((fixture) => { + const parent = path.join(fixture.env.GITHUB_WORKSPACE!, "node_modules/.cache"); + const targets = { + app: path.dirname(APP_DROP_IN), + cache: path.join(parent, "nemoclaw-source-require"), + delegation: path.dirname(DELEGATION_DROP_IN), + userSlice: path.dirname(USER_SLICE_DROP_IN), + staging: `${path.dirname(APP_DROP_IN)}/.nemoclaw-cpu-controller.00000000-0000-4000-8000-000000000000`, + } as const; + const target = targets[targetName]; + fixture.runner.directories.delete(target); + fixture.runner.failIdentityFor = target; + expect(() => runPortableCpuDelegationProofMode("prepare", fixture)).toThrow( + /identity fixture failure/u, + ); + expect(fixture.runner.hasResource(target)).toBe(false); + }); + }, + ); + it("rolls back an ordinary caught cache-parent identity failure (#9188)", () => { + withProofFixture((fixture) => { + const parent = path.join(fixture.env.GITHUB_WORKSPACE!, "node_modules/.cache"); + fixture.filesystem.failLstatOnceFor = parent; + fixture.filesystem.lstatSuccessesBeforeFailure = 1; + expect(() => runPortableCpuDelegationProofMode("prepare", fixture)).toThrow(/lstat fixture/u); + expect(fs.existsSync(parent)).toBe(false); + }); + }); + it.each(["app", "delegation", "userSlice", "cache", "staging"] as const)( + "recovers an exact %s post-create crash snapshot without an identity (#9188)", + (targetName) => { + withProofFixture((fixture) => { + const parent = path.join(fixture.env.GITHUB_WORKSPACE!, "node_modules/.cache"); + const targets = { + app: path.dirname(APP_DROP_IN), + cache: path.join(parent, "nemoclaw-source-require"), + delegation: path.dirname(DELEGATION_DROP_IN), + userSlice: path.dirname(USER_SLICE_DROP_IN), + staging: `${path.dirname(APP_DROP_IN)}/.nemoclaw-cpu-controller.00000000-0000-4000-8000-000000000000`, + } as const; + const receipts = { + app: [ + "E2E_APP_SLICE_DROP_IN_DIR_CREATED", + "E2E_APP_SLICE_DROP_IN_DIR_ID", + "root:root 755", + "nemoclaw-app-slice-drop-in-dir-created", + ], + cache: [ + "E2E_SOURCE_CACHE_CREATED", + "E2E_SOURCE_CACHE_ID", + "root:root 700", + "nemoclaw-source-require-cache-created", + ], + delegation: [ + "E2E_CPU_DELEGATION_DROP_IN_DIR_CREATED", + "E2E_CPU_DELEGATION_DROP_IN_DIR_ID", + "root:root 755", + "nemoclaw-cpu-delegation-drop-in-dir-created", + ], + userSlice: [ + "E2E_USER_SLICE_DROP_IN_DIR_CREATED", + "E2E_USER_SLICE_DROP_IN_DIR_ID", + "root:root 755", + "nemoclaw-user-slice-drop-in-dir-created", + ], + staging: [ + "E2E_APP_SLICE_DROP_IN_TEMP_CREATED", + "E2E_APP_SLICE_DROP_IN_TEMP_ID", + "root:root 700", + "nemoclaw-app-slice-drop-in-created", + ], + } as const; + const target = targets[targetName]; + const [createdEnv, idEnv, ownerMode, markerName] = receipts[targetName]; + Object.assign( + fixture.env, + targetName === "userSlice" + ? { + E2E_CPU_DELEGATION_UID: "1001", + E2E_USER_SLICE_DROP_IN: USER_SLICE_DROP_IN, + E2E_USER_SLICE_DROP_IN_DIR: path.dirname(USER_SLICE_DROP_IN), + E2E_USER_SLICE_DROP_IN_MARKER: path.join( + fixture.env.RUNNER_TEMP!, + "nemoclaw-user-slice-drop-in-created", + ), + E2E_USER_SLICE_DROP_IN_DIR_MARKER: path.join( + fixture.env.RUNNER_TEMP!, + "nemoclaw-user-slice-drop-in-dir-created", + ), + } + : {}, + ); + fixture.env[createdEnv] = "unrecorded"; + fixture.env[idEnv] = ""; + fixture.env.E2E_APP_SLICE_DROP_IN_TEMP = + targetName === "staging" ? target : fixture.env.E2E_APP_SLICE_DROP_IN_TEMP; + fixture.runner.directories.add(target); + fixture.runner.ownerModes.set(target, ownerMode); + expect(fixture.env[idEnv]).toBe(""); + expect(fs.existsSync(path.join(fixture.env.RUNNER_TEMP!, markerName))).toBe(false); + expect(fixture.runner.hasResource(target)).toBe(true); + runPortableCpuDelegationProofMode("cleanup", fixture); + expect(fixture.runner.hasResource(target)).toBe(false); + }); + }, + ); + it("recovers an exact cache-parent post-create crash snapshot without an identity (#9188)", () => { + withProofFixture((fixture) => { + const parent = path.join(fixture.env.GITHUB_WORKSPACE!, "node_modules/.cache"); + fixture.env.E2E_SOURCE_CACHE_PARENT_CREATED = "unrecorded"; + fixture.env.E2E_SOURCE_CACHE_PARENT_ID = ""; + fs.mkdirSync(parent); + expect(fixture.env.E2E_SOURCE_CACHE_PARENT_ID).toBe(""); + expect( + fs.existsSync( + path.join(fixture.env.RUNNER_TEMP!, "nemoclaw-source-require-cache-parent-created"), + ), + ).toBe(false); + expect(fs.existsSync(parent)).toBe(true); + runPortableCpuDelegationProofMode("cleanup", fixture); + expect(fs.existsSync(parent)).toBe(false); + }); + }); + it("rolls back earlier drop-ins when the final publication fails (#9188)", () => { + withProofFixture((fixture) => { + fixture.runner.failMoveFor = DELEGATION_DROP_IN; + expect(() => runPortableCpuDelegationProofMode("prepare", fixture)).toThrow( + /move fixture failure/u, + ); + expect(fixture.runner.files.has(APP_DROP_IN)).toBe(false); + expect(fixture.runner.files.has(DELEGATION_DROP_IN)).toBe(false); + expect(fixture.runner.files.has(USER_SLICE_DROP_IN)).toBe(false); + }); + }); + it.each([ + ["source cache", "cache", "nemoclaw-source-require-cache-created", "E2E_SOURCE_CACHE_ID"], + [ + "drop-in directory", + "directory", + "nemoclaw-user-slice-drop-in-dir-created", + "E2E_USER_SLICE_DROP_IN_DIR_ID", + ], + ["drop-in", "file", "nemoclaw-user-slice-drop-in-created", "E2E_USER_SLICE_DROP_IN_ID"], + ] as const)( + "retries a %s after receipt publication and rollback both fail (#9188)", + (_name, targetName, markerName, idEnv) => { + withProofFixture((fixture) => { + const targets = { + cache: path.join( + fixture.env.GITHUB_WORKSPACE!, + "node_modules/.cache/nemoclaw-source-require", + ), + directory: path.dirname(USER_SLICE_DROP_IN), + file: USER_SLICE_DROP_IN, + } as const; + const target = targets[targetName]; + const marker = path.join(fixture.env.RUNNER_TEMP!, markerName); + fixture.runner.directories.delete(targetName === "directory" ? target : ""); + fixture.filesystem.failWriteTarget = marker; + fixture.runner.failRemovalFor = target; + expect(() => runPortableCpuDelegationProofMode("prepare", fixture)).toThrow( + /filesystem fixture write failure/u, + ); + loadGithubEnv(fixture); + expect(fs.readFileSync(marker, "utf8")).toBe("concurrent\n"); + expect(fixture.env[idEnv]).toMatch(/^[0-9]+:[0-9]+$/u); + expect(fixture.runner.hasResource(target)).toBe(true); + fixture.runner.failRemovalFor = ""; + fs.unlinkSync(marker); + runPortableCpuDelegationProofMode("cleanup", fixture); + expect(fixture.runner.hasResource(target)).toBe(false); + }); + }, + ); + it("retries an identity-recorded temp after content and rollback fail (#9188)", () => { + withProofFixture((fixture) => { + fixture.runner.failTee = true; + fixture.runner.failRemovalPrefix = `${path.dirname(USER_SLICE_DROP_IN)}/.nemoclaw-cpu-controller.`; + expect(() => runPortableCpuDelegationProofMode("prepare", fixture)).toThrow(/tee fixture/u); + loadGithubEnv(fixture); + const temporary = fixture.env.E2E_USER_SLICE_DROP_IN_TEMP!; + expect(fixture.env.E2E_USER_SLICE_DROP_IN_TEMP_ID).toMatch(/^[0-9]+:[0-9]+$/u); + expect(fixture.runner.directories.has(temporary)).toBe(true); + fixture.runner.failRemovalPrefix = ""; + runPortableCpuDelegationProofMode("cleanup", fixture); + expect(fixture.runner.directories.has(temporary)).toBe(false); + }); + }); + it("keeps import inert and rejects invalid CLI argv before a real cleanup subprocess (#9188)", () => { + withProofFixture((fixture) => { + const bin = path.join(fixture.directory, "bin"); + const scriptPath = path.resolve("scripts/checks/run-portable-cpu-delegation-proof.mts"); + fs.mkdirSync(bin); + fs.writeFileSync(path.join(bin, "sudo"), "#!/usr/bin/env node\nprocess.exit(0);\n", { + mode: 0o755, + }); + const env = { + ...process.env, + ...fixture.env, + PATH: `${bin}:${process.env.PATH ?? ""}`, + }; + const imported = spawnSync( + process.execPath, + [ + "--experimental-strip-types", + "--no-warnings", + "--input-type=module", + "-e", + `await import(${JSON.stringify(pathToFileURL(scriptPath).href)})`, + ], + { cwd: path.resolve("."), encoding: "utf8", env }, + ); + expect(imported.status).toBe(0); + expect(imported.stdout).toBe(""); + expect(() => portableCpuDelegationProofCli(["cleanup", "extra"], fixture)).toThrow( + /Expected exactly one mode/u, + ); + const rejected = spawnSync( + process.execPath, + ["--experimental-strip-types", "--no-warnings", scriptPath, "unknown"], + { cwd: path.resolve("."), encoding: "utf8", env }, + ); + expect(rejected.status).toBe(1); + expect(rejected.stderr).toContain("Expected exactly one mode"); + const cleaned = spawnSync( + process.execPath, + ["--experimental-strip-types", "--no-warnings", scriptPath, "cleanup"], + { cwd: path.resolve("."), encoding: "utf8", env }, + ); + expect(cleaned.status).toBe(0); + fs.writeFileSync( + path.join(bin, "sudo"), + "#!/usr/bin/env node\nprocess.kill(process.pid, 'SIGTERM');\n", + { mode: 0o755 }, + ); + const signaled = spawnSync( + process.execPath, + ["--experimental-strip-types", "--no-warnings", scriptPath, "cleanup"], + { cwd: path.resolve("."), encoding: "utf8", env }, + ); + expect(signaled.status).toBe(1); + expect(signaled.stderr).toContain("sudo terminated by SIGTERM"); + }); }); it("pins one rootless socket and fails closed on Docker use", () => { diff --git a/test/portable-cpu-delegation-docs.test.ts b/test/portable-cpu-delegation-docs.test.ts new file mode 100644 index 00000000000..97fc033d99e --- /dev/null +++ b/test/portable-cpu-delegation-docs.test.ts @@ -0,0 +1,1183 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +import { PORTABLE_CPU_DELEGATION_PROOF_CONTRACT } from "../scripts/checks/run-portable-cpu-delegation-proof.mts"; + +const repoRoot = path.join(import.meta.dirname, ".."); +const troubleshootingPath = path.join(repoRoot, "docs", "reference", "troubleshooting.mdx"); +const temporaryDirectories: string[] = []; + +type CommandFixture = { + appSliceDropIn: string; + command: string; + delegationDropIn: string; + environment: NodeJS.ProcessEnv; + mkdirCallMarker: string; + userSliceDropIn: string; +}; + +type RollbackFixture = { + appSliceDropIn: string; + appSliceDropInDirectory: string; + command: string; + delegationDropIn: string; + delegationDropInDirectory: string; + environment: NodeJS.ProcessEnv; + systemctlCallMarker: string; + userSliceDropIn: string; + userSliceDropInDirectory: string; +}; + +type RollbackFixtureOptions = { + appSliceDropInCreated?: boolean; + delegationDropInCreated?: boolean; + expectedDelegationDropInId?: string; + userSliceDropInCreated?: boolean; +}; + +function extractFirstBashCommandAfter(anchor: string): string { + const markdown = fs.readFileSync(troubleshootingPath, "utf8"); + const sectionStart = markdown.indexOf(anchor); + expect(sectionStart).toBeGreaterThanOrEqual(0); + const section = markdown.slice(sectionStart); + const block = section.match(/```bash\n([\s\S]*?)\n```/u); + expect(block).not.toBeNull(); + return block?.[1] ?? ""; +} + +function extractDropInCreationCommand(): string { + return extractFirstBashCommandAfter("Use the three dedicated NemoClaw drop-in paths below."); +} + +function extractControllerClassificationCommand(): string { + return extractFirstBashCommandAfter("### Portable CPU Delegation Preflight Fails"); +} + +function extractMalformedEvidenceInspectionCommand(): string { + return extractFirstBashCommandAfter("Do not print malformed bytes directly to a terminal."); +} + +function extractPartialCreationRollbackCommand(): string { + return extractFirstBashCommandAfter("#### Clean Up a Partial Drop-In Creation"); +} + +function extractUnrecordedDirectoryRecoveryCommand(): string { + return extractFirstBashCommandAfter("#### Recover an Unrecorded Drop-In Directory"); +} + +function extractApplyCommand(): string { + return extractFirstBashCommandAfter("Run the stop, reload, and start sequence:"); +} + +function extractFinalControllerVerificationCommand(): string { + return extractFirstBashCommandAfter( + "Verify the root hierarchy, current user manager, and `app.slice`:", + ); +} + +function extractDropInRollbackCommand(): string { + const markdown = fs.readFileSync(troubleshootingPath, "utf8"); + const sectionStart = markdown.indexOf("#### Remove the CPU Controller Drop-Ins"); + expect(sectionStart).toBeGreaterThanOrEqual(0); + + const section = markdown.slice(sectionStart); + const sectionEnd = section.indexOf("\n### Portable Podman Readiness Fails"); + expect(sectionEnd).toBeGreaterThanOrEqual(0); + const blocks = [...section.slice(0, sectionEnd).matchAll(/```bash\n([\s\S]*?)\n```/gu)]; + expect(blocks).toHaveLength(3); + return blocks[1]?.[1] ?? ""; +} + +function makeTemporaryDirectory(): string { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cpu-delegation-docs-")); + temporaryDirectories.push(directory); + return directory; +} + +function makeCommandFixture(): CommandFixture { + const root = makeTemporaryDirectory(); + const delegationDropIn = path.join(root, "system", "90-nemoclaw-cpu-delegation.conf"); + const appSliceDropIn = path.join(root, "user", "90-nemoclaw-cpu-controller.conf"); + const userSliceDropIn = path.join(root, "user-slice", "90-nemoclaw-cpu-controller.conf"); + const fakeBin = path.join(root, "bin"); + const mkdirCallMarker = path.join(root, "mkdir-call"); + const linkCallMarker = path.join(root, "link-call"); + const sudo = path.join(fakeBin, "sudo"); + fs.mkdirSync(fakeBin); + fs.writeFileSync( + sudo, + `#!/bin/sh +set -eu +if [ -n "\${FAIL_PREDICATE_PATH:-}" ] && [ "\${1-}" = sh ] && [ "\${5-}" = "$FAIL_PREDICATE_PATH" ]; then printf '%s\\n' 'simulated predicate inspection failure' >&2; exit 77; fi +if [ "\${1-}" = mkdir ]; then + mkdir_call=1 + if [ -e "$MKDIR_CALL_MARKER" ]; then + mkdir_call=$(( $(cat "$MKDIR_CALL_MARKER") + 1 )) + fi + printf '%s\\n' "$mkdir_call" > "$MKDIR_CALL_MARKER" + if [ "$mkdir_call" -eq "\${FAIL_MKDIR_CALL:-0}" ]; then + printf '%s\\n' 'simulated directory creation failure' >&2 + exit 73 + fi + exec mkdir -m "$3" "$5" +fi +if [ "\${1-}" = stat ]; then + if [ "\${3-}" = "%d:%i" ]; then + case "\${5##*/}" in + .nemoclaw-cpu-controller.*) is_staging_dir=1 ;; + *) is_staging_dir=0 ;; + esac + if [ "\${FAIL_STAGING_STAT:-0}" = 1 ] && [ "$is_staging_dir" = 1 ]; then + printf '%s\\n' 'simulated staging identity recording failure' >&2 + exit 76 + elif [ "\${FAIL_STAT_ID_PATH:-}" = "\${5-}" ]; then + printf '%s\\n' 'simulated identity recording failure' >&2 + exit 76 + elif [ "\${STAT_ID_OVERRIDE_PATH:-}" = "\${5-}" ]; then + printf '%s\\n' "\${STAT_ID_OVERRIDE:-0:0}" + else + printf '%s\\n' '1:1' + fi + elif [ "\${SUDO_SCENARIO:-}" = existing-directory-metadata ]; then + printf '%s\\n' 'root:root 750' + else + case "\${5##*/}" in + .nemoclaw-cpu-controller.*) printf '%s\\n' 'root:root 700' ;; + *) printf '%s\\n' 'root:root 755' ;; + esac + fi + exit 0 +fi +if [ "\${1-}" = chown ]; then + exit 0 +fi +if [ "\${1-}" = chmod ]; then + exec chmod "$2" "$4" +fi +if [ "\${1-}" = sh ] && [ "\${SUDO_SCENARIO:-}" = write-failure ]; then + case "\${3-}" in + *'cat >'*) + printf '%s\\n' 'partial content' > "$5" + printf '%s\\n' 'simulated temporary file write failure' >&2 + exit 74 + ;; + esac +fi +if [ "\${1-}" = ln ]; then + link_call=1 + if [ -e "$LINK_CALL_MARKER" ]; then + link_call=$(( $(cat "$LINK_CALL_MARKER") + 1 )) + fi + printf '%s\\n' "$link_call" > "$LINK_CALL_MARKER" + if [ "$link_call" -eq "\${FAIL_LINK_CALL:-0}" ]; then + printf '%s\\n' 'simulated publish link failure' >&2 + exit 75 + fi + if [ "\${SUDO_SCENARIO:-}" = concurrent ]; then + printf '%s\\n' 'concurrent content' > "$FAILURE_TARGET" + fi + ln "$4" "$5" + if [ "$link_call" -eq "\${FAIL_AFTER_LINK_CALL:-0}" ]; then + printf '%s\\n' 'simulated interruption after publish link' >&2 + exit 78 + fi + exit 0 +fi +exec "$@" +`, + { mode: 0o755 }, + ); + + const command = extractDropInCreationCommand() + .replace('uid="$(id -u)"', 'uid="1000"') + .replace( + 'delegation_drop_in="/etc/systemd/system/user@.service.d/90-nemoclaw-cpu-delegation.conf"', + `delegation_drop_in=${JSON.stringify(delegationDropIn)}`, + ) + .replace( + 'app_slice_drop_in="/etc/systemd/user/app.slice.d/90-nemoclaw-cpu-controller.conf"', + `app_slice_drop_in=${JSON.stringify(appSliceDropIn)}`, + ) + .replace( + 'user_slice_drop_in="/etc/systemd/system/user-${uid}.slice.d/90-nemoclaw-cpu-controller.conf"', + `user_slice_drop_in=${JSON.stringify(userSliceDropIn)}`, + ); + + return { + appSliceDropIn, + command, + delegationDropIn, + environment: { + ...process.env, + FAILURE_TARGET: delegationDropIn, + LINK_CALL_MARKER: linkCallMarker, + LC_ALL: "C", + MKDIR_CALL_MARKER: mkdirCallMarker, + PATH: `${fakeBin}:${process.env.PATH ?? ""}`, + }, + mkdirCallMarker, + userSliceDropIn, + }; +} + +function fileIdentity(filePath: string): string { + const metadata = fs.statSync(filePath); + return `${metadata.dev}:${metadata.ino}`; +} + +function makeRollbackFixture(options: RollbackFixtureOptions = {}): RollbackFixture { + const root = makeTemporaryDirectory(); + const delegationDropInDirectory = path.join(root, "system"); + const appSliceDropInDirectory = path.join(root, "user"); + const userSliceDropInDirectory = path.join(root, "user-slice"); + const delegationDropIn = path.join(delegationDropInDirectory, "90-nemoclaw-cpu-delegation.conf"); + const appSliceDropIn = path.join(appSliceDropInDirectory, "90-nemoclaw-cpu-controller.conf"); + const userSliceDropIn = path.join(userSliceDropInDirectory, "90-nemoclaw-cpu-controller.conf"); + const fakeBin = path.join(root, "bin"); + const systemctlCallMarker = path.join(root, "systemctl-calls"); + const sudo = path.join(fakeBin, "sudo"); + const delegationDropInCreated = options.delegationDropInCreated ?? true; + const appSliceDropInCreated = options.appSliceDropInCreated ?? true; + const userSliceDropInCreated = options.userSliceDropInCreated ?? true; + fs.mkdirSync(delegationDropInDirectory); + fs.mkdirSync(appSliceDropInDirectory); + fs.mkdirSync(userSliceDropInDirectory); + fs.mkdirSync(fakeBin); + [ + { + content: "[Service]\nDelegate=cpu memory pids\n", + created: delegationDropInCreated, + file: delegationDropIn, + }, + { + content: "[Slice]\nCPUWeight=100\n", + created: appSliceDropInCreated, + file: appSliceDropIn, + }, + { + content: "[Slice]\nCPUWeight=100\n", + created: userSliceDropInCreated, + file: userSliceDropIn, + }, + ] + .filter(({ created }) => created) + .forEach(({ content, file }) => fs.writeFileSync(file, content)); + fs.writeFileSync( + sudo, + `#!/bin/sh +set -eu +if [ -n "\${FAIL_PREDICATE_PATH:-}" ] && [ "\${1-}" = sh ] && [ "\${5-}" = "$FAIL_PREDICATE_PATH" ]; then printf '%s\\n' 'simulated predicate inspection failure' >&2; exit 77; fi +if [ "\${1-}" = systemctl ]; then + printf '%s\n' "$*" >> "$SYSTEMCTL_CALL_MARKER" + if [ "\${2-}" = start ] && [ "\${START_FAILURE_219:-0}" = 1 ]; then + printf '%s\n' 'Job failed with result cgroup' >&2 + exit 1 + fi + if [ "\${2-}" = status ] && [ "\${START_FAILURE_219:-0}" = 1 ]; then + printf '%s\n' 'Process: 100 ExecStart=/bin/false (code=exited, status=219/CGROUP)' + exit 3 + fi + exit 0 +fi +if [ "\${1-}" = stat ]; then + if [ "\${2-}" != -Lc ] || [ "\${4-}" != -- ] || [ "$#" -ne 5 ]; then + printf 'unexpected stat invocation: %s\n' "$*" >&2 + exit 1 + fi + exec node -e ' + const fs = require("node:fs"); + const metadata = fs.statSync(process.argv[2]); + process.stdout.write( + process.argv[1] + .replace("%d", String(metadata.dev)) + .replace("%i", String(metadata.ino)), + ); + ' "$3" "$5" +fi +if [ "\${1-}" = rm ] || [ "\${1-}" = rmdir ]; then + command="$1" + shift + if [ "\${1-}" = -- ]; then + shift + fi + exec "$command" "$@" +fi +exec "$@" +`, + { mode: 0o755 }, + ); + + const command = extractDropInRollbackCommand() + .replace('uid=""', 'uid="1000"') + .replace( + 'delegation_drop_in="/etc/systemd/system/user@.service.d/90-nemoclaw-cpu-delegation.conf"', + `delegation_drop_in=${JSON.stringify(delegationDropIn)}`, + ) + .replace( + 'app_slice_drop_in="/etc/systemd/user/app.slice.d/90-nemoclaw-cpu-controller.conf"', + `app_slice_drop_in=${JSON.stringify(appSliceDropIn)}`, + ) + .replace( + 'user_slice_drop_in="/etc/systemd/system/user-${uid}.slice.d/90-nemoclaw-cpu-controller.conf"', + `user_slice_drop_in=${JSON.stringify(userSliceDropIn)}`, + ) + .replace( + 'delegation_drop_in_created=""', + `delegation_drop_in_created=${JSON.stringify(delegationDropInCreated ? "1" : "0")}`, + ) + .replace( + 'expected_delegation_drop_in_id=""', + `expected_delegation_drop_in_id=${JSON.stringify( + delegationDropInCreated + ? (options.expectedDelegationDropInId ?? fileIdentity(delegationDropIn)) + : "", + )}`, + ) + .replace( + 'app_slice_drop_in_created=""', + `app_slice_drop_in_created=${JSON.stringify(appSliceDropInCreated ? "1" : "0")}`, + ) + .replace( + 'expected_app_slice_drop_in_id=""', + `expected_app_slice_drop_in_id=${JSON.stringify( + appSliceDropInCreated ? fileIdentity(appSliceDropIn) : "", + )}`, + ) + .replace( + 'user_slice_drop_in_created=""', + `user_slice_drop_in_created=${JSON.stringify(userSliceDropInCreated ? "1" : "0")}`, + ) + .replace( + 'expected_user_slice_drop_in_id=""', + `expected_user_slice_drop_in_id=${JSON.stringify( + userSliceDropInCreated ? fileIdentity(userSliceDropIn) : "", + )}`, + ) + .replace( + 'delegation_drop_in_dir_created=""', + 'delegation_drop_in_dir_created="1"', + ) + .replace( + 'delegation_drop_in_dir_id=""', + `delegation_drop_in_dir_id=${JSON.stringify(fileIdentity(delegationDropInDirectory))}`, + ) + .replace( + 'app_slice_drop_in_dir_created=""', + 'app_slice_drop_in_dir_created="1"', + ) + .replace( + 'app_slice_drop_in_dir_id=""', + `app_slice_drop_in_dir_id=${JSON.stringify(fileIdentity(appSliceDropInDirectory))}`, + ) + .replace( + 'user_slice_drop_in_dir_created=""', + 'user_slice_drop_in_dir_created="1"', + ) + .replace( + 'user_slice_drop_in_dir_id=""', + `user_slice_drop_in_dir_id=${JSON.stringify(fileIdentity(userSliceDropInDirectory))}`, + ); + + return { + appSliceDropIn, + appSliceDropInDirectory, + command, + delegationDropIn, + delegationDropInDirectory, + environment: { + ...process.env, + LC_ALL: "C", + PATH: `${fakeBin}:${process.env.PATH ?? ""}`, + SYSTEMCTL_CALL_MARKER: systemctlCallMarker, + }, + systemctlCallMarker, + userSliceDropIn, + userSliceDropInDirectory, + }; +} + +function runDocumentedCommand(fixture: CommandFixture, environment: NodeJS.ProcessEnv = {}) { + return spawnSync("bash", ["-c", fixture.command], { + encoding: "utf8", + env: { ...fixture.environment, ...environment }, + }); +} + +function runDocumentedRollback(fixture: RollbackFixture, environment: NodeJS.ProcessEnv = {}) { + return spawnSync("bash", ["-c", fixture.command], { + encoding: "utf8", + env: { ...fixture.environment, ...environment }, + }); +} + +function finalRecord(output: string, name: string): string | undefined { + const matches = [...output.matchAll(new RegExp(`Record for rollback: ${name}=([^\\n]*)`, "gu"))]; + return matches.at(-1)?.[1]; +} + +function partialCreationRollbackCommand(fixture: CommandFixture, creationOutput: string): string { + const values = new Map(); + for (const prefix of ["delegation", "app_slice", "user_slice"]) { + for (const suffix of [ + "drop_in_created", + "drop_in_id", + "drop_in_dir_created", + "drop_in_dir_id", + "staging_dir_path", + "staging_dir_created", + "staging_dir_id", + ]) { + const name = `${prefix}_${suffix}`; + values.set(name, finalRecord(creationOutput, name) ?? ""); + } + } + + let command = extractPartialCreationRollbackCommand() + .replace( + 'delegation_drop_in="/etc/systemd/system/user@.service.d/90-nemoclaw-cpu-delegation.conf"', + `delegation_drop_in=${JSON.stringify(fixture.delegationDropIn)}`, + ) + .replace( + 'app_slice_drop_in="/etc/systemd/user/app.slice.d/90-nemoclaw-cpu-controller.conf"', + `app_slice_drop_in=${JSON.stringify(fixture.appSliceDropIn)}`, + ) + .replace( + 'user_slice_drop_in="/etc/systemd/system/user-.slice.d/90-nemoclaw-cpu-controller.conf"', + `user_slice_drop_in=${JSON.stringify(fixture.userSliceDropIn)}`, + ); + + for (const [name, value] of values) { + command = command.replace( + new RegExp(`${name}="<[^"]+>"`, "u"), + `${name}=${JSON.stringify(value)}`, + ); + } + return command; +} + +function runPartialCreationRollback( + fixture: CommandFixture, + creationOutput: string, + environment: NodeJS.ProcessEnv = {}, +) { + return spawnSync("bash", ["-c", partialCreationRollbackCommand(fixture, creationOutput)], { + encoding: "utf8", + env: { ...fixture.environment, ...environment }, + }); +} + +function runUnrecordedDirectoryRecovery( + fixture: CommandFixture, + directory: string, + environment: NodeJS.ProcessEnv = {}, + creationOutput = "", +) { + const command = extractUnrecordedDirectoryRecoveryCommand() + .replace( + 'unrecorded_directory=""', + `unrecorded_directory=${JSON.stringify(directory)}`, + ) + .replace( + 'delegation_drop_in_dir="/etc/systemd/system/user@.service.d"', + `delegation_drop_in_dir=${JSON.stringify(path.dirname(fixture.delegationDropIn))}`, + ) + .replace( + 'app_slice_drop_in_dir="/etc/systemd/user/app.slice.d"', + `app_slice_drop_in_dir=${JSON.stringify(path.dirname(fixture.appSliceDropIn))}`, + ) + .replace( + 'user_slice_drop_in_dir="/etc/systemd/system/user-.slice.d"', + `user_slice_drop_in_dir=${JSON.stringify(path.dirname(fixture.userSliceDropIn))}`, + ) + .replace( + 'delegation_staging_dir_path=""', + `delegation_staging_dir_path=${JSON.stringify(finalRecord(creationOutput, "delegation_staging_dir_path") ?? "")}`, + ) + .replace( + 'app_slice_staging_dir_path=""', + `app_slice_staging_dir_path=${JSON.stringify(finalRecord(creationOutput, "app_slice_staging_dir_path") ?? "")}`, + ) + .replace( + 'user_slice_staging_dir_path=""', + `user_slice_staging_dir_path=${JSON.stringify(finalRecord(creationOutput, "user_slice_staging_dir_path") ?? "")}`, + ); + + return spawnSync("bash", ["-c", command], { + encoding: "utf8", + env: { ...fixture.environment, ...environment }, + }); +} + +function runDocumentedApply(fixture: RollbackFixture) { + const command = extractApplyCommand().replace('uid=""', 'uid="1000"'); + return spawnSync("bash", ["-c", command], { + encoding: "utf8", + env: { ...fixture.environment, START_FAILURE_219: "1" }, + }); +} + +function runClassificationWithUserManagerEvidence(evidence: Buffer | string) { + const root = makeTemporaryDirectory(); + const rootControllers = path.join(root, "root.controllers"); + const userSlice = path.join(root, "user-slice"); + const userSliceControllers = path.join(userSlice, "cgroup.controllers"); + const userManager = path.join(root, "user-manager"); + const userManagerControllers = path.join(userManager, "cgroup.controllers"); + const appSliceControllers = path.join(userManager, "app.slice", "cgroup.controllers"); + fs.mkdirSync(path.dirname(appSliceControllers), { recursive: true }); + fs.mkdirSync(userSlice); + fs.writeFileSync(rootControllers, "cpuset cpu memory pids\n"); + fs.writeFileSync(userSliceControllers, "cpu memory pids\n"); + fs.writeFileSync(userManagerControllers, evidence); + fs.writeFileSync(appSliceControllers, "cpu memory pids\n"); + const command = extractControllerClassificationCommand() + .replace('uid="$(id -u)"', 'uid="1000"') + .replace( + 'user_slice="/sys/fs/cgroup/user.slice/user-${uid}.slice"', + `user_slice=${JSON.stringify(userSlice)}`, + ) + .replace( + 'user_manager="/sys/fs/cgroup/user.slice/user-${uid}.slice/user@${uid}.service"', + `user_manager=${JSON.stringify(userManager)}`, + ) + .replace("/sys/fs/cgroup/cgroup.controllers", JSON.stringify(rootControllers)); + const verificationCommand = extractFinalControllerVerificationCommand() + .replace('uid="$(id -u)"', 'uid="1000"') + .replace( + 'user_slice="/sys/fs/cgroup/user.slice/user-${uid}.slice"', + `user_slice=${JSON.stringify(userSlice)}`, + ) + .replace( + 'user_manager="/sys/fs/cgroup/user.slice/user-${uid}.slice/user@${uid}.service"', + `user_manager=${JSON.stringify(userManager)}`, + ) + .replace("/sys/fs/cgroup/cgroup.controllers", JSON.stringify(rootControllers)); + + return { + appSliceControllers, + command, + result: spawnSync("bash", ["-c", command], { encoding: "utf8" }), + rootControllers, + userManagerControllers, + userSliceControllers, + verificationCommand, + verificationResult: spawnSync("bash", ["-c", verificationCommand], { encoding: "utf8" }), + }; +} + +function listTemporaryDropIns(fixture: CommandFixture): string[] { + const directories = new Set([ + path.dirname(fixture.delegationDropIn), + path.dirname(fixture.appSliceDropIn), + path.dirname(fixture.userSliceDropIn), + ]); + + return [...directories] + .filter((directory) => fs.existsSync(directory)) + .flatMap((directory) => + fs + .readdirSync(directory) + .filter((entry) => entry.startsWith(".nemoclaw-cpu-controller.")) + .map((entry) => path.join(directory, entry)), + ); +} + +afterEach(() => { + for (const directory of temporaryDirectories.splice(0)) { + fs.rmSync(directory, { force: true, recursive: true }); + } +}); + +describe("portable CPU delegation documentation (#9188)", () => { + it("classifies malformed controller evidence without printing its content (#9188)", () => { + const { + appSliceControllers, + command, + result, + rootControllers, + userManagerControllers, + userSliceControllers, + verificationResult, + } = runClassificationWithUserManagerEvidence("cpu memory\nDelegate=cpu\n"); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(""); + expect(result.stdout).toContain(`${rootControllers}: readable (cpuset cpu memory pids)`); + expect(result.stdout).toContain(`${userSliceControllers}: readable (cpu memory pids)`); + expect(result.stdout).toContain(`${userManagerControllers}: malformed`); + expect(result.stdout).toContain(`${appSliceControllers}: readable (cpu memory pids)`); + expect(result.stdout).not.toContain("Delegate=cpu"); + expect(command).toContain("Buffer.alloc(4097)"); + expect(command).toContain("content.length > 4096"); + expect(command).not.toContain('cat -- "$controllers"'); + expect(verificationResult.status).not.toBe(0); + expect(verificationResult.stdout).toContain(`${userManagerControllers}: malformed`); + expect(verificationResult.stdout).not.toContain("Delegate=cpu"); + }); + + it.each([ + { caseName: "NUL-containing", evidence: Buffer.from("cpu\0memory\n"), leaked: "cpu\0memory" }, + { caseName: "invalid UTF-8", evidence: Buffer.from([0x63, 0x70, 0x75, 0xff]), leaked: "�" }, + { caseName: "oversized", evidence: Buffer.alloc(1024 * 1024, 0x61), leaked: "a".repeat(256) }, + { caseName: "duplicate", evidence: "cpu memory cpu\n", leaked: "cpu memory cpu" }, + ])("classifies $caseName evidence as malformed (#9188)", ({ evidence, leaked }) => { + const { result, userManagerControllers, verificationResult } = + runClassificationWithUserManagerEvidence(evidence); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(""); + expect(result.stdout).toContain(`${userManagerControllers}: malformed`); + expect(result.stdout).not.toContain(leaked); + expect(result.stdout.length).toBeLessThan(1024); + expect(verificationResult.status).not.toBe(0); + expect(verificationResult.stdout).toContain(`${userManagerControllers}: malformed`); + expect(verificationResult.stdout).not.toContain(leaked); + }); + + it("refuses to inspect malformed evidence outside the exact controller paths (#9188)", () => { + const root = makeTemporaryDirectory(); + const rootControllers = path.join(root, "root.controllers"); + const userManager = path.join(root, "user-manager"); + const unexpectedPath = path.join(root, "unexpected.controllers"); + const command = extractMalformedEvidenceInspectionCommand() + .replace( + 'reported_path=""', + `reported_path=${JSON.stringify(unexpectedPath)}`, + ) + .replace('uid="$(id -u)"', 'uid="1000"') + .replace( + 'user_manager="/sys/fs/cgroup/user.slice/user-${uid}.slice/user@${uid}.service"', + `user_manager=${JSON.stringify(userManager)}`, + ) + .replace("/sys/fs/cgroup/cgroup.controllers", rootControllers); + + const result = spawnSync("bash", ["-c", command], { encoding: "utf8" }); + + expect(result.status).not.toBe(0); + expect(result.stdout).toBe(""); + expect(result.stderr).toContain(`Refusing unexpected cgroup evidence path: ${unexpectedPath}`); + expect(command).toContain('"${user_manager}/cgroup.controllers"'); + expect(command).toContain('"${user_manager}/app.slice/cgroup.controllers"'); + expect(command).toContain('"${user_slice}/cgroup.controllers"'); + }); + + it("bounds hexadecimal inspection for an exact controller evidence path (#9188)", () => { + const root = makeTemporaryDirectory(); + const rootControllers = path.join(root, "root.controllers"); + const userManager = path.join(root, "user-manager"); + const fakeBin = path.join(root, "bin"); + const sudo = path.join(fakeBin, "sudo"); + const findmnt = path.join(fakeBin, "findmnt"); + fs.mkdirSync(fakeBin); + fs.writeFileSync( + rootControllers, + Buffer.concat([Buffer.alloc(256, 0x41), Buffer.from("SECRET_AFTER_LIMIT", "utf8")]), + ); + fs.writeFileSync(sudo, '#!/bin/sh\nexec "$@"\n', { mode: 0o755 }); + fs.writeFileSync(findmnt, "#!/bin/sh\nprintf '%s\\n' 'bounded mount inspection'\n", { + mode: 0o755, + }); + const command = extractMalformedEvidenceInspectionCommand() + .replace( + 'reported_path=""', + `reported_path=${JSON.stringify(rootControllers)}`, + ) + .replace('uid="$(id -u)"', 'uid="1000"') + .replace( + 'user_manager="/sys/fs/cgroup/user.slice/user-${uid}.slice/user@${uid}.service"', + `user_manager=${JSON.stringify(userManager)}`, + ) + .replace("/sys/fs/cgroup/cgroup.controllers", JSON.stringify(rootControllers)); + + const result = spawnSync("bash", ["-c", command], { + encoding: "utf8", + env: { ...process.env, LC_ALL: "C", PATH: `${fakeBin}:${process.env.PATH ?? ""}` }, + }); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(""); + expect(result.stdout).toMatch(/41\s+41\s+41\s+41/u); + expect(result.stdout).toContain("bounded mount inspection"); + expect(result.stdout).not.toContain("53 45 43 52 45 54"); + expect(command).toContain('od -An -tx1 -N 256 -v -- "$reported_path"'); + }); + + it("keeps the three-setting repair and warns before each host interruption (#9188)", () => { + const markdown = fs.readFileSync(troubleshootingPath, "utf8"); + const sectionStart = markdown.indexOf("### Portable CPU Delegation Preflight Fails"); + const sectionEnd = markdown.indexOf("### Portable Podman Readiness Fails", sectionStart); + const section = markdown.slice(sectionStart, sectionEnd); + const rollbackStart = section.indexOf("#### Remove the CPU Controller Drop-Ins"); + const applySection = section.slice(0, rollbackStart); + const rollbackSection = section.slice(rollbackStart); + const applyWarning = applySection.indexOf("Save the affected user's work"); + const applyStop = applySection.indexOf('sudo systemctl stop "user@${uid}.service"'); + const rebootWarning = applySection.indexOf("Save work for every host user"); + const reboot = applySection.indexOf("sudo systemctl reboot"); + const rollbackWarning = rollbackSection.indexOf("Save the affected user's work"); + const rollbackStop = rollbackSection.indexOf('sudo systemctl stop "user@${uid}.service"'); + const cleanupRouting = applySection.indexOf( + "Choose the cleanup route that matches the final records:", + ); + const incompleteReceiptRoute = applySection.indexOf( + "If any final `*_created` value is `0` but the same command printed its matching `*_id`", + ); + const unrecordedCleanupRoute = applySection.indexOf( + "If any final `*_drop_in_dir_created` or `*_staging_dir_created` value is `unrecorded`", + ); + const recordedCleanupRoute = applySection.indexOf( + "Otherwise, when every final `*_created` value is `0` or `1`", + ); + const unrecordedCleanupHeading = applySection.indexOf( + "#### Recover an Unrecorded Drop-In Directory", + ); + const generalCleanupHeading = applySection.indexOf("#### Clean Up a Partial Drop-In Creation"); + + const creationCommand = extractDropInCreationCommand(); + + expect(creationCommand).toContain( + `delegation_drop_in=${JSON.stringify(PORTABLE_CPU_DELEGATION_PROOF_CONTRACT.delegationDropIn)}`, + ); + expect(creationCommand).toContain( + `app_slice_drop_in=${JSON.stringify(PORTABLE_CPU_DELEGATION_PROOF_CONTRACT.appSliceDropIn)}`, + ); + expect(creationCommand).toContain( + 'user_slice_drop_in="/etc/systemd/system/user-${uid}.slice.d/90-nemoclaw-cpu-controller.conf"', + ); + expect(creationCommand).toContain( + "delegation \"$delegation_drop_in\" '[Service]' 'Delegate=cpu memory pids'", + ); + expect(creationCommand).toContain( + "create_drop_in app_slice \"$app_slice_drop_in\" '[Slice]' 'CPUWeight=100'", + ); + expect(creationCommand).toContain( + "create_drop_in user_slice \"$user_slice_drop_in\" '[Slice]' 'CPUWeight=100'", + ); + expect(PORTABLE_CPU_DELEGATION_PROOF_CONTRACT.controllerEvidenceReadBytes).toBe(4097); + expect(PORTABLE_CPU_DELEGATION_PROOF_CONTRACT.immediateStartFailure).toBe("219/CGROUP"); + expect(creationCommand).toContain("randomBytes(16)"); + expect(creationCommand).not.toContain("mktemp"); + expect(creationCommand.indexOf("drop_in_dir_created=unrecorded")).toBeLessThan( + creationCommand.indexOf("sudo mkdir -m 0755"), + ); + expect(creationCommand.indexOf("staging_dir_created=unrecorded")).toBeLessThan( + creationCommand.indexOf("sudo mkdir -m 0700"), + ); + expect(creationCommand).not.toContain("CPUAccounting"); + expect(applySection).toContain('"${user_slice}/cgroup.controllers"'); + expect(applySection).toContain("printf '%s: malformed\\n' \"$controllers\""); + expect(applySection).toContain( + "Do not use a boot, delegation, or service lifecycle action to correct an unreadable or malformed file.", + ); + expect(applySection).toContain( + 'start_output="$(sudo systemctl start "user@${uid}.service" 2>&1)"', + ); + expect(applySection).toContain( + "Immediate user-manager start failed with 219/CGROUP; use later-login recovery.", + ); + expect(applyWarning).toBeGreaterThanOrEqual(0); + expect(applyStop).toBeGreaterThan(applyWarning); + expect(rebootWarning).toBeGreaterThanOrEqual(0); + expect(reboot).toBeGreaterThan(rebootWarning); + expect(rollbackWarning).toBeGreaterThanOrEqual(0); + expect(rollbackStop).toBeGreaterThan(rollbackWarning); + expect(cleanupRouting).toBeGreaterThanOrEqual(0); + expect(incompleteReceiptRoute).toBeGreaterThan(cleanupRouting); + expect(unrecordedCleanupRoute).toBeGreaterThan(incompleteReceiptRoute); + expect(recordedCleanupRoute).toBeGreaterThan(unrecordedCleanupRoute); + expect(unrecordedCleanupHeading).toBeGreaterThan(recordedCleanupRoute); + expect(generalCleanupHeading).toBeGreaterThan(unrecordedCleanupHeading); + expect(applySection).not.toContain( + "The initial `0` records make cleanup executable even when only the first file or directory was created.", + ); + expect(applySection.slice(generalCleanupHeading)).toContain( + "Enter this procedure only when every final `*_created` value is `0` or `1`.", + ); + expect(applySection).toContain( + "a later login can create it under the corrected cgroup hierarchy", + ); + expect(rollbackSection).toContain( + "sign in again so systemd creates the user manager under the restored hierarchy", + ); + }); + + it("executes apply-side 219/CGROUP diagnosis after the inactive reload (#9188)", () => { + const fixture = makeRollbackFixture(); + const result = runDocumentedApply(fixture); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("status=219/CGROUP"); + expect(result.stderr).toContain("use later-login recovery"); + expect(fs.readFileSync(fixture.systemctlCallMarker, "utf8")).toBe( + "systemctl stop user@1000.service\n" + + "systemctl daemon-reload\n" + + "systemctl start user@1000.service\n" + + "systemctl status user@1000.service --no-pager\n", + ); + }); + + it("fails closed on rollback inspection before removing recorded objects (#9188)", () => { + const fixture = makeRollbackFixture(); + const inspectionFailure = runDocumentedRollback(fixture, { + FAIL_PREDICATE_PATH: fixture.delegationDropIn, + }); + expect(inspectionFailure.status).not.toBe(0); + expect(inspectionFailure.stderr).toContain("simulated predicate inspection failure"); + expect(fs.existsSync(fixture.delegationDropIn)).toBe(true); + expect(fs.existsSync(fixture.appSliceDropIn)).toBe(true); + expect(fs.existsSync(fixture.userSliceDropIn)).toBe(true); + expect(fs.existsSync(fixture.delegationDropInDirectory)).toBe(true); + expect(fs.existsSync(fixture.appSliceDropInDirectory)).toBe(true); + expect(fs.existsSync(fixture.systemctlCallMarker)).toBe(false); + const result = runDocumentedRollback(fixture); + expect(result.status).toBe(0); + expect(result.stderr).toBe(""); + expect(fs.existsSync(fixture.delegationDropIn)).toBe(false); + expect(fs.existsSync(fixture.appSliceDropIn)).toBe(false); + expect(fs.existsSync(fixture.userSliceDropIn)).toBe(false); + expect(fs.existsSync(fixture.delegationDropInDirectory)).toBe(false); + expect(fs.existsSync(fixture.appSliceDropInDirectory)).toBe(false); + expect(fs.existsSync(fixture.userSliceDropInDirectory)).toBe(false); + expect(fs.readFileSync(fixture.systemctlCallMarker, "utf8")).toBe( + "systemctl stop user@1000.service\n" + + "systemctl daemon-reload\n" + + "systemctl start user@1000.service\n", + ); + }); + + it("accepts recorded rollback resources that are already absent (#9188)", () => { + const fixture = makeRollbackFixture(); + fs.rmSync(fixture.delegationDropInDirectory, { recursive: true }); + fs.rmSync(fixture.appSliceDropInDirectory, { recursive: true }); + fs.rmSync(fixture.userSliceDropInDirectory, { recursive: true }); + const result = runDocumentedRollback(fixture); + expect(result.status).toBe(0); + expect(result.stderr).toBe(""); + expect(fs.readFileSync(fixture.systemctlCallMarker, "utf8")).toContain( + "systemctl start user@1000.service\n", + ); + }); + + it("removes a partial publication and accepts the same record on retry (#9188)", () => { + const fixture = makeRollbackFixture({ appSliceDropInCreated: false }); + const firstResult = runDocumentedRollback(fixture); + const retryResult = runDocumentedRollback(fixture); + expect(firstResult.status).toBe(0); + expect(firstResult.stderr).toBe(""); + expect(retryResult.status).toBe(0); + expect(retryResult.stderr).toBe(""); + expect(fs.existsSync(fixture.delegationDropIn)).toBe(false); + expect(fs.existsSync(fixture.appSliceDropIn)).toBe(false); + expect(fs.existsSync(fixture.userSliceDropIn)).toBe(false); + expect(fs.existsSync(fixture.delegationDropInDirectory)).toBe(false); + expect(fs.existsSync(fixture.appSliceDropInDirectory)).toBe(false); + expect(fs.existsSync(fixture.userSliceDropInDirectory)).toBe(false); + expect(fs.readFileSync(fixture.systemctlCallMarker, "utf8")).toBe( + ( + "systemctl stop user@1000.service\n" + + "systemctl daemon-reload\n" + + "systemctl start user@1000.service\n" + ).repeat(2), + ); + }); + + it("preserves a drop-in whose identity changed after creation (#9188)", () => { + const fixture = makeRollbackFixture({ expectedDelegationDropInId: "0:0" }); + const result = runDocumentedRollback(fixture); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("Refusing CPU controller drop-in whose identity changed"); + expect(fs.readFileSync(fixture.delegationDropIn, "utf8")).toBe( + "[Service]\nDelegate=cpu memory pids\n", + ); + expect(fs.existsSync(fixture.delegationDropInDirectory)).toBe(true); + expect(fs.readFileSync(fixture.appSliceDropIn, "utf8")).toBe("[Slice]\nCPUWeight=100\n"); + expect(fs.existsSync(fixture.appSliceDropInDirectory)).toBe(true); + expect(fs.existsSync(fixture.systemctlCallMarker)).toBe(false); + }); + + it("reports immediate 219/CGROUP and accepts rollback after later-login recovery (#9188)", () => { + const fixture = makeRollbackFixture(); + const failedStart = runDocumentedRollback(fixture, { START_FAILURE_219: "1" }); + const recoveredRetry = runDocumentedRollback(fixture); + expect(failedStart.status).not.toBe(0); + expect(failedStart.stderr).toContain("status=219/CGROUP"); + expect(failedStart.stderr).toContain("use later-login recovery"); + expect(fs.existsSync(fixture.delegationDropIn)).toBe(false); + expect(fs.existsSync(fixture.appSliceDropIn)).toBe(false); + expect(recoveredRetry.status).toBe(0); + expect(recoveredRetry.stderr).toBe(""); + expect(fs.readFileSync(fixture.systemctlCallMarker, "utf8")).toBe( + "systemctl stop user@1000.service\n" + + "systemctl daemon-reload\n" + + "systemctl start user@1000.service\n" + + "systemctl status user@1000.service --no-pager\n" + + "systemctl stop user@1000.service\n" + + "systemctl daemon-reload\n" + + "systemctl start user@1000.service\n", + ); + }); + + it("creates all three drop-ins with their required content and mode (#9188)", () => { + const fixture = makeCommandFixture(); + const result = runDocumentedCommand(fixture); + expect(result.status).toBe(0); + expect(result.stderr).toBe(""); + expect(fs.readFileSync(fixture.delegationDropIn, "utf8")).toBe( + "[Service]\nDelegate=cpu memory pids\n", + ); + expect(fs.readFileSync(fixture.appSliceDropIn, "utf8")).toBe("[Slice]\nCPUWeight=100\n"); + expect(fs.readFileSync(fixture.userSliceDropIn, "utf8")).toBe("[Slice]\nCPUWeight=100\n"); + expect(fs.statSync(fixture.delegationDropIn).mode & 0o777).toBe(0o644); + expect(fs.statSync(fixture.appSliceDropIn).mode & 0o777).toBe(0o644); + expect(fs.statSync(fixture.userSliceDropIn).mode & 0o777).toBe(0o644); + expect(listTemporaryDropIns(fixture)).toEqual([]); + }); + + it("does not replace a drop-in created before the publish link (#9188)", () => { + const fixture = makeCommandFixture(); + const result = runDocumentedCommand(fixture, { SUDO_SCENARIO: "concurrent" }); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("File exists"); + expect(result.stderr).toContain( + `CPU controller drop-in creation failed: ${fixture.delegationDropIn}`, + ); + expect(result.stderr).not.toContain("Refusing to replace existing file"); + expect(fs.readFileSync(fixture.delegationDropIn, "utf8")).toBe("concurrent content\n"); + expect(fs.existsSync(fixture.appSliceDropIn)).toBe(false); + expect(listTemporaryDropIns(fixture)).toEqual([]); + }); + + it.each([ + { failedMkdirCall: 1, recordsFirstDirectory: false }, + { failedMkdirCall: 2, recordsFirstDirectory: true }, + ])( + "does not create a drop-in when mkdir call $failedMkdirCall fails (#9188)", + ({ failedMkdirCall, recordsFirstDirectory }) => { + const fixture = makeCommandFixture(); + const result = runDocumentedCommand(fixture, { + FAIL_MKDIR_CALL: String(failedMkdirCall), + }); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("simulated directory creation failure"); + expect(fs.readFileSync(fixture.mkdirCallMarker, "utf8")).toBe(`${failedMkdirCall}\n`); + expect(fs.existsSync(fixture.delegationDropIn)).toBe(false); + expect(fs.existsSync(fixture.appSliceDropIn)).toBe(false); + expect(listTemporaryDropIns(fixture)).toEqual([]); + expect(result.stdout.includes("delegation_drop_in_dir_created=1")).toBe( + recordsFirstDirectory, + ); + expect(result.stdout.includes("delegation_drop_in_dir_id=1:1")).toBe(recordsFirstDirectory); + expect(result.stdout).toContain("delegation_drop_in_created=0"); + expect(result.stdout).toContain("app_slice_drop_in_created=0"); + }, + ); + + it("recovers an exact empty directory when identity recording fails after mkdir (#9188)", () => { + const fixture = makeCommandFixture(); + const delegationDirectory = path.dirname(fixture.delegationDropIn); + const creation = runDocumentedCommand(fixture, { + FAIL_STAT_ID_PATH: delegationDirectory, + }); + expect(creation.stderr).toContain("simulated identity recording failure"); + expect(creation.stderr).toContain( + `CPU controller drop-in directory identity recording failed: ${delegationDirectory}`, + ); + expect(finalRecord(creation.stdout, "delegation_drop_in_dir_created")).toBe("unrecorded"); + expect(fs.existsSync(fixture.delegationDropIn)).toBe(false); + const rejectedGeneralCleanup = runPartialCreationRollback(fixture, creation.stdout); + expect(rejectedGeneralCleanup.status).not.toBe(0); + expect(fs.existsSync(delegationDirectory)).toBe(true); + const inspectionFailure = runUnrecordedDirectoryRecovery(fixture, delegationDirectory, { + FAIL_PREDICATE_PATH: delegationDirectory, + }); + expect(inspectionFailure.status).not.toBe(0); + expect(inspectionFailure.stderr).toContain("simulated predicate inspection failure"); + expect(fs.existsSync(delegationDirectory)).toBe(true); + const concurrentFile = path.join(delegationDirectory, "concurrent.conf"); + fs.writeFileSync(concurrentFile, "preserve\n"); + const refusedRecovery = runUnrecordedDirectoryRecovery(fixture, delegationDirectory); + expect(refusedRecovery.status).not.toBe(0); + expect(refusedRecovery.stderr).toContain("Refusing nonempty unrecorded drop-in directory"); + expect(fs.readFileSync(concurrentFile, "utf8")).toBe("preserve\n"); + fs.rmSync(concurrentFile); + const recovery = runUnrecordedDirectoryRecovery(fixture, delegationDirectory); + expect(recovery.status).toBe(0); + expect(fs.existsSync(delegationDirectory)).toBe(false); + const stagingFixture = makeCommandFixture(); + const stagingCreation = runDocumentedCommand(stagingFixture, { FAIL_STAGING_STAT: "1" }); + const stagingDirectory = + finalRecord(stagingCreation.stdout, "delegation_staging_dir_path") ?? ""; + expect(finalRecord(stagingCreation.stdout, "delegation_staging_dir_created")).toBe( + "unrecorded", + ); + const rejectedStagingCleanup = runPartialCreationRollback( + stagingFixture, + stagingCreation.stdout, + ); + expect(rejectedStagingCleanup.status).not.toBe(0); + expect(fs.existsSync(stagingDirectory)).toBe(true); + const stagingRecovery = runUnrecordedDirectoryRecovery( + stagingFixture, + stagingDirectory, + {}, + stagingCreation.stdout, + ); + expect(stagingRecovery.status).toBe(0); + expect(fs.existsSync(stagingDirectory)).toBe(false); + }); + + it("prints each creation identity before a later drop-in publish fails (#9188)", () => { + const fixture = makeCommandFixture(); + const result = runDocumentedCommand(fixture, { FAIL_LINK_CALL: "2" }); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("simulated publish link failure"); + expect(result.stdout).toMatch( + /delegation_drop_in_dir_id=1:1\nRecord for rollback: delegation_drop_in_dir_created=1/u, + ); + expect(result.stdout).toMatch( + /app_slice_drop_in_dir_id=1:1\nRecord for rollback: app_slice_drop_in_dir_created=1/u, + ); + expect(result.stdout).toMatch( + /delegation_drop_in_id=1:1\nRecord for rollback: delegation_drop_in_created=1/u, + ); + expect(finalRecord(result.stdout, "app_slice_drop_in_created")).toBe("0"); + expect(finalRecord(result.stdout, "app_slice_drop_in_id")).toBe("1:1"); + expect(fs.existsSync(fixture.delegationDropIn)).toBe(true); + expect(fs.existsSync(fixture.appSliceDropIn)).toBe(false); + expect(listTemporaryDropIns(fixture)).toEqual([]); + }); + + it("repairs an incomplete receipt and fails closed on partial-cleanup inspection (#9188)", () => { + const fixture = makeCommandFixture(); + const creation = runDocumentedCommand(fixture, { FAIL_AFTER_LINK_CALL: "1" }); + const interruptedReceipt = creation.stdout; + const incompleteCleanup = runPartialCreationRollback(fixture, interruptedReceipt); + const completedReceipt = `${interruptedReceipt}Record for rollback: delegation_drop_in_created=1\n`; + const inspectionFailure = runPartialCreationRollback(fixture, completedReceipt, { + FAIL_PREDICATE_PATH: fixture.delegationDropIn, + }); + + expect(creation.stderr).toContain("simulated interruption after publish link"); + expect(finalRecord(interruptedReceipt, "delegation_drop_in_id")).toBe("1:1"); + expect(finalRecord(interruptedReceipt, "delegation_drop_in_created")).toBe("0"); + expect(incompleteCleanup.status).not.toBe(0); + expect(incompleteCleanup.stderr).toContain("Unexpected identity for unrecorded"); + expect(inspectionFailure.status).not.toBe(0); + expect(inspectionFailure.stderr).toContain("simulated predicate inspection failure"); + expect(fs.existsSync(fixture.delegationDropIn)).toBe(true); + expect(fs.existsSync(path.dirname(fixture.delegationDropIn))).toBe(true); + expect(fs.existsSync(path.dirname(fixture.appSliceDropIn))).toBe(true); + const firstCleanup = runPartialCreationRollback(fixture, completedReceipt); + const retry = runPartialCreationRollback(fixture, completedReceipt); + expect(firstCleanup.stderr).toBe(""); + expect(firstCleanup.status).toBe(0); + expect(retry.status).toBe(0); + expect(retry.stderr).toBe(""); + expect(fs.existsSync(fixture.delegationDropIn)).toBe(false); + expect(fs.existsSync(fixture.appSliceDropIn)).toBe(false); + expect(fs.existsSync(path.dirname(fixture.delegationDropIn))).toBe(false); + expect(fs.existsSync(path.dirname(fixture.appSliceDropIn))).toBe(false); + }); + + it("partial cleanup preserves a published file whose identity changed (#9188)", () => { + const fixture = makeCommandFixture(); + const creation = runDocumentedCommand(fixture, { FAIL_LINK_CALL: "2" }); + const retainedOriginal = path.join( + path.dirname(path.dirname(fixture.delegationDropIn)), + "delegation-drop-in.original", + ); + fs.renameSync(fixture.delegationDropIn, retainedOriginal); + fs.writeFileSync(fixture.delegationDropIn, "replacement\n"); + + const cleanup = runPartialCreationRollback(fixture, creation.stdout, { + STAT_ID_OVERRIDE: "0:0", + STAT_ID_OVERRIDE_PATH: fixture.delegationDropIn, + }); + + expect(creation.status).not.toBe(0); + expect(cleanup.status).not.toBe(0); + expect(cleanup.stderr).toContain("Refusing CPU controller drop-in whose identity changed"); + expect(fs.existsSync(retainedOriginal)).toBe(true); + expect(fs.readFileSync(fixture.delegationDropIn, "utf8")).toBe("replacement\n"); + expect(fs.existsSync(path.dirname(fixture.delegationDropIn))).toBe(true); + expect(fs.existsSync(path.dirname(fixture.appSliceDropIn))).toBe(true); + }); + + it("partial cleanup preserves valid pre-existing drop-in directories (#9188)", () => { + const fixture = makeCommandFixture(); + const delegationDirectory = path.dirname(fixture.delegationDropIn); + const appSliceDirectory = path.dirname(fixture.appSliceDropIn); + fs.mkdirSync(delegationDirectory, { mode: 0o755 }); + fs.mkdirSync(appSliceDirectory, { mode: 0o755 }); + const creation = runDocumentedCommand(fixture, { FAIL_LINK_CALL: "2" }); + const completedReceipt = `${creation.stdout}Record for rollback: app_slice_drop_in_created=1\n`; + + const cleanup = runPartialCreationRollback(fixture, completedReceipt); + + expect(creation.status).not.toBe(0); + expect(cleanup.status).toBe(0); + expect(cleanup.stderr).toBe(""); + expect(fs.existsSync(fixture.delegationDropIn)).toBe(false); + expect(fs.existsSync(delegationDirectory)).toBe(true); + expect(fs.existsSync(appSliceDirectory)).toBe(true); + }); + + it("refuses and preserves pre-existing directory metadata (#9188)", () => { + const fixture = makeCommandFixture(); + const delegationDirectory = path.dirname(fixture.delegationDropIn); + const appSliceDirectory = path.dirname(fixture.appSliceDropIn); + fs.mkdirSync(delegationDirectory, { mode: 0o750 }); + fs.mkdirSync(appSliceDirectory, { mode: 0o750 }); + + const result = runDocumentedCommand(fixture, { + SUDO_SCENARIO: "existing-directory-metadata", + }); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("Refusing to change existing drop-in directory owner or mode"); + expect(fs.statSync(delegationDirectory).mode & 0o777).toBe(0o750); + expect(fs.statSync(appSliceDirectory).mode & 0o777).toBe(0o750); + expect(fs.existsSync(fixture.delegationDropIn)).toBe(false); + expect(fs.existsSync(fixture.appSliceDropIn)).toBe(false); + }); + + it("records valid pre-existing directories as preserved (#9188)", () => { + const fixture = makeCommandFixture(); + const delegationDirectory = path.dirname(fixture.delegationDropIn); + const appSliceDirectory = path.dirname(fixture.appSliceDropIn); + fs.mkdirSync(delegationDirectory, { mode: 0o755 }); + fs.mkdirSync(appSliceDirectory, { mode: 0o755 }); + + const result = runDocumentedCommand(fixture); + + expect(result.status).toBe(0); + expect(result.stdout).toContain("delegation_drop_in_dir_created=0"); + expect(result.stdout).toContain("app_slice_drop_in_dir_created=0"); + expect(result.stdout).not.toContain("delegation_drop_in_dir_id="); + expect(result.stdout).not.toContain("app_slice_drop_in_dir_id="); + expect(fs.statSync(delegationDirectory).mode & 0o777).toBe(0o755); + expect(fs.statSync(appSliceDirectory).mode & 0o777).toBe(0o755); + }); + + it("removes the temporary file after its write fails (#9188)", () => { + const fixture = makeCommandFixture(); + const result = runDocumentedCommand(fixture, { SUDO_SCENARIO: "write-failure" }); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("simulated temporary file write failure"); + expect(result.stderr).toContain( + `CPU controller drop-in creation failed: ${fixture.delegationDropIn}`, + ); + expect(result.stderr).not.toContain("Refusing to replace existing file"); + expect(fs.existsSync(fixture.delegationDropIn)).toBe(false); + expect(fs.existsSync(fixture.appSliceDropIn)).toBe(false); + expect(listTemporaryDropIns(fixture)).toEqual([]); + }); +});