diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 43971788b2d..7cbde1baa03 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -651,6 +651,182 @@ jobs: NEMOCLAW_RUN_LIVE_E2E: "1" OPENSHELL_DOCKER_SUPERVISOR_IMAGE: ghcr.io/nvidia/openshell/supervisor@sha256:f4226253a3525c3832adac5b38b419a0f27d1e915effe565b5885e20f93cd5e9 steps: + - id: trusted_hermes_swap + name: Provision trusted Hermes E2E swap + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' && matrix.agent == 'hermes' }} + shell: /bin/bash --noprofile --norc -e -o pipefail {0} + env: + BASH_ENV: /dev/null + CHECKOUT_SHA: ${{ inputs.checkout_sha }} + DISPATCH_SHA: ${{ github.sha }} + ENV: /dev/null + EVENT_NAME: ${{ github.event_name }} + EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }} + LC_ALL: C + REF: ${{ github.ref }} + REPOSITORY: ${{ github.repository }} + RUNNER_ARCH_KIND: ${{ runner.arch }} + RUNNER_ENVIRONMENT_KIND: ${{ runner.environment }} + RUNNER_OS_KIND: ${{ runner.os }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + run: &trusted-hermes-e2e-swap | + set -euo pipefail + readonly swap_dir="/mnt/nemoclaw-hermes-e2e-swap" + readonly required_swap_bytes=34359738368 + readonly swap_file_bytes=34359742464 + readonly reserve_bytes=17179869184 + readonly activation_observation_attempts=5 + readonly activation_observation_delay_seconds=1 + swap_file="" + swap_activation_succeeded=0 + + fail() { + printf 'Trusted Hermes E2E swap setup failed: %s\n' "$1" >&2 + exit 1 + } + + if [[ "${REPOSITORY}" != "NVIDIA/NemoClaw" || "${EVENT_NAME}" != "workflow_dispatch" || "${REF}" != "refs/heads/main" ]]; then + fail "workflow must run from NVIDIA/NemoClaw main" + fi + if [[ ! "${CHECKOUT_SHA}" =~ ^[0-9a-f]{40}$ ]]; then + fail "checkout SHA must be lowercase 40-hex" + fi + if [[ ! "${EXPECTED_WORKFLOW_SHA}" =~ ^[0-9a-f]{40}$ || "${WORKFLOW_SHA}" != "${EXPECTED_WORKFLOW_SHA}" || "${WORKFLOW_SHA}" != "${DISPATCH_SHA}" ]]; then + fail "workflow source must match the trusted dispatch revision" + fi + if [[ "${RUNNER_ENVIRONMENT_KIND}" != "github-hosted" || "${RUNNER_OS_KIND}" != "Linux" || "${RUNNER_ARCH_KIND}" != "X64" ]]; then + fail "swap fallback requires an ephemeral GitHub-hosted Linux x64 runner" + fi + mnt_metadata="$(/usr/bin/stat -c "%F:%u:%g" -- /mnt)" + if [[ "${mnt_metadata}" != "directory:0:0" ]]; then + fail "/mnt must be a root-owned directory" + fi + + read_active_swap_bytes() { + /usr/bin/sudo -n /usr/sbin/swapon --show=SIZE --bytes --noheadings | + /usr/bin/awk '{ total += $1 } END { printf "%.0f", total }' + } + + active_swap_bytes="$(read_active_swap_bytes)" + active_swap_bytes="${active_swap_bytes:-0}" + if [[ ! "${active_swap_bytes}" =~ ^[0-9]+$ ]]; then + fail "unable to determine active swap capacity" + fi + if (( active_swap_bytes >= required_swap_bytes )); then + printf 'Hermes E2E swap is already sufficient: %s bytes active\n' "${active_swap_bytes}" + exit 0 + fi + + available_bytes="$(/usr/bin/df --block-size=1 --output=avail /mnt | /usr/bin/tail -n 1 | /usr/bin/tr -d "[:space:]")" + if [[ ! "${available_bytes}" =~ ^[0-9]+$ ]]; then + fail "unable to determine available disk capacity under /mnt" + fi + required_disk_bytes=$((swap_file_bytes + reserve_bytes)) + if (( available_bytes < required_disk_bytes )); then + fail "insufficient disk capacity: ${available_bytes} bytes available, ${required_disk_bytes} required" + fi + + if /usr/bin/sudo -n /usr/bin/test -e "${swap_dir}" || /usr/bin/sudo -n /usr/bin/test -L "${swap_dir}"; then + fail "refusing unexpected pre-existing swap path" + fi + + directory_created=0 + cleanup_partial_swap() { + status="$?" + if (( status != 0 && directory_created == 1 )); then + if active_swap_names="$(/usr/bin/sudo -n /usr/sbin/swapon --show=NAME --noheadings --raw 2>/dev/null)"; then + fixed_swap_active=0 + while IFS= read -r active_swap_name; do + if [[ -n "${swap_file}" && "${active_swap_name}" == "${swap_file}" ]]; then + fixed_swap_active=1 + break + fi + done <<< "${active_swap_names}" + if (( fixed_swap_active == 1 || swap_activation_succeeded == 1 )); then + if /usr/bin/sudo -n /usr/sbin/swapoff "${swap_file}" 2>/dev/null; then + /usr/bin/sudo -n /usr/bin/rm -f -- "${swap_file}" || true + /usr/bin/sudo -n /usr/bin/rmdir -- "${swap_dir}" || true + else + printf 'Preserving active Hermes E2E swap after setup failure: %s\n' "${swap_file}" >&2 + fi + else + if [[ -n "${swap_file}" ]]; then + /usr/bin/sudo -n /usr/bin/rm -f -- "${swap_file}" || true + fi + /usr/bin/sudo -n /usr/bin/rmdir -- "${swap_dir}" || true + fi + else + printf 'Preserving Hermes E2E swap because active swap could not be queried: %s\n' "${swap_file}" >&2 + fi + fi + trap - EXIT + exit "${status}" + } + trap cleanup_partial_swap EXIT + + /usr/bin/sudo -n /usr/bin/mkdir -m 0700 -- "${swap_dir}" + directory_created=1 + directory_metadata="$(/usr/bin/sudo -n /usr/bin/stat -c "%F:%u:%g:%a" -- "${swap_dir}")" + if [[ "${directory_metadata}" != "directory:0:0:700" ]]; then + fail "swap directory must be a root-owned mode-0700 directory" + fi + swap_file="$(/usr/bin/sudo -n /usr/bin/mktemp --tmpdir="${swap_dir}" nemoclaw-hermes.XXXXXXXX.swap)" + if ! /usr/bin/sudo -n /usr/bin/test -f "${swap_file}" || /usr/bin/sudo -n /usr/bin/test -L "${swap_file}"; then + fail "swap file must be a regular non-symlink" + fi + file_metadata="$(/usr/bin/sudo -n /usr/bin/stat -c "%u:%g:%a" -- "${swap_file}")" + if [[ "${file_metadata}" != "0:0:600" ]]; then + fail "swap file must be root-owned mode 0600" + fi + /usr/bin/sudo -n /usr/bin/fallocate -l "${swap_file_bytes}" "${swap_file}" + file_size_bytes="$(/usr/bin/sudo -n /usr/bin/stat -c "%s" -- "${swap_file}")" + if [[ ! "${file_size_bytes}" =~ ^[0-9]+$ || "${file_size_bytes}" -ne "${swap_file_bytes}" ]]; then + fail "swap file size does not match the fixed backing allocation" + fi + remaining_bytes="$(/usr/bin/df --block-size=1 --output=avail /mnt | /usr/bin/tail -n 1 | /usr/bin/tr -d "[:space:]")" + if [[ ! "${remaining_bytes}" =~ ^[0-9]+$ || "${remaining_bytes}" -lt "${reserve_bytes}" ]]; then + fail "swap allocation did not preserve the required disk reserve" + fi + /usr/bin/sudo -n /usr/sbin/mkswap --quiet "${swap_file}" + /usr/bin/sudo -n /usr/sbin/swapon "${swap_file}" + swap_activation_succeeded=1 + + observe_provisioned_swap() { + activation_observation_attempt=1 + while (( activation_observation_attempt <= activation_observation_attempts )); do + provisioned_swap_active=0 + if active_swap_names="$(/usr/bin/sudo -n /usr/sbin/swapon --show=NAME --noheadings --raw 2>/dev/null)"; then + while IFS= read -r active_swap_name; do + if [[ "${active_swap_name}" == "${swap_file}" ]]; then + provisioned_swap_active=1 + break + fi + done <<< "${active_swap_names}" + fi + if observed_swap_bytes="$(read_active_swap_bytes 2>/dev/null)"; then + observed_swap_bytes="${observed_swap_bytes:-0}" + if [[ "${observed_swap_bytes}" =~ ^[0-9]+$ ]] && + (( provisioned_swap_active == 1 && observed_swap_bytes >= required_swap_bytes )); then + active_swap_bytes="${observed_swap_bytes}" + return 0 + fi + fi + if (( activation_observation_attempt < activation_observation_attempts )); then + /usr/bin/sleep "${activation_observation_delay_seconds}" + fi + activation_observation_attempt=$((activation_observation_attempt + 1)) + done + return 1 + } + + if ! observe_provisioned_swap; then + fail "unable to verify the required active swap capacity after bounded observation" + fi + + trap - EXIT + printf 'Hermes E2E swap ready: %s bytes active\n' "${active_swap_bytes}" + /usr/bin/sudo -n /usr/sbin/swapon --show + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: ref: ${{ inputs.checkout_sha || github.sha }} @@ -1154,6 +1330,26 @@ jobs: NEMOCLAW_COMPAT_MODEL: nvidia/nvidia/nemotron-3-ultra NEMOCLAW_PREFERRED_API: openai-completions steps: + - id: trusted_hermes_swap + name: Provision trusted Hermes E2E swap + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' }} + shell: /bin/bash --noprofile --norc -e -o pipefail {0} + env: + BASH_ENV: /dev/null + CHECKOUT_SHA: ${{ inputs.checkout_sha }} + DISPATCH_SHA: ${{ github.sha }} + ENV: /dev/null + EVENT_NAME: ${{ github.event_name }} + EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }} + LC_ALL: C + REF: ${{ github.ref }} + REPOSITORY: ${{ github.repository }} + RUNNER_ARCH_KIND: ${{ runner.arch }} + RUNNER_ENVIRONMENT_KIND: ${{ runner.environment }} + RUNNER_OS_KIND: ${{ runner.os }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + run: *trusted-hermes-e2e-swap + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: ref: ${{ inputs.checkout_sha || github.sha }} @@ -1276,6 +1472,26 @@ jobs: NEMOCLAW_SWITCH_MOCK_ANTHROPIC: ${{ matrix.switch_mock_anthropic }} OPENSHELL_GATEWAY: "nemoclaw" steps: + - id: trusted_hermes_swap + name: Provision trusted Hermes E2E swap + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' }} + shell: /bin/bash --noprofile --norc -e -o pipefail {0} + env: + BASH_ENV: /dev/null + CHECKOUT_SHA: ${{ inputs.checkout_sha }} + DISPATCH_SHA: ${{ github.sha }} + ENV: /dev/null + EVENT_NAME: ${{ github.event_name }} + EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }} + LC_ALL: C + REF: ${{ github.ref }} + REPOSITORY: ${{ github.repository }} + RUNNER_ARCH_KIND: ${{ runner.arch }} + RUNNER_ENVIRONMENT_KIND: ${{ runner.environment }} + RUNNER_OS_KIND: ${{ runner.os }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + run: *trusted-hermes-e2e-swap + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: ref: ${{ inputs.checkout_sha || github.sha }} @@ -1763,6 +1979,26 @@ jobs: NEMOCLAW_SANDBOX_NAME: e2e-hermes NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS: "60" steps: + - id: trusted_hermes_swap + name: Provision trusted Hermes E2E swap + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' && (contains(format(',{0},', inputs.jobs), ',hermes-e2e,') || contains(format(',{0},', inputs.targets), ',hermes-e2e,')) }} + shell: /bin/bash --noprofile --norc -e -o pipefail {0} + env: + BASH_ENV: /dev/null + CHECKOUT_SHA: ${{ inputs.checkout_sha }} + DISPATCH_SHA: ${{ github.sha }} + ENV: /dev/null + EVENT_NAME: ${{ github.event_name }} + EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }} + LC_ALL: C + REF: ${{ github.ref }} + REPOSITORY: ${{ github.repository }} + RUNNER_ARCH_KIND: ${{ runner.arch }} + RUNNER_ENVIRONMENT_KIND: ${{ runner.environment }} + RUNNER_OS_KIND: ${{ runner.os }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + run: *trusted-hermes-e2e-swap + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: ref: ${{ inputs.checkout_sha || github.sha }} @@ -2083,6 +2319,26 @@ jobs: NEMOCLAW_E2E_HERMES_DASHBOARD: "1" NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS: "60" steps: + - id: trusted_hermes_swap + name: Provision trusted Hermes E2E swap + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' }} + shell: /bin/bash --noprofile --norc -e -o pipefail {0} + env: + BASH_ENV: /dev/null + CHECKOUT_SHA: ${{ inputs.checkout_sha }} + DISPATCH_SHA: ${{ github.sha }} + ENV: /dev/null + EVENT_NAME: ${{ github.event_name }} + EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }} + LC_ALL: C + REF: ${{ github.ref }} + REPOSITORY: ${{ github.repository }} + RUNNER_ARCH_KIND: ${{ runner.arch }} + RUNNER_ENVIRONMENT_KIND: ${{ runner.environment }} + RUNNER_OS_KIND: ${{ runner.os }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + run: *trusted-hermes-e2e-swap + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: ref: ${{ inputs.checkout_sha || github.sha }} @@ -2429,6 +2685,26 @@ jobs: NEMOCLAW_SANDBOX_NAME: e2e-hermes-shields OPENSHELL_GATEWAY: nemoclaw steps: + - id: trusted_hermes_swap + name: Provision trusted Hermes E2E swap + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' }} + shell: /bin/bash --noprofile --norc -e -o pipefail {0} + env: + BASH_ENV: /dev/null + CHECKOUT_SHA: ${{ inputs.checkout_sha }} + DISPATCH_SHA: ${{ github.sha }} + ENV: /dev/null + EVENT_NAME: ${{ github.event_name }} + EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }} + LC_ALL: C + REF: ${{ github.ref }} + REPOSITORY: ${{ github.repository }} + RUNNER_ARCH_KIND: ${{ runner.arch }} + RUNNER_ENVIRONMENT_KIND: ${{ runner.environment }} + RUNNER_OS_KIND: ${{ runner.os }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + run: *trusted-hermes-e2e-swap + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: ref: ${{ inputs.checkout_sha || github.sha }} @@ -3377,6 +3653,26 @@ jobs: NEMOCLAW_SANDBOX_NAME: ${{ matrix.sandbox_name }} OPENSHELL_GATEWAY: nemoclaw steps: + - id: trusted_hermes_swap + name: Provision trusted Hermes E2E swap + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' && matrix.agent == 'hermes' }} + shell: /bin/bash --noprofile --norc -e -o pipefail {0} + env: + BASH_ENV: /dev/null + CHECKOUT_SHA: ${{ inputs.checkout_sha }} + DISPATCH_SHA: ${{ github.sha }} + ENV: /dev/null + EVENT_NAME: ${{ github.event_name }} + EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }} + LC_ALL: C + REF: ${{ github.ref }} + REPOSITORY: ${{ github.repository }} + RUNNER_ARCH_KIND: ${{ runner.arch }} + RUNNER_ENVIRONMENT_KIND: ${{ runner.environment }} + RUNNER_OS_KIND: ${{ runner.os }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + run: *trusted-hermes-e2e-swap + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: ref: ${{ inputs.checkout_sha || github.sha }} @@ -4573,6 +4869,26 @@ jobs: NEMOCLAW_SANDBOX_NAME: e2e-bedrock-${{ matrix.agent }} OPENSHELL_GATEWAY: "nemoclaw" steps: + - id: trusted_hermes_swap + name: Provision trusted Hermes E2E swap + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' && matrix.agent == 'hermes' }} + shell: /bin/bash --noprofile --norc -e -o pipefail {0} + env: + BASH_ENV: /dev/null + CHECKOUT_SHA: ${{ inputs.checkout_sha }} + DISPATCH_SHA: ${{ github.sha }} + ENV: /dev/null + EVENT_NAME: ${{ github.event_name }} + EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }} + LC_ALL: C + REF: ${{ github.ref }} + REPOSITORY: ${{ github.repository }} + RUNNER_ARCH_KIND: ${{ runner.arch }} + RUNNER_ENVIRONMENT_KIND: ${{ runner.environment }} + RUNNER_OS_KIND: ${{ runner.os }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + run: *trusted-hermes-e2e-swap + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: ref: ${{ inputs.checkout_sha || github.sha }} diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index f58d2aab64d..92acafee315 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -271,6 +271,11 @@ "test": "routes only the measured heavy lanes on trusted main (#7145)", "category": "security" }, + { + "file": "test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts", + "test": "keeps the fixed privileged program before candidate checkout in every protected job (#7145)", + "category": "security" + }, { "file": "test/fetch-guard-patch-regression.test.ts", "test": "requires classifier review and integrity evidence when the OpenClaw build pin changes", diff --git a/test/e2e/README.md b/test/e2e/README.md index 9aba76f25df..ccc5db72f3e 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -63,16 +63,45 @@ standard runners even though they use the trusted workflow definition from `main`. Exact-head PR-gate dispatches use a bounded swap fallback for the hosted -Hermes image-building lanes that remain on those standard runners. The live -Vitest helper activates the fallback only when GitHub Actions supplies a -validated lowercase 40-hex checkout SHA. It reuses at least 32 GiB of active -swap when available; otherwise, it creates one fixed 32 GiB swap file under -`/mnt` before agent-turn latency, Hermes inference switch and shields, the -Hermes Bedrock and stable MCP shards, or the `hermes-e2e`, `hermes-dashboard`, -and Hermes security-posture tests. Setup failure stops before Vitest. Scheduled -and ordinary manual `main` runs, larger-runner executions, rebuild lanes with +Hermes image-building lanes that remain on those standard runners. The trusted +workflow provisions the fallback as the first job step, before checking out or +executing the candidate revision. It requires a controller-supplied lowercase +40-hex checkout SHA, matching trusted workflow and dispatch revisions, and an +ephemeral GitHub-hosted Linux x64 runner. + +The trusted step requires at least 32 GiB (34,359,738,368 bytes) of usable swap. +It reuses active swap that meets this requirement. +Otherwise, it preserves at least 16 GiB of available disk capacity under +`/mnt`, creates a root-owned mode-`0700` directory, and creates an exclusive +randomized mode-`0600` file. +The file allocation is 32 GiB plus 4,096 bytes (34,359,742,464 bytes). +The additional 4,096 bytes keep the usable swap capacity at or above 32 GiB +after formatting. +Setup failure stops before candidate checkout and removes partial state only +after proving the file inactive or successfully disabling it. +After `swapon` succeeds, both rollout paths make up to five activation +observations, one second apart. +If visibility remains stale, cleanup treats the file as active. +Cleanup removes it only after `swapoff` succeeds. +Successful state is discarded with the ephemeral runner. + +This rollout adds the trusted pre-checkout setup. +During rollout, the PR temporarily retains the reviewed live Vitest helper. +The helper exists only because the PR must validate against the older workflow +definition on `main` before this change lands. +The compatibility path runs only when GitHub Actions supplies a validated +lowercase 40-hex checkout SHA. +When the trusted step already provides 32 GiB of usable swap, the helper exits +before it creates its fixed swap file. +A follow-up must remove the candidate-side helper and its compatibility tests +after this change lands. + +The fallback covers agent-turn latency, Hermes inference switch and shields, +the Hermes Bedrock and stable MCP shards, and the `hermes-e2e`, +`hermes-dashboard`, and Hermes security-posture tests. Scheduled and ordinary +manual `main` runs, larger-runner executions, rebuild lanes with workflow-managed swap, dedicated-runner lanes, `mcp-bridge-dev`, and non-Hermes -shards do not use this fallback. +shards do not use it. The fallback exists because the alternate-checkout trust boundary deliberately keeps PR-authored code from selecting the administrator-managed larger-runner @@ -709,7 +738,8 @@ provisions the same swap file on GitHub Actions when a trusted control-plane run uses the workflow definition from `main`. Those paths build large Hermes image layers and can otherwise exhaust the runner's default memory and swap during Docker layer export. Other E2E jobs keep the standard runner memory -configuration. +configuration except for the exact-head Hermes PR-gate fallback described in +[Larger-runner routing](#larger-runner-routing). These assertions run inside the existing `full-e2e` lifecycle instead of a second standalone onboarding run. This keeps the measurement on the job's first diff --git a/test/e2e/support/live-vitest-invocation.test.ts b/test/e2e/support/live-vitest-invocation.test.ts index 82151de6e0a..4708d69d9c3 100644 --- a/test/e2e/support/live-vitest-invocation.test.ts +++ b/test/e2e/support/live-vitest-invocation.test.ts @@ -12,6 +12,7 @@ import { buildLiveVitestArgs, HERMES_E2E_SWAP_BYTES, HERMES_E2E_SWAP_FILE, + HERMES_E2E_SWAP_FILE_BYTES, HERMES_E2E_SWAP_SCRIPT, LIVE_VITEST_PROJECT, type LiveVitestSpawner, @@ -30,6 +31,8 @@ const EXACT_HEAD_SHA = "a".repeat(40); interface FakeSwapScriptOptions { failCleanupQuery?: boolean; failSwapoff?: boolean; + hiddenActivationReads?: number; + provisionedSwapBytes?: number; } interface FakeSwapScriptResult { @@ -49,15 +52,32 @@ function runHermesSwapScriptFailure(options: FakeSwapScriptOptions = {}): FakeSw const callLog = path.join(fakeBin, "calls.log"); const swapState = path.join(fakeBin, "swap-state"); const nameQueryCount = path.join(fakeBin, "name-query-count"); + const activationQueryCount = path.join(fakeBin, "activation-query-count"); writeFileSync(swapState, "inactive\n"); writeFakeCommand(fakeBin, "swapon", [ `printf 'swapon:%s\\n' "$*" >> "$FAKE_CALL_LOG"`, 'case "$*" in', - ' *"--output SIZE"*)', - ` printf '1\\n'`, + ' *"--show=SIZE"*)', + ' swap_state="inactive"', + ' if [ -f "$FAKE_SWAP_STATE_FILE" ]; then', + ' IFS= read -r swap_state < "$FAKE_SWAP_STATE_FILE" || swap_state="inactive"', + " fi", + ' if [ "$swap_state" = "active" ]; then', + " activation_count=0", + ' if [ -f "$FAKE_ACTIVATION_QUERY_COUNT_FILE" ]; then', + ' IFS= read -r activation_count < "$FAKE_ACTIVATION_QUERY_COUNT_FILE" || activation_count=0', + " fi", + ' if [ "$activation_count" -le "$FAKE_HIDDEN_ACTIVATION_READS" ]; then', + " printf '0\\n'", + " else", + ' printf "%s\\n" "$FAKE_PROVISIONED_SWAP_BYTES"', + " fi", + " else", + " printf '1\\n'", + " fi", " ;;", - ' *"--output NAME"*)', + ' *"--show=NAME"*)', " query_count=0", ' if [ -f "$FAKE_NAME_QUERY_COUNT_FILE" ]; then', ' IFS= read -r query_count < "$FAKE_NAME_QUERY_COUNT_FILE" || query_count=0', @@ -74,16 +94,33 @@ function runHermesSwapScriptFailure(options: FakeSwapScriptOptions = {}): FakeSw " fi", ` printf 'swapon-name-query:%s:%s\\n' "$query_count" "$swap_state" >> "$FAKE_CALL_LOG"`, ' if [ "$swap_state" = "active" ]; then', - ` printf '%s\\n' "$FAKE_FIXED_SWAP"`, + " activation_count=0", + ' if [ -f "$FAKE_ACTIVATION_QUERY_COUNT_FILE" ]; then', + ' IFS= read -r activation_count < "$FAKE_ACTIVATION_QUERY_COUNT_FILE" || activation_count=0', + " fi", + " activation_count=$((activation_count + 1))", + ' printf "%s\\n" "$activation_count" > "$FAKE_ACTIVATION_QUERY_COUNT_FILE"', + ' if [ "$activation_count" -gt "$FAKE_HIDDEN_ACTIVATION_READS" ]; then', + ` printf '%s\\n' "$FAKE_FIXED_SWAP"`, + " fi", " fi", " ;;", + ' "--show")', + " ;;", + ' *"--show"*)', + " exit 43", + " ;;", " *)", ` printf 'active\\n' > "$FAKE_SWAP_STATE_FILE"`, ` printf 'swapon-activate:%s\\n' "$1" >> "$FAKE_CALL_LOG"`, " ;;", "esac", ]); - writeFakeCommand(fakeBin, "awk", ["while IFS= read -r _line; do :; done", `printf '1\\n'`]); + writeFakeCommand(fakeBin, "awk", [ + "total=0", + "while IFS= read -r value; do total=$((total + value)); done", + `printf '%s\\n' "$total"`, + ]); writeFakeCommand(fakeBin, "swapoff", [ `printf 'swapoff:%s\\n' "$1" >> "$FAKE_CALL_LOG"`, 'if [ "${FAKE_FAIL_SWAPOFF:-0}" = "1" ]; then', @@ -95,6 +132,7 @@ function runHermesSwapScriptFailure(options: FakeSwapScriptOptions = {}): FakeSw writeFakeCommand(fakeBin, "fallocate", [`printf 'fallocate:%s\\n' "$*" >> "$FAKE_CALL_LOG"`]); writeFakeCommand(fakeBin, "chmod", [`printf 'chmod:%s\\n' "$*" >> "$FAKE_CALL_LOG"`]); writeFakeCommand(fakeBin, "mkswap", [`printf 'mkswap:%s\\n' "$*" >> "$FAKE_CALL_LOG"`]); + writeFakeCommand(fakeBin, "sleep", [`printf 'sleep:%s\\n' "$*" >> "$FAKE_CALL_LOG"`]); try { const result = spawnSync( @@ -107,15 +145,19 @@ function runHermesSwapScriptFailure(options: FakeSwapScriptOptions = {}): FakeSw "hermes-e2e-swap-test", HERMES_E2E_SWAP_FILE, String(HERMES_E2E_SWAP_BYTES), + String(HERMES_E2E_SWAP_FILE_BYTES), ], { encoding: "utf8", env: { FAKE_CALL_LOG: callLog, - FAKE_FAIL_NAME_QUERY_AT: options.failCleanupQuery ? "2" : "0", + FAKE_ACTIVATION_QUERY_COUNT_FILE: activationQueryCount, + FAKE_FAIL_NAME_QUERY_AT: options.failCleanupQuery ? "7" : "0", FAKE_FAIL_SWAPOFF: options.failSwapoff ? "1" : "0", FAKE_FIXED_SWAP: HERMES_E2E_SWAP_FILE, + FAKE_HIDDEN_ACTIVATION_READS: String(options.hiddenActivationReads ?? 0), FAKE_NAME_QUERY_COUNT_FILE: nameQueryCount, + FAKE_PROVISIONED_SWAP_BYTES: String(options.provisionedSwapBytes ?? 1), FAKE_SWAP_STATE_FILE: swapState, LC_ALL: "C", PATH: fakeBin, @@ -377,6 +419,7 @@ describe("runLiveVitestCommand Hermes resource setup (#7145)", () => { expect(calls).toHaveLength(2); expect(calls[0]?.[0]).toBe("/usr/bin/sudo"); expect(HERMES_E2E_SWAP_BYTES).toBe(34_359_738_368); + expect(HERMES_E2E_SWAP_FILE_BYTES).toBe(34_359_742_464); expect(HERMES_E2E_SWAP_FILE).toBe("/mnt/nemoclaw-hermes-e2e.swap"); expect(calls[0]?.[1].slice(0, 9)).toEqual([ "-n", @@ -393,6 +436,7 @@ describe("runLiveVitestCommand Hermes resource setup (#7145)", () => { "nemoclaw-hermes-e2e-swap", HERMES_E2E_SWAP_FILE, String(HERMES_E2E_SWAP_BYTES), + String(HERMES_E2E_SWAP_FILE_BYTES), ]); const script = calls[0]?.[1][9] ?? ""; expect( @@ -400,24 +444,26 @@ describe("runLiveVitestCommand Hermes resource setup (#7145)", () => { input: script, }).status, ).toBe(0); - expect(script).toContain("if (( active_swap_bytes >= swap_size_bytes )); then"); - expect(script).toContain( - 'active_swap_names="$(swapon --show --noheadings --raw --output NAME)"', - ); + expect(script).toContain("if (( active_swap_bytes >= required_swap_bytes )); then"); + expect(script).toContain('active_swap_names="$(swapon --show=NAME --noheadings --raw)"'); expect(script).toContain('if [[ "$active_swap_name" == "$swap_file" ]]; then'); expect(script).toContain( - 'if (( fixed_swap_active == 1 )); then\n swapoff "$swap_file"\nfi\nrm -f -- "$swap_file"', + "if (( cleanup_swap_active == 1 || swap_activation_succeeded == 1 )); then", ); expect(script).toContain( - 'if cleanup_swap_names="$(swapon --show --noheadings --raw --output NAME 2>/dev/null)"; then', + 'if cleanup_swap_names="$(swapon --show=NAME --noheadings --raw 2>/dev/null)"; then', ); + expect(script).toContain("swapon --show=SIZE --bytes --noheadings"); + expect(script).not.toContain("swapon --output"); expect(script).toContain('if swapoff "$swap_file" 2>/dev/null; then'); expect(script).toContain("Preserving active Hermes E2E swap after setup failure"); expect(script).toContain("Preserving Hermes E2E swap because active swap could not be queried"); expect(script).not.toContain('swapoff "$swap_file" 2>/dev/null || true'); expect(script).not.toContain("swap_enabled"); - expect(script).toContain('fallocate -l "$swap_size_bytes" "$swap_file"'); - expect(script).toContain("if (( active_swap_bytes < swap_size_bytes )); then"); + expect(script).toContain('fallocate -l "$swap_file_bytes" "$swap_file"'); + expect(script).toContain("activation_observation_attempts=5"); + expect(script).toContain("activation_observation_delay_seconds=1"); + expect(script).toContain("if ! observe_provisioned_swap; then"); expect(calls[1]?.[0]).toBe("npx"); }); @@ -467,43 +513,42 @@ describe("runLiveVitestCommand Hermes resource setup (#7145)", () => { }); describe("HERMES_E2E_SWAP_SCRIPT failure cleanup (#7145)", () => { - const provisioningFailureCalls = [ - "swapon:--show --bytes --noheadings --output SIZE", - "swapon:--show --noheadings --raw --output NAME", - "swapon-name-query:1:inactive", - `rm:-f -- ${HERMES_E2E_SWAP_FILE}`, - `fallocate:-l ${HERMES_E2E_SWAP_BYTES} ${HERMES_E2E_SWAP_FILE}`, - `chmod:0600 ${HERMES_E2E_SWAP_FILE}`, - `mkswap:${HERMES_E2E_SWAP_FILE}`, - `swapon:${HERMES_E2E_SWAP_FILE}`, - `swapon-activate:${HERMES_E2E_SWAP_FILE}`, - "swapon:--show --bytes --noheadings --output SIZE", - "swapon:--show --noheadings --raw --output NAME", - ]; + it("waits for delayed activation visibility before accepting the swap", () => { + const result = runHermesSwapScriptFailure({ + hiddenActivationReads: 2, + provisionedSwapBytes: HERMES_E2E_SWAP_BYTES, + }); + + expect(result.status).toBe(0); + expect(result.calls.filter((call) => call === "sleep:1")).toHaveLength(2); + expect( + result.calls.filter((call) => call === `swapon-activate:${HERMES_E2E_SWAP_FILE}`), + ).toHaveLength(1); + expect(result.calls.filter((call) => call.startsWith("swapoff:"))).toEqual([]); + }); it("removes the active fixed swap only after cleanup swapoff succeeds", () => { const result = runHermesSwapScriptFailure(); + const swapoffIndex = result.calls.indexOf(`swapoff:${HERMES_E2E_SWAP_FILE}`); + const removeIndex = result.calls.lastIndexOf(`rm:-f -- ${HERMES_E2E_SWAP_FILE}`); expect(result.status).toBe(1); expect(result.stderr).toContain("Hermes E2E swap provisioning failed"); - expect(result.calls).toEqual([ - ...provisioningFailureCalls, - "swapon-name-query:2:active", - `swapoff:${HERMES_E2E_SWAP_FILE}`, - `rm:-f -- ${HERMES_E2E_SWAP_FILE}`, - ]); + expect(result.calls.filter((call) => call === "sleep:1")).toHaveLength(4); + expect(swapoffIndex).toBeGreaterThan(-1); + expect(removeIndex).toBeGreaterThan(swapoffIndex); }, 15_000); - it("preserves the active fixed swap when cleanup swapoff fails", () => { - const result = runHermesSwapScriptFailure({ failSwapoff: true }); + it("preserves the activated swap when visibility stays stale and cleanup swapoff fails", () => { + const result = runHermesSwapScriptFailure({ + failSwapoff: true, + hiddenActivationReads: 5, + }); expect(result.status).toBe(1); expect(result.stderr).toContain("Preserving active Hermes E2E swap after setup failure"); - expect(result.calls).toEqual([ - ...provisioningFailureCalls, - "swapon-name-query:2:active", - `swapoff:${HERMES_E2E_SWAP_FILE}`, - ]); + expect(result.calls.filter((call) => call === "sleep:1")).toHaveLength(4); + expect(result.calls).toContain(`swapoff:${HERMES_E2E_SWAP_FILE}`); expect( result.calls .slice(result.calls.indexOf(`swapon-activate:${HERMES_E2E_SWAP_FILE}`) + 1) @@ -518,7 +563,7 @@ describe("HERMES_E2E_SWAP_SCRIPT failure cleanup (#7145)", () => { expect(result.stderr).toContain( "Preserving Hermes E2E swap because active swap could not be queried", ); - expect(result.calls).toEqual([...provisioningFailureCalls, "swapon-name-query:2:fail"]); + expect(result.calls).toContain("swapon-name-query:7:fail"); expect( result.calls .slice(result.calls.indexOf(`swapon-activate:${HERMES_E2E_SWAP_FILE}`) + 1) diff --git a/test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts b/test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts new file mode 100644 index 00000000000..9b6af84d15a --- /dev/null +++ b/test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts @@ -0,0 +1,421 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { + TRUSTED_HERMES_SWAP_SCRIPT, + TRUSTED_HERMES_SWAP_STEP_NAME, + validateTrustedHermesSwapWorkflow, +} from "../../../tools/e2e/trusted-hermes-swap-workflow-boundary.mts"; +import { readWorkflow } from "../../helpers/e2e-workflow-contract"; +import { requireFixture } from "./require-fixture"; + +type WorkflowStep = { + "continue-on-error"?: boolean; + env?: Record; + id?: string; + if?: string; + name?: string; + run?: string; + uses?: string; +}; + +type SwapWorkflow = { + jobs: Record; +}; + +const PROTECTED_JOBS = [ + "agent-turn-latency", + "bedrock-runtime-compatible-anthropic", + "hermes-dashboard", + "hermes-e2e", + "hermes-inference-switch", + "hermes-shields-config", + "mcp-bridge", + "security-posture", +] as const; + +function trustedSwapStep(workflow: SwapWorkflow, jobName: string): WorkflowStep { + const step = workflow.jobs[jobName]?.steps?.find( + (candidate) => candidate.name === TRUSTED_HERMES_SWAP_STEP_NAME, + ); + requireFixture(step, `${jobName} trusted Hermes swap step is missing`); + return step; +} + +type SwapHarnessOptions = { + activeSwapBytes?: number; + diskBytes?: number; + failCleanupQuery?: boolean; + failMkswap?: boolean; + failSwapoff?: boolean; + hiddenActivationReads?: number; + provisionedSwapBytes?: number; +}; + +type SwapHarnessResult = { + calls: string[]; + status: number | null; + stderr: string; +}; + +function writeFakeCommand(directory: string, name: string, lines: string[]): string { + const commandPath = path.join(directory, name); + writeFileSync(commandPath, `${["#!/bin/sh", "set -eu", ...lines].join("\n")}\n`); + chmodSync(commandPath, 0o755); + return commandPath; +} + +function runTrustedSwapHarness(options: SwapHarnessOptions = {}): SwapHarnessResult { + const fakeBin = mkdtempSync(path.join(tmpdir(), "nemoclaw-trusted-swap-")); + const callLog = path.join(fakeBin, "calls.log"); + const queryCount = path.join(fakeBin, "query-count"); + const swapState = path.join(fakeBin, "swap-state"); + const swapFile = "/mnt/nemoclaw-hermes-e2e-swap/nemoclaw-hermes.fake.swap"; + writeFileSync(swapState, "inactive\n"); + + const commands = new Map(); + commands.set( + "/usr/sbin/swapon", + writeFakeCommand(fakeBin, "swapon", [ + `printf 'swapon:%s\\n' "$*" >> "$FAKE_CALL_LOG"`, + 'case "$*" in', + ' *"--show=SIZE"*)', + ' state="$(head -n 1 "$FAKE_SWAP_STATE")"', + ' if [ "$state" = "active" ]; then', + " count=0", + ' [ ! -f "$FAKE_QUERY_COUNT" ] || count="$(head -n 1 "$FAKE_QUERY_COUNT")"', + ' if [ "$count" -le "$FAKE_HIDDEN_ACTIVATION_READS" ]; then', + ' printf "0\\n"', + " else", + ' printf "%s\\n" "$FAKE_PROVISIONED_SWAP_BYTES"', + " fi", + " else", + ' printf "%s\\n" "$FAKE_ACTIVE_SWAP_BYTES"', + " fi", + " ;;", + ' *"--show=NAME"*)', + " count=0", + ' [ ! -f "$FAKE_QUERY_COUNT" ] || count="$(head -n 1 "$FAKE_QUERY_COUNT")"', + " count=$((count + 1))", + ' printf "%s\\n" "$count" > "$FAKE_QUERY_COUNT"', + ' if [ "${FAKE_FAIL_QUERY_AT:-0}" -eq "$count" ]; then exit 41; fi', + ' state="$(head -n 1 "$FAKE_SWAP_STATE")"', + ' if [ "$state" = "active" ] && [ "$count" -gt "$FAKE_HIDDEN_ACTIVATION_READS" ]; then', + ' printf "%s\\n" "$FAKE_SWAP_FILE"', + " fi", + " ;;", + ' "--show")', + " ;;", + ' *"--show"*)', + " exit 42", + " ;;", + " *)", + ' printf "active\\n" > "$FAKE_SWAP_STATE"', + ' printf "swapon-activate:%s\\n" "$1" >> "$FAKE_CALL_LOG"', + " ;;", + "esac", + ]), + ); + commands.set( + "/usr/sbin/swapoff", + writeFakeCommand(fakeBin, "swapoff", [ + `printf 'swapoff:%s\\n' "$1" >> "$FAKE_CALL_LOG"`, + '[ "${FAKE_FAIL_SWAPOFF:-0}" != "1" ] || exit 42', + 'printf "inactive\\n" > "$FAKE_SWAP_STATE"', + ]), + ); + commands.set( + "/usr/sbin/mkswap", + writeFakeCommand(fakeBin, "mkswap", [ + `printf 'mkswap:%s\\n' "$*" >> "$FAKE_CALL_LOG"`, + '[ "${FAKE_FAIL_MKSWAP:-0}" != "1" ] || exit 43', + ]), + ); + commands.set( + "/usr/bin/stat", + writeFakeCommand(fakeBin, "stat", [ + `printf 'stat:%s\\n' "$*" >> "$FAKE_CALL_LOG"`, + 'case "$*" in', + ' *"%F:%u:%g:%a"*) printf "directory:0:0:700\\n" ;;', + ' *"%F:%u:%g"*) printf "directory:0:0\\n" ;;', + ' *"%u:%g:%a"*) printf "0:0:600\\n" ;;', + ' *"%s"*) printf "34359742464\\n" ;;', + " *) exit 44 ;;", + "esac", + ]), + ); + commands.set( + "/usr/bin/df", + writeFakeCommand(fakeBin, "df", [ + `printf 'df:%s\\n' "$*" >> "$FAKE_CALL_LOG"`, + 'printf "Avail\\n%s\\n" "$FAKE_DISK_BYTES"', + ]), + ); + commands.set( + "/usr/bin/test", + writeFakeCommand(fakeBin, "test", [ + `printf 'test:%s\\n' "$*" >> "$FAKE_CALL_LOG"`, + 'case "$1:$2" in', + ' "-f:$FAKE_SWAP_FILE") exit 0 ;;', + ' "-L:$FAKE_SWAP_FILE") exit 1 ;;', + " *) exit 1 ;;", + "esac", + ]), + ); + commands.set( + "/usr/bin/mktemp", + writeFakeCommand(fakeBin, "mktemp", [ + `printf 'mktemp:%s\\n' "$*" >> "$FAKE_CALL_LOG"`, + 'printf "%s\\n" "$FAKE_SWAP_FILE"', + ]), + ); + commands.set( + "/usr/bin/sleep", + writeFakeCommand(fakeBin, "sleep", [`printf 'sleep:%s\\n' "$*" >> "$FAKE_CALL_LOG"`]), + ); + for (const command of ["mkdir", "fallocate", "rm", "rmdir"]) { + commands.set( + `/usr/bin/${command}`, + writeFakeCommand(fakeBin, command, [`printf '${command}:%s\\n' "$*" >> "$FAKE_CALL_LOG"`]), + ); + } + + let script = TRUSTED_HERMES_SWAP_SCRIPT.replaceAll("/usr/bin/sudo -n ", ""); + for (const [absolute, fake] of [...commands].sort( + ([left], [right]) => right.length - left.length, + )) { + script = script.replaceAll(absolute, fake); + } + + try { + const workflowSha = "b".repeat(40); + const result = spawnSync("/bin/bash", ["--noprofile", "--norc", "-c", script], { + encoding: "utf8", + env: { + BASH_ENV: "/dev/null", + CHECKOUT_SHA: "a".repeat(40), + DISPATCH_SHA: workflowSha, + ENV: "/dev/null", + EVENT_NAME: "workflow_dispatch", + EXPECTED_WORKFLOW_SHA: workflowSha, + FAKE_ACTIVE_SWAP_BYTES: String(options.activeSwapBytes ?? 0), + FAKE_CALL_LOG: callLog, + FAKE_DISK_BYTES: String(options.diskBytes ?? 100_000_000_000), + FAKE_FAIL_MKSWAP: options.failMkswap ? "1" : "0", + FAKE_FAIL_QUERY_AT: options.failCleanupQuery ? "6" : "0", + FAKE_FAIL_SWAPOFF: options.failSwapoff ? "1" : "0", + FAKE_HIDDEN_ACTIVATION_READS: String(options.hiddenActivationReads ?? 0), + FAKE_PROVISIONED_SWAP_BYTES: String(options.provisionedSwapBytes ?? 1), + FAKE_QUERY_COUNT: queryCount, + FAKE_SWAP_FILE: swapFile, + FAKE_SWAP_STATE: swapState, + LC_ALL: "C", + PATH: "/usr/bin:/bin", + REF: "refs/heads/main", + REPOSITORY: "NVIDIA/NemoClaw", + RUNNER_ARCH_KIND: "X64", + RUNNER_ENVIRONMENT_KIND: "github-hosted", + RUNNER_OS_KIND: "Linux", + WORKFLOW_SHA: workflowSha, + }, + }); + const calls = readFileSync(callLog, "utf8").trimEnd().split("\n"); + return { calls, status: result.status, stderr: result.stderr }; + } finally { + rmSync(fakeBin, { force: true, recursive: true }); + } +} + +describe("trusted Hermes swap workflow boundary", () => { + // source-shape-contract: security -- Pins the trusted privileged program to the first pre-checkout step in every eligible lane + it("keeps the fixed privileged program before candidate checkout in every protected job (#7145)", () => { + const workflow = readWorkflow() as SwapWorkflow; + + expect(validateTrustedHermesSwapWorkflow(workflow)).toEqual([]); + for (const jobName of PROTECTED_JOBS) { + const steps = workflow.jobs[jobName]!.steps!; + const provision = trustedSwapStep(workflow, jobName); + const checkout = steps.find((step) => step.uses?.startsWith("actions/checkout@")); + + expect(steps.indexOf(provision)).toBe(0); + expect(steps.indexOf(checkout!)).toBeGreaterThan(0); + expect(provision.run?.trimEnd()).toBe(TRUSTED_HERMES_SWAP_SCRIPT); + expect(JSON.stringify(provision.env)).not.toContain("secrets."); + } + }); + + it("keeps the trusted program fail-closed, bounded, and syntactically valid (#7145)", () => { + expect( + spawnSync("/bin/bash", ["--noprofile", "--norc", "-n"], { + input: TRUSTED_HERMES_SWAP_SCRIPT, + }).status, + ).toBe(0); + expect(TRUSTED_HERMES_SWAP_SCRIPT).toContain('"${RUNNER_ENVIRONMENT_KIND}" != "github-hosted"'); + expect(TRUSTED_HERMES_SWAP_SCRIPT).toContain("readonly required_swap_bytes=34359738368"); + expect(TRUSTED_HERMES_SWAP_SCRIPT).toContain("readonly swap_file_bytes=34359742464"); + expect(TRUSTED_HERMES_SWAP_SCRIPT).toContain("readonly reserve_bytes=17179869184"); + expect(TRUSTED_HERMES_SWAP_SCRIPT).toContain("readonly activation_observation_attempts=5"); + expect(TRUSTED_HERMES_SWAP_SCRIPT).toContain("readonly activation_observation_delay_seconds=1"); + expect(TRUSTED_HERMES_SWAP_SCRIPT).toContain( + '/usr/bin/sudo -n /usr/bin/mktemp --tmpdir="${swap_dir}"', + ); + expect(TRUSTED_HERMES_SWAP_SCRIPT).toContain( + '/usr/bin/sudo -n /usr/sbin/swapoff "${swap_file}"', + ); + expect(TRUSTED_HERMES_SWAP_SCRIPT).toContain( + "/usr/bin/sudo -n /usr/sbin/swapon --show=SIZE --bytes --noheadings", + ); + expect(TRUSTED_HERMES_SWAP_SCRIPT).toContain( + "/usr/bin/sudo -n /usr/sbin/swapon --show=NAME --noheadings --raw", + ); + expect(TRUSTED_HERMES_SWAP_SCRIPT).not.toContain("/usr/sbin/swapon --output"); + expect(TRUSTED_HERMES_SWAP_SCRIPT).not.toContain("/bin/bash -c"); + expect(TRUSTED_HERMES_SWAP_SCRIPT).not.toContain("${{"); + }); + + it("exits before privileged allocation when enough swap is already active (#7145)", () => { + const result = runTrustedSwapHarness({ activeSwapBytes: 34_359_738_368 }); + + expect(result.status).toBe(0); + expect(result.calls).toEqual([ + "stat:-c %F:%u:%g -- /mnt", + "swapon:--show=SIZE --bytes --noheadings", + ]); + }); + + it("provisions bounded swap without cleanup when setup succeeds (#7145)", () => { + const result = runTrustedSwapHarness({ provisionedSwapBytes: 34_359_738_368 }); + + expect(result.status).toBe(0); + expect(result.calls).toEqual( + expect.arrayContaining([ + "fallocate:-l 34359742464 /mnt/nemoclaw-hermes-e2e-swap/nemoclaw-hermes.fake.swap", + "mkswap:--quiet /mnt/nemoclaw-hermes-e2e-swap/nemoclaw-hermes.fake.swap", + "swapon:/mnt/nemoclaw-hermes-e2e-swap/nemoclaw-hermes.fake.swap", + "swapon-activate:/mnt/nemoclaw-hermes-e2e-swap/nemoclaw-hermes.fake.swap", + "swapon:--show=NAME --noheadings --raw", + "swapon:--show=SIZE --bytes --noheadings", + "swapon:--show", + ]), + ); + expect(result.calls.filter((call) => call.startsWith("swapon-activate:"))).toHaveLength(1); + expect( + result.calls.filter( + (call) => + call.startsWith("swapoff:") || call.startsWith("rm:") || call.startsWith("rmdir:"), + ), + ).toEqual([]); + }); + + it("waits for delayed activation visibility without repeating activation (#7145)", () => { + const result = runTrustedSwapHarness({ + hiddenActivationReads: 2, + provisionedSwapBytes: 34_359_738_368, + }); + + expect(result.status).toBe(0); + expect(result.calls.filter((call) => call === "sleep:1")).toHaveLength(2); + expect(result.calls.filter((call) => call.startsWith("swapon-activate:"))).toHaveLength(1); + expect(result.calls.filter((call) => call.startsWith("swapoff:"))).toEqual([]); + }); + + it("removes an inactive partial allocation after setup fails (#7145)", () => { + const result = runTrustedSwapHarness({ failMkswap: true }); + + expect(result.status).toBe(43); + expect(result.calls).toEqual( + expect.arrayContaining([ + "rm:-f -- /mnt/nemoclaw-hermes-e2e-swap/nemoclaw-hermes.fake.swap", + "rmdir:-- /mnt/nemoclaw-hermes-e2e-swap", + ]), + ); + }); + + it("disables an active partial allocation before removing it (#7145)", () => { + const result = runTrustedSwapHarness(); + const swapoffIndex = result.calls.indexOf( + "swapoff:/mnt/nemoclaw-hermes-e2e-swap/nemoclaw-hermes.fake.swap", + ); + const removeIndex = result.calls.indexOf( + "rm:-f -- /mnt/nemoclaw-hermes-e2e-swap/nemoclaw-hermes.fake.swap", + ); + + expect(result.status).toBe(1); + expect(swapoffIndex).toBeGreaterThan(-1); + expect(removeIndex).toBeGreaterThan(swapoffIndex); + }); + + it.each([ + { + expected: "Preserving Hermes E2E swap because active swap could not be queried", + name: "the active-swap query fails", + options: { failCleanupQuery: true }, + }, + { + expected: "Preserving active Hermes E2E swap after setup failure", + name: "activation visibility stays stale and swapoff fails", + options: { failSwapoff: true, hiddenActivationReads: 5 }, + }, + ])("preserves the partial allocation when $name (#7145)", ({ expected, options }) => { + const result = runTrustedSwapHarness(options); + + expect(result.status).toBe(1); + expect(result.stderr).toContain(expected); + expect(result.calls.filter((call) => call.startsWith("rm:"))).toEqual([]); + }); + + it("fails before allocation when disk reserve is unavailable (#7145)", () => { + const result = runTrustedSwapHarness({ diskBytes: 1 }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("insufficient disk capacity"); + expect(result.calls.some((call) => call.startsWith("mkdir:"))).toBe(false); + }); + + it("rejects eligibility, ordering, environment, and program drift (#7145)", () => { + const workflow = readWorkflow() as SwapWorkflow; + const latencySteps = workflow.jobs["agent-turn-latency"]!.steps!; + const latencyProvision = trustedSwapStep(workflow, "agent-turn-latency"); + latencySteps.splice(latencySteps.indexOf(latencyProvision), 1); + latencySteps.push(latencyProvision); + workflow.jobs["agent-turn-latency"]!.needs = "candidate-plan"; + latencyProvision["continue-on-error"] = true; + + const securityProvision = trustedSwapStep(workflow, "security-posture"); + securityProvision.if = securityProvision.if!.replace(" && matrix.agent == 'hermes'", ""); + securityProvision.env!.NVIDIA_INFERENCE_API_KEY = "${{ secrets.NVIDIA_INFERENCE_API_KEY }}"; + + const bedrockProvision = trustedSwapStep(workflow, "bedrock-runtime-compatible-anthropic"); + bedrockProvision.run = "sudo bash tools/e2e/live-vitest-invocation.mts"; + + const hermesE2eProvision = trustedSwapStep(workflow, "hermes-e2e"); + hermesE2eProvision.if = hermesE2eProvision.if!.replace( + " && (contains(format(',{0},', inputs.jobs), ',hermes-e2e,') || contains(format(',{0},', inputs.targets), ',hermes-e2e,'))", + "", + ); + + workflow.jobs["mcp-bridge-dev"]!.steps!.unshift({ + ...trustedSwapStep(workflow, "mcp-bridge"), + }); + + expect(validateTrustedHermesSwapWorkflow(workflow)).toEqual( + expect.arrayContaining([ + "agent-turn-latency trusted Hermes swap job must depend on controller validation", + "agent-turn-latency trusted Hermes swap step must preserve its fail-closed shape", + "agent-turn-latency trusted Hermes swap step must run before candidate checkout", + "hermes-e2e trusted Hermes swap step must preserve the exact-head main guard", + "security-posture trusted Hermes swap step must preserve the exact-head main guard", + "security-posture trusted Hermes swap step must bind only trusted workflow, checkout, and runner identity", + "bedrock-runtime-compatible-anthropic trusted Hermes swap step must preserve the fixed privileged program", + "mcp-bridge-dev job must not provision trusted Hermes swap", + ]), + ); + }); +}); diff --git a/test/helpers/vitest-watch-triggers.ts b/test/helpers/vitest-watch-triggers.ts index 1831911662d..080f7390bdc 100644 --- a/test/helpers/vitest-watch-triggers.ts +++ b/test/helpers/vitest-watch-triggers.ts @@ -34,6 +34,7 @@ const E2E_WORKFLOW_CONTRACTS = [ "test/e2e/support/security-posture-workflow-boundary.test.ts", "test/e2e/support/shared-e2e-workflow-boundary.test.ts", "test/e2e/support/spark-install-workflow-boundary.test.ts", + "test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts", "test/e2e/support/tunnel-lifecycle-workflow-boundary.test.ts", "test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts", "test/e2e/support/workflow-plan.test.ts", diff --git a/test/vitest-watch-triggers.test.ts b/test/vitest-watch-triggers.test.ts index b682eba6d54..0e67b043dc0 100644 --- a/test/vitest-watch-triggers.test.ts +++ b/test/vitest-watch-triggers.test.ts @@ -40,6 +40,7 @@ const E2E_WORKFLOW_CONTRACTS = [ "test/e2e/support/security-posture-workflow-boundary.test.ts", "test/e2e/support/shared-e2e-workflow-boundary.test.ts", "test/e2e/support/spark-install-workflow-boundary.test.ts", + "test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts", "test/e2e/support/tunnel-lifecycle-workflow-boundary.test.ts", "test/e2e/support/upload-e2e-artifacts-workflow-boundary.test.ts", "test/e2e/support/workflow-plan.test.ts", diff --git a/tools/e2e/live-vitest-invocation.mts b/tools/e2e/live-vitest-invocation.mts index feea5c2717a..1193d5bea30 100644 --- a/tools/e2e/live-vitest-invocation.mts +++ b/tools/e2e/live-vitest-invocation.mts @@ -20,9 +20,11 @@ const { spawnExitCode } = processExit; export const LIVE_VITEST_PROJECT = "e2e-live"; export const LIVE_TEST_ROOT = "test/e2e/live/"; export const RISK_SIGNAL_REPORTER = "test/e2e/risk-signal-reporter.ts"; -// Credentialed E2E trusts the workflow from main but executes this helper from -// the reviewed PR checkout, so exact-head resource setup must live here. +// Keep this exact-head helper during the trusted workflow's first rollout +// phase. Once the pre-checkout workflow step is on main, it provisions enough +// swap for this helper to return before candidate-side privileged mutation. export const HERMES_E2E_SWAP_BYTES = 32 * 1024 * 1024 * 1024; +export const HERMES_E2E_SWAP_FILE_BYTES = HERMES_E2E_SWAP_BYTES + 4096; export const HERMES_E2E_SWAP_FILE = "/mnt/nemoclaw-hermes-e2e.swap"; const SHELL_METACHARACTER = /[^A-Za-z0-9_./^$=:@+-]/u; @@ -39,16 +41,26 @@ const HERMES_SHARED_E2E_TARGETS = new Set(["hermes-dashboard", "hermes-e2e", "se const HERMES_MCP_BUILD_TEST = "test/e2e/live/mcp-bridge.test.ts"; export const HERMES_E2E_SWAP_SCRIPT = `set -euo pipefail swap_file="$1" -swap_size_bytes="$2" +required_swap_bytes="$2" +swap_file_bytes="$3" +activation_observation_attempts=5 +activation_observation_delay_seconds=1 +swap_activation_succeeded=0 -case "$swap_size_bytes" in +case "$required_swap_bytes" in ""|*[!0-9]*) - echo "Hermes E2E swap size must be an integer byte count" >&2 + echo "Hermes E2E required swap size must be an integer byte count" >&2 + exit 2 + ;; +esac +case "$swap_file_bytes" in + ""|*[!0-9]*) + echo "Hermes E2E swap file size must be an integer byte count" >&2 exit 2 ;; esac -active_swap_bytes="$(swapon --show --bytes --noheadings --output SIZE | awk '{ total += $1 } END { printf "%.0f", total }')" +active_swap_bytes="$(swapon --show=SIZE --bytes --noheadings | awk '{ total += $1 } END { printf "%.0f", total }')" active_swap_bytes="\${active_swap_bytes:-0}" case "$active_swap_bytes" in ""|*[!0-9]*) @@ -57,12 +69,12 @@ case "$active_swap_bytes" in ;; esac -if (( active_swap_bytes >= swap_size_bytes )); then +if (( active_swap_bytes >= required_swap_bytes )); then printf 'Hermes E2E swap is already sufficient: %s bytes active\\n' "$active_swap_bytes" exit 0 fi -active_swap_names="$(swapon --show --noheadings --raw --output NAME)" +active_swap_names="$(swapon --show=NAME --noheadings --raw)" fixed_swap_active=0 while IFS= read -r active_swap_name; do if [[ "$active_swap_name" == "$swap_file" ]]; then @@ -78,7 +90,7 @@ rm -f -- "$swap_file" cleanup_partial_swap() { status="$?" if (( status != 0 )); then - if cleanup_swap_names="$(swapon --show --noheadings --raw --output NAME 2>/dev/null)"; then + if cleanup_swap_names="$(swapon --show=NAME --noheadings --raw 2>/dev/null)"; then cleanup_swap_active=0 while IFS= read -r cleanup_swap_name; do if [[ "$cleanup_swap_name" == "$swap_file" ]]; then @@ -86,7 +98,7 @@ cleanup_partial_swap() { break fi done <<< "$cleanup_swap_names" - if (( cleanup_swap_active == 1 )); then + if (( cleanup_swap_active == 1 || swap_activation_succeeded == 1 )); then if swapoff "$swap_file" 2>/dev/null; then rm -f -- "$swap_file" || true else @@ -104,21 +116,42 @@ cleanup_partial_swap() { } trap cleanup_partial_swap EXIT -fallocate -l "$swap_size_bytes" "$swap_file" +fallocate -l "$swap_file_bytes" "$swap_file" chmod 0600 "$swap_file" mkswap "$swap_file" swapon "$swap_file" +swap_activation_succeeded=1 + +observe_provisioned_swap() { + activation_observation_attempt=1 + while (( activation_observation_attempt <= activation_observation_attempts )); do + provisioned_swap_active=0 + if active_swap_names="$(swapon --show=NAME --noheadings --raw 2>/dev/null)"; then + while IFS= read -r active_swap_name; do + if [[ "$active_swap_name" == "$swap_file" ]]; then + provisioned_swap_active=1 + break + fi + done <<< "$active_swap_names" + fi + if observed_swap_bytes="$(swapon --show=SIZE --bytes --noheadings 2>/dev/null | awk '{ total += $1 } END { printf "%.0f", total }')"; then + observed_swap_bytes="\${observed_swap_bytes:-0}" + if [[ "$observed_swap_bytes" != *[!0-9]* ]] && + (( provisioned_swap_active == 1 && observed_swap_bytes >= required_swap_bytes )); then + active_swap_bytes="$observed_swap_bytes" + return 0 + fi + fi + if (( activation_observation_attempt < activation_observation_attempts )); then + sleep "$activation_observation_delay_seconds" + fi + activation_observation_attempt=$((activation_observation_attempt + 1)) + done + return 1 +} -active_swap_bytes="$(swapon --show --bytes --noheadings --output SIZE | awk '{ total += $1 } END { printf "%.0f", total }')" -active_swap_bytes="\${active_swap_bytes:-0}" -case "$active_swap_bytes" in - ""|*[!0-9]*) - echo "Unable to verify active swap capacity" >&2 - exit 2 - ;; -esac -if (( active_swap_bytes < swap_size_bytes )); then - printf 'Hermes E2E swap provisioning failed: %s of %s bytes active\\n' "$active_swap_bytes" "$swap_size_bytes" >&2 +if ! observe_provisioned_swap; then + printf 'Hermes E2E swap provisioning failed: required swap was not visible after %s attempts\\n' "$activation_observation_attempts" >&2 exit 1 fi @@ -290,6 +323,7 @@ export function provisionHermesE2ESwap( "nemoclaw-hermes-e2e-swap", HERMES_E2E_SWAP_FILE, String(HERMES_E2E_SWAP_BYTES), + String(HERMES_E2E_SWAP_FILE_BYTES), ], { stdio: "inherit" }, ), diff --git a/tools/e2e/trusted-hermes-swap-workflow-boundary.mts b/tools/e2e/trusted-hermes-swap-workflow-boundary.mts new file mode 100644 index 00000000000..69ea06efe4e --- /dev/null +++ b/tools/e2e/trusted-hermes-swap-workflow-boundary.mts @@ -0,0 +1,291 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { isDeepStrictEqual } from "node:util"; + +type WorkflowRecord = Record; +type WorkflowStep = WorkflowRecord & { + env?: WorkflowRecord; + id?: string; + if?: string; + name?: string; + run?: string; + shell?: string; + uses?: string; +}; + +export const TRUSTED_HERMES_SWAP_STEP_NAME = "Provision trusted Hermes E2E swap"; +export const TRUSTED_HERMES_SWAP_STEP_ID = "trusted_hermes_swap"; + +const TRUSTED_HERMES_SWAP_IF = + "github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != ''"; +const TRUSTED_HERMES_E2E_SELECTION = + "(contains(format(',{0},', inputs.jobs), ',hermes-e2e,') || contains(format(',{0},', inputs.targets), ',hermes-e2e,'))"; +const TRUSTED_HERMES_SWAP_SHELL = "/bin/bash --noprofile --norc -e -o pipefail {0}"; +const TRUSTED_HERMES_SWAP_ENV = { + BASH_ENV: "/dev/null", + CHECKOUT_SHA: "${{ inputs.checkout_sha }}", + DISPATCH_SHA: "${{ github.sha }}", + ENV: "/dev/null", + EVENT_NAME: "${{ github.event_name }}", + EXPECTED_WORKFLOW_SHA: "${{ inputs.workflow_sha }}", + LC_ALL: "C", + REF: "${{ github.ref }}", + REPOSITORY: "${{ github.repository }}", + RUNNER_ARCH_KIND: "${{ runner.arch }}", + RUNNER_ENVIRONMENT_KIND: "${{ runner.environment }}", + RUNNER_OS_KIND: "${{ runner.os }}", + WORKFLOW_SHA: "${{ github.workflow_sha }}", +} as const; + +export const TRUSTED_HERMES_SWAP_SCRIPT = [ + "set -euo pipefail", + 'readonly swap_dir="/mnt/nemoclaw-hermes-e2e-swap"', + "readonly required_swap_bytes=34359738368", + "readonly swap_file_bytes=34359742464", + "readonly reserve_bytes=17179869184", + "readonly activation_observation_attempts=5", + "readonly activation_observation_delay_seconds=1", + 'swap_file=""', + "swap_activation_succeeded=0", + "", + "fail() {", + " printf 'Trusted Hermes E2E swap setup failed: %s\\n' \"$1\" >&2", + " exit 1", + "}", + "", + 'if [[ "${REPOSITORY}" != "NVIDIA/NemoClaw" || "${EVENT_NAME}" != "workflow_dispatch" || "${REF}" != "refs/heads/main" ]]; then', + ' fail "workflow must run from NVIDIA/NemoClaw main"', + "fi", + 'if [[ ! "${CHECKOUT_SHA}" =~ ^[0-9a-f]{40}$ ]]; then', + ' fail "checkout SHA must be lowercase 40-hex"', + "fi", + 'if [[ ! "${EXPECTED_WORKFLOW_SHA}" =~ ^[0-9a-f]{40}$ || "${WORKFLOW_SHA}" != "${EXPECTED_WORKFLOW_SHA}" || "${WORKFLOW_SHA}" != "${DISPATCH_SHA}" ]]; then', + ' fail "workflow source must match the trusted dispatch revision"', + "fi", + 'if [[ "${RUNNER_ENVIRONMENT_KIND}" != "github-hosted" || "${RUNNER_OS_KIND}" != "Linux" || "${RUNNER_ARCH_KIND}" != "X64" ]]; then', + ' fail "swap fallback requires an ephemeral GitHub-hosted Linux x64 runner"', + "fi", + 'mnt_metadata="$(/usr/bin/stat -c "%F:%u:%g" -- /mnt)"', + 'if [[ "${mnt_metadata}" != "directory:0:0" ]]; then', + ' fail "/mnt must be a root-owned directory"', + "fi", + "", + "read_active_swap_bytes() {", + " /usr/bin/sudo -n /usr/sbin/swapon --show=SIZE --bytes --noheadings |", + " /usr/bin/awk '{ total += $1 } END { printf \"%.0f\", total }'", + "}", + "", + 'active_swap_bytes="$(read_active_swap_bytes)"', + 'active_swap_bytes="${active_swap_bytes:-0}"', + 'if [[ ! "${active_swap_bytes}" =~ ^[0-9]+$ ]]; then', + ' fail "unable to determine active swap capacity"', + "fi", + "if (( active_swap_bytes >= required_swap_bytes )); then", + " printf 'Hermes E2E swap is already sufficient: %s bytes active\\n' \"${active_swap_bytes}\"", + " exit 0", + "fi", + "", + 'available_bytes="$(/usr/bin/df --block-size=1 --output=avail /mnt | /usr/bin/tail -n 1 | /usr/bin/tr -d "[:space:]")"', + 'if [[ ! "${available_bytes}" =~ ^[0-9]+$ ]]; then', + ' fail "unable to determine available disk capacity under /mnt"', + "fi", + "required_disk_bytes=$((swap_file_bytes + reserve_bytes))", + "if (( available_bytes < required_disk_bytes )); then", + ' fail "insufficient disk capacity: ${available_bytes} bytes available, ${required_disk_bytes} required"', + "fi", + "", + 'if /usr/bin/sudo -n /usr/bin/test -e "${swap_dir}" || /usr/bin/sudo -n /usr/bin/test -L "${swap_dir}"; then', + ' fail "refusing unexpected pre-existing swap path"', + "fi", + "", + "directory_created=0", + "cleanup_partial_swap() {", + ' status="$?"', + " if (( status != 0 && directory_created == 1 )); then", + ' if active_swap_names="$(/usr/bin/sudo -n /usr/sbin/swapon --show=NAME --noheadings --raw 2>/dev/null)"; then', + " fixed_swap_active=0", + " while IFS= read -r active_swap_name; do", + ' if [[ -n "${swap_file}" && "${active_swap_name}" == "${swap_file}" ]]; then', + " fixed_swap_active=1", + " break", + " fi", + ' done <<< "${active_swap_names}"', + " if (( fixed_swap_active == 1 || swap_activation_succeeded == 1 )); then", + ' if /usr/bin/sudo -n /usr/sbin/swapoff "${swap_file}" 2>/dev/null; then', + ' /usr/bin/sudo -n /usr/bin/rm -f -- "${swap_file}" || true', + ' /usr/bin/sudo -n /usr/bin/rmdir -- "${swap_dir}" || true', + " else", + " printf 'Preserving active Hermes E2E swap after setup failure: %s\\n' \"${swap_file}\" >&2", + " fi", + " else", + ' if [[ -n "${swap_file}" ]]; then', + ' /usr/bin/sudo -n /usr/bin/rm -f -- "${swap_file}" || true', + " fi", + ' /usr/bin/sudo -n /usr/bin/rmdir -- "${swap_dir}" || true', + " fi", + " else", + " printf 'Preserving Hermes E2E swap because active swap could not be queried: %s\\n' \"${swap_file}\" >&2", + " fi", + " fi", + " trap - EXIT", + ' exit "${status}"', + "}", + "trap cleanup_partial_swap EXIT", + "", + '/usr/bin/sudo -n /usr/bin/mkdir -m 0700 -- "${swap_dir}"', + "directory_created=1", + 'directory_metadata="$(/usr/bin/sudo -n /usr/bin/stat -c "%F:%u:%g:%a" -- "${swap_dir}")"', + 'if [[ "${directory_metadata}" != "directory:0:0:700" ]]; then', + ' fail "swap directory must be a root-owned mode-0700 directory"', + "fi", + 'swap_file="$(/usr/bin/sudo -n /usr/bin/mktemp --tmpdir="${swap_dir}" nemoclaw-hermes.XXXXXXXX.swap)"', + 'if ! /usr/bin/sudo -n /usr/bin/test -f "${swap_file}" || /usr/bin/sudo -n /usr/bin/test -L "${swap_file}"; then', + ' fail "swap file must be a regular non-symlink"', + "fi", + 'file_metadata="$(/usr/bin/sudo -n /usr/bin/stat -c "%u:%g:%a" -- "${swap_file}")"', + 'if [[ "${file_metadata}" != "0:0:600" ]]; then', + ' fail "swap file must be root-owned mode 0600"', + "fi", + '/usr/bin/sudo -n /usr/bin/fallocate -l "${swap_file_bytes}" "${swap_file}"', + 'file_size_bytes="$(/usr/bin/sudo -n /usr/bin/stat -c "%s" -- "${swap_file}")"', + 'if [[ ! "${file_size_bytes}" =~ ^[0-9]+$ || "${file_size_bytes}" -ne "${swap_file_bytes}" ]]; then', + ' fail "swap file size does not match the fixed backing allocation"', + "fi", + 'remaining_bytes="$(/usr/bin/df --block-size=1 --output=avail /mnt | /usr/bin/tail -n 1 | /usr/bin/tr -d "[:space:]")"', + 'if [[ ! "${remaining_bytes}" =~ ^[0-9]+$ || "${remaining_bytes}" -lt "${reserve_bytes}" ]]; then', + ' fail "swap allocation did not preserve the required disk reserve"', + "fi", + '/usr/bin/sudo -n /usr/sbin/mkswap --quiet "${swap_file}"', + '/usr/bin/sudo -n /usr/sbin/swapon "${swap_file}"', + "swap_activation_succeeded=1", + "", + "observe_provisioned_swap() {", + " activation_observation_attempt=1", + " while (( activation_observation_attempt <= activation_observation_attempts )); do", + " provisioned_swap_active=0", + ' if active_swap_names="$(/usr/bin/sudo -n /usr/sbin/swapon --show=NAME --noheadings --raw 2>/dev/null)"; then', + " while IFS= read -r active_swap_name; do", + ' if [[ "${active_swap_name}" == "${swap_file}" ]]; then', + " provisioned_swap_active=1", + " break", + " fi", + ' done <<< "${active_swap_names}"', + " fi", + ' if observed_swap_bytes="$(read_active_swap_bytes 2>/dev/null)"; then', + ' observed_swap_bytes="${observed_swap_bytes:-0}"', + ' if [[ "${observed_swap_bytes}" =~ ^[0-9]+$ ]] &&', + " (( provisioned_swap_active == 1 && observed_swap_bytes >= required_swap_bytes )); then", + ' active_swap_bytes="${observed_swap_bytes}"', + " return 0", + " fi", + " fi", + " if (( activation_observation_attempt < activation_observation_attempts )); then", + ' /usr/bin/sleep "${activation_observation_delay_seconds}"', + " fi", + " activation_observation_attempt=$((activation_observation_attempt + 1))", + " done", + " return 1", + "}", + "", + "if ! observe_provisioned_swap; then", + ' fail "unable to verify the required active swap capacity after bounded observation"', + "fi", + "", + "trap - EXIT", + "printf 'Hermes E2E swap ready: %s bytes active\\n' \"${active_swap_bytes}\"", + "/usr/bin/sudo -n /usr/sbin/swapon --show", +].join("\n"); + +const JOB_CONDITIONS = { + "agent-turn-latency": `\${{ ${TRUSTED_HERMES_SWAP_IF} }}`, + "bedrock-runtime-compatible-anthropic": `\${{ ${TRUSTED_HERMES_SWAP_IF} && matrix.agent == 'hermes' }}`, + "hermes-dashboard": `\${{ ${TRUSTED_HERMES_SWAP_IF} }}`, + "hermes-e2e": `\${{ ${TRUSTED_HERMES_SWAP_IF} && ${TRUSTED_HERMES_E2E_SELECTION} }}`, + "hermes-inference-switch": `\${{ ${TRUSTED_HERMES_SWAP_IF} }}`, + "hermes-shields-config": `\${{ ${TRUSTED_HERMES_SWAP_IF} }}`, + "mcp-bridge": `\${{ ${TRUSTED_HERMES_SWAP_IF} && matrix.agent == 'hermes' }}`, + "security-posture": `\${{ ${TRUSTED_HERMES_SWAP_IF} && matrix.agent == 'hermes' }}`, +} as const; + +function asRecord(value: unknown): WorkflowRecord { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as WorkflowRecord) + : {}; +} + +function asSteps(value: unknown): WorkflowStep[] { + return Array.isArray(value) ? value.map((step) => asRecord(step) as WorkflowStep) : []; +} + +export function validateTrustedHermesSwapWorkflow(workflowValue: unknown): string[] { + const errors: string[] = []; + const jobs = asRecord(asRecord(workflowValue).jobs); + + for (const [jobName, jobValue] of Object.entries(jobs)) { + const job = asRecord(jobValue); + const expectedCondition = JOB_CONDITIONS[jobName as keyof typeof JOB_CONDITIONS]; + const steps = asSteps(job.steps); + const provisionSteps = steps.filter( + (step) => + step.name === TRUSTED_HERMES_SWAP_STEP_NAME || step.id === TRUSTED_HERMES_SWAP_STEP_ID, + ); + + if (expectedCondition === undefined) { + if (provisionSteps.length > 0) { + errors.push(`${jobName} job must not provision trusted Hermes swap`); + } + continue; + } + + if (job.needs !== "generate-matrix") { + errors.push(`${jobName} trusted Hermes swap job must depend on controller validation`); + } + if (provisionSteps.length !== 1) { + errors.push(`${jobName} job must contain exactly one trusted Hermes swap step`); + continue; + } + + const provision = provisionSteps[0]!; + if ( + !isDeepStrictEqual(Object.keys(provision).sort(), ["env", "id", "if", "name", "run", "shell"]) + ) { + errors.push(`${jobName} trusted Hermes swap step must preserve its fail-closed shape`); + } + if (provision.id !== TRUSTED_HERMES_SWAP_STEP_ID) { + errors.push(`${jobName} trusted Hermes swap step must preserve its fixed id`); + } + if (provision.name !== TRUSTED_HERMES_SWAP_STEP_NAME) { + errors.push(`${jobName} trusted Hermes swap step must preserve its fixed name`); + } + if (provision.if !== expectedCondition) { + errors.push(`${jobName} trusted Hermes swap step must preserve the exact-head main guard`); + } + if (provision.shell !== TRUSTED_HERMES_SWAP_SHELL) { + errors.push(`${jobName} trusted Hermes swap step must use the isolated Bash shell`); + } + if (!isDeepStrictEqual(asRecord(provision.env), TRUSTED_HERMES_SWAP_ENV)) { + errors.push( + `${jobName} trusted Hermes swap step must bind only trusted workflow, checkout, and runner identity`, + ); + } + if ((provision.run ?? "").trimEnd() !== TRUSTED_HERMES_SWAP_SCRIPT) { + errors.push(`${jobName} trusted Hermes swap step must preserve the fixed privileged program`); + } + + const checkoutIndex = steps.findIndex((step) => + (step.uses ?? "").startsWith("actions/checkout@"), + ); + if (steps.indexOf(provision) !== 0 || checkoutIndex <= 0) { + errors.push(`${jobName} trusted Hermes swap step must run before candidate checkout`); + } + } + + for (const jobName of Object.keys(JOB_CONDITIONS)) { + if (!(jobName in jobs)) { + errors.push(`workflow missing trusted Hermes swap job ${jobName}`); + } + } + + return errors; +} diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index 51f4f980eaa..57732a03570 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -39,6 +39,7 @@ import { validateRunnerComparisonWorkflowBoundary } from "./runner-comparison-wo import { validateRunnerPressureWorkflow } from "./runner-pressure-workflow-boundary.mts"; import { validateSandboxOperationsWorkflow } from "./sandbox-operations-workflow-boundary.mts"; import { validateSecurityPostureWorkflow } from "./security-posture-workflow-boundary.mts"; +import { validateTrustedHermesSwapWorkflow } from "./trusted-hermes-swap-workflow-boundary.mts"; import { validateUploadE2eArtifactsWorkflowBoundary } from "./upload-e2e-artifacts-workflow-boundary.mts"; const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); @@ -4124,6 +4125,7 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { errors.push(...validateE2eOperationsWorkflow(workflow as unknown as OperationsWorkflow)); errors.push(...validateSecurityPostureWorkflow(workflow)); errors.push(...validateRunnerPressureWorkflow(workflow)); + errors.push(...validateTrustedHermesSwapWorkflow(workflow)); errors.push(...validateRunnerComparisonWorkflowBoundary(workflow)); const triggers = asRecord(workflow.on ?? workflow[true as unknown as string]);