fix(ci): trust exact-head Hermes swap setup

ci/source-shape-test-budget.json

@@ -271,6 +271,11 @@
       "test": "routes only the measured heavy lanes on trusted main (#7145)",
       "category": "security"
     },
+    {
+      "file": "test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts",
+      "test": "keeps the fixed privileged program before candidate checkout in every protected job (#7145)",
+      "category": "security"
+    },
     {
       "file": "test/fetch-guard-patch-regression.test.ts",
       "test": "requires classifier review and integrity evidence when the OpenClaw build pin changes",

test/e2e/README.md

@@ -63,16 +63,45 @@ standard runners even though they use the trusted workflow definition from
 `main`.
 
 Exact-head PR-gate dispatches use a bounded swap fallback for the hosted
-Hermes image-building lanes that remain on those standard runners. The live
-Vitest helper activates the fallback only when GitHub Actions supplies a
-validated lowercase 40-hex checkout SHA. It reuses at least 32 GiB of active
-swap when available; otherwise, it creates one fixed 32 GiB swap file under
-`/mnt` before agent-turn latency, Hermes inference switch and shields, the
-Hermes Bedrock and stable MCP shards, or the `hermes-e2e`, `hermes-dashboard`,
-and Hermes security-posture tests. Setup failure stops before Vitest. Scheduled
-and ordinary manual `main` runs, larger-runner executions, rebuild lanes with
+Hermes image-building lanes that remain on those standard runners. The trusted
+workflow provisions the fallback as the first job step, before checking out or
+executing the candidate revision. It requires a controller-supplied lowercase
+40-hex checkout SHA, matching trusted workflow and dispatch revisions, and an
+ephemeral GitHub-hosted Linux x64 runner.
+
+The trusted step requires at least 32 GiB (34,359,738,368 bytes) of usable swap.
+It reuses active swap that meets this requirement.
+Otherwise, it preserves at least 16 GiB of available disk capacity under
+`/mnt`, creates a root-owned mode-`0700` directory, and creates an exclusive
+randomized mode-`0600` file.
+The file allocation is 32 GiB plus 4,096 bytes (34,359,742,464 bytes).
+The additional 4,096 bytes keep the usable swap capacity at or above 32 GiB
+after formatting.
+Setup failure stops before candidate checkout and removes partial state only
+after proving the file inactive or successfully disabling it.
+After `swapon` succeeds, both rollout paths make up to five activation
+observations, one second apart.
+If visibility remains stale, cleanup treats the file as active.
+Cleanup removes it only after `swapoff` succeeds.
+Successful state is discarded with the ephemeral runner.
+
+This rollout adds the trusted pre-checkout setup.
+During rollout, the PR temporarily retains the reviewed live Vitest helper.
+The helper exists only because the PR must validate against the older workflow
+definition on `main` before this change lands.
+The compatibility path runs only when GitHub Actions supplies a validated
+lowercase 40-hex checkout SHA.
+When the trusted step already provides 32 GiB of usable swap, the helper exits
+before it creates its fixed swap file.
+A follow-up must remove the candidate-side helper and its compatibility tests
+after this change lands.
+
+The fallback covers agent-turn latency, Hermes inference switch and shields,
+the Hermes Bedrock and stable MCP shards, and the `hermes-e2e`,
+`hermes-dashboard`, and Hermes security-posture tests. Scheduled and ordinary
+manual `main` runs, larger-runner executions, rebuild lanes with
 workflow-managed swap, dedicated-runner lanes, `mcp-bridge-dev`, and non-Hermes
-shards do not use this fallback.
+shards do not use it.
 
 The fallback exists because the alternate-checkout trust boundary deliberately
 keeps PR-authored code from selecting the administrator-managed larger-runner
@@ -709,7 +738,8 @@ provisions the same swap file on GitHub Actions when a trusted control-plane
 run uses the workflow definition from `main`. Those paths build large Hermes
 image layers and can otherwise exhaust the runner's default memory and swap
 during Docker layer export. Other E2E jobs keep the standard runner memory
-configuration.
+configuration except for the exact-head Hermes PR-gate fallback described in
+[Larger-runner routing](#larger-runner-routing).
 
 These assertions run inside the existing `full-e2e` lifecycle instead of a
 second standalone onboarding run. This keeps the measurement on the job's first

test/e2e/support/live-vitest-invocation.test.ts

@@ -12,6 +12,7 @@ import {
   buildLiveVitestArgs,
   HERMES_E2E_SWAP_BYTES,
   HERMES_E2E_SWAP_FILE,
+  HERMES_E2E_SWAP_FILE_BYTES,
   HERMES_E2E_SWAP_SCRIPT,
   LIVE_VITEST_PROJECT,
   type LiveVitestSpawner,
@@ -30,6 +31,8 @@ const EXACT_HEAD_SHA = "a".repeat(40);
 interface FakeSwapScriptOptions {
   failCleanupQuery?: boolean;
   failSwapoff?: boolean;
+  hiddenActivationReads?: number;
+  provisionedSwapBytes?: number;
 }
 
 interface FakeSwapScriptResult {
@@ -49,15 +52,32 @@ function runHermesSwapScriptFailure(options: FakeSwapScriptOptions = {}): FakeSw
   const callLog = path.join(fakeBin, "calls.log");
   const swapState = path.join(fakeBin, "swap-state");
   const nameQueryCount = path.join(fakeBin, "name-query-count");
+  const activationQueryCount = path.join(fakeBin, "activation-query-count");
   writeFileSync(swapState, "inactive\n");
 
   writeFakeCommand(fakeBin, "swapon", [
     `printf 'swapon:%s\\n' "$*" >> "$FAKE_CALL_LOG"`,
     'case "$*" in',
-    '  *"--output SIZE"*)',
-    `    printf '1\\n'`,
+    '  *"--show=SIZE"*)',
+    '    swap_state="inactive"',
+    '    if [ -f "$FAKE_SWAP_STATE_FILE" ]; then',
+    '      IFS= read -r swap_state < "$FAKE_SWAP_STATE_FILE" || swap_state="inactive"',
+    "    fi",
+    '    if [ "$swap_state" = "active" ]; then',
+    "      activation_count=0",
+    '      if [ -f "$FAKE_ACTIVATION_QUERY_COUNT_FILE" ]; then',
+    '        IFS= read -r activation_count < "$FAKE_ACTIVATION_QUERY_COUNT_FILE" || activation_count=0',
+    "      fi",
+    '      if [ "$activation_count" -le "$FAKE_HIDDEN_ACTIVATION_READS" ]; then',
+    "        printf '0\\n'",
+    "      else",
+    '        printf "%s\\n" "$FAKE_PROVISIONED_SWAP_BYTES"',
+    "      fi",
+    "    else",
+    "      printf '1\\n'",
+    "    fi",
     "    ;;",
-    '  *"--output NAME"*)',
+    '  *"--show=NAME"*)',
     "    query_count=0",
     '    if [ -f "$FAKE_NAME_QUERY_COUNT_FILE" ]; then',
     '      IFS= read -r query_count < "$FAKE_NAME_QUERY_COUNT_FILE" || query_count=0',
@@ -74,16 +94,33 @@ function runHermesSwapScriptFailure(options: FakeSwapScriptOptions = {}): FakeSw
     "    fi",
     `    printf 'swapon-name-query:%s:%s\\n' "$query_count" "$swap_state" >> "$FAKE_CALL_LOG"`,
     '    if [ "$swap_state" = "active" ]; then',
-    `      printf '%s\\n' "$FAKE_FIXED_SWAP"`,
+    "      activation_count=0",
+    '      if [ -f "$FAKE_ACTIVATION_QUERY_COUNT_FILE" ]; then',
+    '        IFS= read -r activation_count < "$FAKE_ACTIVATION_QUERY_COUNT_FILE" || activation_count=0',
+    "      fi",
+    "      activation_count=$((activation_count + 1))",
+    '      printf "%s\\n" "$activation_count" > "$FAKE_ACTIVATION_QUERY_COUNT_FILE"',
+    '      if [ "$activation_count" -gt "$FAKE_HIDDEN_ACTIVATION_READS" ]; then',
+    `        printf '%s\\n' "$FAKE_FIXED_SWAP"`,
+    "      fi",
     "    fi",
     "    ;;",
+    '  "--show")',
+    "    ;;",
+    '  *"--show"*)',
+    "    exit 43",
+    "    ;;",
     "  *)",
     `    printf 'active\\n' > "$FAKE_SWAP_STATE_FILE"`,
     `    printf 'swapon-activate:%s\\n' "$1" >> "$FAKE_CALL_LOG"`,
     "    ;;",
     "esac",
   ]);
-  writeFakeCommand(fakeBin, "awk", ["while IFS= read -r _line; do :; done", `printf '1\\n'`]);
+  writeFakeCommand(fakeBin, "awk", [
+    "total=0",
+    "while IFS= read -r value; do total=$((total + value)); done",
+    `printf '%s\\n' "$total"`,
+  ]);
   writeFakeCommand(fakeBin, "swapoff", [
     `printf 'swapoff:%s\\n' "$1" >> "$FAKE_CALL_LOG"`,
     'if [ "${FAKE_FAIL_SWAPOFF:-0}" = "1" ]; then',
@@ -95,6 +132,7 @@ function runHermesSwapScriptFailure(options: FakeSwapScriptOptions = {}): FakeSw
   writeFakeCommand(fakeBin, "fallocate", [`printf 'fallocate:%s\\n' "$*" >> "$FAKE_CALL_LOG"`]);
   writeFakeCommand(fakeBin, "chmod", [`printf 'chmod:%s\\n' "$*" >> "$FAKE_CALL_LOG"`]);
   writeFakeCommand(fakeBin, "mkswap", [`printf 'mkswap:%s\\n' "$*" >> "$FAKE_CALL_LOG"`]);
+  writeFakeCommand(fakeBin, "sleep", [`printf 'sleep:%s\\n' "$*" >> "$FAKE_CALL_LOG"`]);
 
   try {
     const result = spawnSync(
@@ -107,15 +145,19 @@ function runHermesSwapScriptFailure(options: FakeSwapScriptOptions = {}): FakeSw
         "hermes-e2e-swap-test",
         HERMES_E2E_SWAP_FILE,
         String(HERMES_E2E_SWAP_BYTES),
+        String(HERMES_E2E_SWAP_FILE_BYTES),
       ],
       {
         encoding: "utf8",
         env: {
           FAKE_CALL_LOG: callLog,
-          FAKE_FAIL_NAME_QUERY_AT: options.failCleanupQuery ? "2" : "0",
+          FAKE_ACTIVATION_QUERY_COUNT_FILE: activationQueryCount,
+          FAKE_FAIL_NAME_QUERY_AT: options.failCleanupQuery ? "7" : "0",
           FAKE_FAIL_SWAPOFF: options.failSwapoff ? "1" : "0",
           FAKE_FIXED_SWAP: HERMES_E2E_SWAP_FILE,
+          FAKE_HIDDEN_ACTIVATION_READS: String(options.hiddenActivationReads ?? 0),
           FAKE_NAME_QUERY_COUNT_FILE: nameQueryCount,
+          FAKE_PROVISIONED_SWAP_BYTES: String(options.provisionedSwapBytes ?? 1),
           FAKE_SWAP_STATE_FILE: swapState,
           LC_ALL: "C",
           PATH: fakeBin,
@@ -377,6 +419,7 @@ describe("runLiveVitestCommand Hermes resource setup (#7145)", () => {
     expect(calls).toHaveLength(2);
     expect(calls[0]?.[0]).toBe("/usr/bin/sudo");
     expect(HERMES_E2E_SWAP_BYTES).toBe(34_359_738_368);
+    expect(HERMES_E2E_SWAP_FILE_BYTES).toBe(34_359_742_464);
     expect(HERMES_E2E_SWAP_FILE).toBe("/mnt/nemoclaw-hermes-e2e.swap");
     expect(calls[0]?.[1].slice(0, 9)).toEqual([
       "-n",
@@ -393,31 +436,34 @@ describe("runLiveVitestCommand Hermes resource setup (#7145)", () => {
       "nemoclaw-hermes-e2e-swap",
       HERMES_E2E_SWAP_FILE,
       String(HERMES_E2E_SWAP_BYTES),
+      String(HERMES_E2E_SWAP_FILE_BYTES),
     ]);
     const script = calls[0]?.[1][9] ?? "";
     expect(
       spawnSync("/bin/bash", ["--noprofile", "--norc", "-n"], {
         input: script,
       }).status,
     ).toBe(0);
-    expect(script).toContain("if (( active_swap_bytes >= swap_size_bytes )); then");
-    expect(script).toContain(
-      'active_swap_names="$(swapon --show --noheadings --raw --output NAME)"',
-    );
+    expect(script).toContain("if (( active_swap_bytes >= required_swap_bytes )); then");
+    expect(script).toContain('active_swap_names="$(swapon --show=NAME --noheadings --raw)"');
     expect(script).toContain('if [[ "$active_swap_name" == "$swap_file" ]]; then');
     expect(script).toContain(
-      'if (( fixed_swap_active == 1 )); then\n  swapoff "$swap_file"\nfi\nrm -f -- "$swap_file"',
+      "if (( cleanup_swap_active == 1 || swap_activation_succeeded == 1 )); then",
     );
     expect(script).toContain(
-      'if cleanup_swap_names="$(swapon --show --noheadings --raw --output NAME 2>/dev/null)"; then',
+      'if cleanup_swap_names="$(swapon --show=NAME --noheadings --raw 2>/dev/null)"; then',
     );
+    expect(script).toContain("swapon --show=SIZE --bytes --noheadings");
+    expect(script).not.toContain("swapon --output");
     expect(script).toContain('if swapoff "$swap_file" 2>/dev/null; then');
     expect(script).toContain("Preserving active Hermes E2E swap after setup failure");
     expect(script).toContain("Preserving Hermes E2E swap because active swap could not be queried");
     expect(script).not.toContain('swapoff "$swap_file" 2>/dev/null || true');
     expect(script).not.toContain("swap_enabled");
-    expect(script).toContain('fallocate -l "$swap_size_bytes" "$swap_file"');
-    expect(script).toContain("if (( active_swap_bytes < swap_size_bytes )); then");
+    expect(script).toContain('fallocate -l "$swap_file_bytes" "$swap_file"');
+    expect(script).toContain("activation_observation_attempts=5");
+    expect(script).toContain("activation_observation_delay_seconds=1");
+    expect(script).toContain("if ! observe_provisioned_swap; then");
     expect(calls[1]?.[0]).toBe("npx");
   });
 
@@ -467,43 +513,42 @@ describe("runLiveVitestCommand Hermes resource setup (#7145)", () => {
 });
 
 describe("HERMES_E2E_SWAP_SCRIPT failure cleanup (#7145)", () => {
-  const provisioningFailureCalls = [
-    "swapon:--show --bytes --noheadings --output SIZE",
-    "swapon:--show --noheadings --raw --output NAME",
-    "swapon-name-query:1:inactive",
-    `rm:-f -- ${HERMES_E2E_SWAP_FILE}`,
-    `fallocate:-l ${HERMES_E2E_SWAP_BYTES} ${HERMES_E2E_SWAP_FILE}`,
-    `chmod:0600 ${HERMES_E2E_SWAP_FILE}`,
-    `mkswap:${HERMES_E2E_SWAP_FILE}`,
-    `swapon:${HERMES_E2E_SWAP_FILE}`,
-    `swapon-activate:${HERMES_E2E_SWAP_FILE}`,
-    "swapon:--show --bytes --noheadings --output SIZE",
-    "swapon:--show --noheadings --raw --output NAME",
-  ];
+  it("waits for delayed activation visibility before accepting the swap", () => {
+    const result = runHermesSwapScriptFailure({
+      hiddenActivationReads: 2,
+      provisionedSwapBytes: HERMES_E2E_SWAP_BYTES,
+    });
+
+    expect(result.status).toBe(0);
+    expect(result.calls.filter((call) => call === "sleep:1")).toHaveLength(2);
+    expect(
+      result.calls.filter((call) => call === `swapon-activate:${HERMES_E2E_SWAP_FILE}`),
+    ).toHaveLength(1);
+    expect(result.calls.filter((call) => call.startsWith("swapoff:"))).toEqual([]);
+  });
 
   it("removes the active fixed swap only after cleanup swapoff succeeds", () => {
     const result = runHermesSwapScriptFailure();
+    const swapoffIndex = result.calls.indexOf(`swapoff:${HERMES_E2E_SWAP_FILE}`);
+    const removeIndex = result.calls.lastIndexOf(`rm:-f -- ${HERMES_E2E_SWAP_FILE}`);
 
     expect(result.status).toBe(1);
     expect(result.stderr).toContain("Hermes E2E swap provisioning failed");
-    expect(result.calls).toEqual([
-      ...provisioningFailureCalls,
-      "swapon-name-query:2:active",
-      `swapoff:${HERMES_E2E_SWAP_FILE}`,
-      `rm:-f -- ${HERMES_E2E_SWAP_FILE}`,
-    ]);
+    expect(result.calls.filter((call) => call === "sleep:1")).toHaveLength(4);
+    expect(swapoffIndex).toBeGreaterThan(-1);
+    expect(removeIndex).toBeGreaterThan(swapoffIndex);
   }, 15_000);
 
-  it("preserves the active fixed swap when cleanup swapoff fails", () => {
-    const result = runHermesSwapScriptFailure({ failSwapoff: true });
+  it("preserves the activated swap when visibility stays stale and cleanup swapoff fails", () => {
+    const result = runHermesSwapScriptFailure({
+      failSwapoff: true,
+      hiddenActivationReads: 5,
+    });
 
     expect(result.status).toBe(1);
     expect(result.stderr).toContain("Preserving active Hermes E2E swap after setup failure");
-    expect(result.calls).toEqual([
-      ...provisioningFailureCalls,
-      "swapon-name-query:2:active",
-      `swapoff:${HERMES_E2E_SWAP_FILE}`,
-    ]);
+    expect(result.calls.filter((call) => call === "sleep:1")).toHaveLength(4);
+    expect(result.calls).toContain(`swapoff:${HERMES_E2E_SWAP_FILE}`);
     expect(
       result.calls
         .slice(result.calls.indexOf(`swapon-activate:${HERMES_E2E_SWAP_FILE}`) + 1)
@@ -518,7 +563,7 @@ describe("HERMES_E2E_SWAP_SCRIPT failure cleanup (#7145)", () => {
     expect(result.stderr).toContain(
       "Preserving Hermes E2E swap because active swap could not be queried",
     );
-    expect(result.calls).toEqual([...provisioningFailureCalls, "swapon-name-query:2:fail"]);
+    expect(result.calls).toContain("swapon-name-query:7:fail");
     expect(
       result.calls
         .slice(result.calls.indexOf(`swapon-activate:${HERMES_E2E_SWAP_FILE}`) + 1)