diff --git a/test/e2e/README.md b/test/e2e/README.md index ccc5db72f3e..bca2f0e1c0f 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -67,7 +67,8 @@ Hermes image-building lanes that remain on those standard runners. The trusted workflow provisions the fallback as the first job step, before checking out or executing the candidate revision. It requires a controller-supplied lowercase 40-hex checkout SHA, matching trusted workflow and dispatch revisions, and an -ephemeral GitHub-hosted Linux x64 runner. +ephemeral GitHub-hosted Linux x64 runner. Candidate code cannot supply the +program or arguments passed to `sudo`. The trusted step requires at least 32 GiB (34,359,738,368 bytes) of usable swap. It reuses active swap that meets this requirement. @@ -79,23 +80,12 @@ The additional 4,096 bytes keep the usable swap capacity at or above 32 GiB after formatting. Setup failure stops before candidate checkout and removes partial state only after proving the file inactive or successfully disabling it. -After `swapon` succeeds, both rollout paths make up to five activation +After `swapon` succeeds, the trusted step makes up to five activation observations, one second apart. If visibility remains stale, cleanup treats the file as active. Cleanup removes it only after `swapoff` succeeds. Successful state is discarded with the ephemeral runner. -This rollout adds the trusted pre-checkout setup. -During rollout, the PR temporarily retains the reviewed live Vitest helper. -The helper exists only because the PR must validate against the older workflow -definition on `main` before this change lands. -The compatibility path runs only when GitHub Actions supplies a validated -lowercase 40-hex checkout SHA. -When the trusted step already provides 32 GiB of usable swap, the helper exits -before it creates its fixed swap file. -A follow-up must remove the candidate-side helper and its compatibility tests -after this change lands. - The fallback covers agent-turn latency, Hermes inference switch and shields, the Hermes Bedrock and stable MCP shards, and the `hermes-e2e`, `hermes-dashboard`, and Hermes security-posture tests. Scheduled and ordinary diff --git a/test/e2e/support/live-vitest-invocation.test.ts b/test/e2e/support/live-vitest-invocation.test.ts index 4708d69d9c3..ae51ec2f5a0 100644 --- a/test/e2e/support/live-vitest-invocation.test.ts +++ b/test/e2e/support/live-vitest-invocation.test.ts @@ -2,21 +2,14 @@ // SPDX-License-Identifier: Apache-2.0 import { spawnSync } from "node:child_process"; -import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; import { buildLiveVitestArgs, - HERMES_E2E_SWAP_BYTES, - HERMES_E2E_SWAP_FILE, - HERMES_E2E_SWAP_FILE_BYTES, - HERMES_E2E_SWAP_SCRIPT, LIVE_VITEST_PROJECT, type LiveVitestSpawner, - needsHermesE2ESwap, RISK_SIGNAL_REPORTER, runLiveVitestCommand, validateLiveProject, @@ -26,151 +19,6 @@ import { const LIVE_VITEST_TOOL = path.resolve("tools/e2e/live-vitest-invocation.mts"); const TSX = path.resolve("node_modules", ".bin", "tsx"); -const EXACT_HEAD_SHA = "a".repeat(40); - -interface FakeSwapScriptOptions { - failCleanupQuery?: boolean; - failSwapoff?: boolean; - hiddenActivationReads?: number; - provisionedSwapBytes?: number; -} - -interface FakeSwapScriptResult { - calls: string[]; - status: number | null; - stderr: string; -} - -function writeFakeCommand(directory: string, name: string, lines: string[]): void { - const commandPath = path.join(directory, name); - writeFileSync(commandPath, `${["#!/bin/sh", "set -eu", ...lines].join("\n")}\n`); - chmodSync(commandPath, 0o755); -} - -function runHermesSwapScriptFailure(options: FakeSwapScriptOptions = {}): FakeSwapScriptResult { - const fakeBin = mkdtempSync(path.join(tmpdir(), "nemoclaw-hermes-swap-")); - const callLog = path.join(fakeBin, "calls.log"); - const swapState = path.join(fakeBin, "swap-state"); - const nameQueryCount = path.join(fakeBin, "name-query-count"); - const activationQueryCount = path.join(fakeBin, "activation-query-count"); - writeFileSync(swapState, "inactive\n"); - - writeFakeCommand(fakeBin, "swapon", [ - `printf 'swapon:%s\\n' "$*" >> "$FAKE_CALL_LOG"`, - 'case "$*" in', - ' *"--show=SIZE"*)', - ' swap_state="inactive"', - ' if [ -f "$FAKE_SWAP_STATE_FILE" ]; then', - ' IFS= read -r swap_state < "$FAKE_SWAP_STATE_FILE" || swap_state="inactive"', - " fi", - ' if [ "$swap_state" = "active" ]; then', - " activation_count=0", - ' if [ -f "$FAKE_ACTIVATION_QUERY_COUNT_FILE" ]; then', - ' IFS= read -r activation_count < "$FAKE_ACTIVATION_QUERY_COUNT_FILE" || activation_count=0', - " fi", - ' if [ "$activation_count" -le "$FAKE_HIDDEN_ACTIVATION_READS" ]; then', - " printf '0\\n'", - " else", - ' printf "%s\\n" "$FAKE_PROVISIONED_SWAP_BYTES"', - " fi", - " else", - " printf '1\\n'", - " fi", - " ;;", - ' *"--show=NAME"*)', - " query_count=0", - ' if [ -f "$FAKE_NAME_QUERY_COUNT_FILE" ]; then', - ' IFS= read -r query_count < "$FAKE_NAME_QUERY_COUNT_FILE" || query_count=0', - " fi", - " query_count=$((query_count + 1))", - ` printf '%s\\n' "$query_count" > "$FAKE_NAME_QUERY_COUNT_FILE"`, - ' if [ "${FAKE_FAIL_NAME_QUERY_AT:-0}" -eq "$query_count" ]; then', - ` printf 'swapon-name-query:%s:fail\\n' "$query_count" >> "$FAKE_CALL_LOG"`, - " exit 41", - " fi", - ' swap_state="inactive"', - ' if [ -f "$FAKE_SWAP_STATE_FILE" ]; then', - ' IFS= read -r swap_state < "$FAKE_SWAP_STATE_FILE" || swap_state="inactive"', - " fi", - ` printf 'swapon-name-query:%s:%s\\n' "$query_count" "$swap_state" >> "$FAKE_CALL_LOG"`, - ' if [ "$swap_state" = "active" ]; then', - " activation_count=0", - ' if [ -f "$FAKE_ACTIVATION_QUERY_COUNT_FILE" ]; then', - ' IFS= read -r activation_count < "$FAKE_ACTIVATION_QUERY_COUNT_FILE" || activation_count=0', - " fi", - " activation_count=$((activation_count + 1))", - ' printf "%s\\n" "$activation_count" > "$FAKE_ACTIVATION_QUERY_COUNT_FILE"', - ' if [ "$activation_count" -gt "$FAKE_HIDDEN_ACTIVATION_READS" ]; then', - ` printf '%s\\n' "$FAKE_FIXED_SWAP"`, - " fi", - " fi", - " ;;", - ' "--show")', - " ;;", - ' *"--show"*)', - " exit 43", - " ;;", - " *)", - ` printf 'active\\n' > "$FAKE_SWAP_STATE_FILE"`, - ` printf 'swapon-activate:%s\\n' "$1" >> "$FAKE_CALL_LOG"`, - " ;;", - "esac", - ]); - writeFakeCommand(fakeBin, "awk", [ - "total=0", - "while IFS= read -r value; do total=$((total + value)); done", - `printf '%s\\n' "$total"`, - ]); - writeFakeCommand(fakeBin, "swapoff", [ - `printf 'swapoff:%s\\n' "$1" >> "$FAKE_CALL_LOG"`, - 'if [ "${FAKE_FAIL_SWAPOFF:-0}" = "1" ]; then', - " exit 42", - "fi", - `printf 'inactive\\n' > "$FAKE_SWAP_STATE_FILE"`, - ]); - writeFakeCommand(fakeBin, "rm", [`printf 'rm:%s\\n' "$*" >> "$FAKE_CALL_LOG"`]); - writeFakeCommand(fakeBin, "fallocate", [`printf 'fallocate:%s\\n' "$*" >> "$FAKE_CALL_LOG"`]); - writeFakeCommand(fakeBin, "chmod", [`printf 'chmod:%s\\n' "$*" >> "$FAKE_CALL_LOG"`]); - writeFakeCommand(fakeBin, "mkswap", [`printf 'mkswap:%s\\n' "$*" >> "$FAKE_CALL_LOG"`]); - writeFakeCommand(fakeBin, "sleep", [`printf 'sleep:%s\\n' "$*" >> "$FAKE_CALL_LOG"`]); - - try { - const result = spawnSync( - "/bin/bash", - [ - "--noprofile", - "--norc", - "-c", - HERMES_E2E_SWAP_SCRIPT, - "hermes-e2e-swap-test", - HERMES_E2E_SWAP_FILE, - String(HERMES_E2E_SWAP_BYTES), - String(HERMES_E2E_SWAP_FILE_BYTES), - ], - { - encoding: "utf8", - env: { - FAKE_CALL_LOG: callLog, - FAKE_ACTIVATION_QUERY_COUNT_FILE: activationQueryCount, - FAKE_FAIL_NAME_QUERY_AT: options.failCleanupQuery ? "7" : "0", - FAKE_FAIL_SWAPOFF: options.failSwapoff ? "1" : "0", - FAKE_FIXED_SWAP: HERMES_E2E_SWAP_FILE, - FAKE_HIDDEN_ACTIVATION_READS: String(options.hiddenActivationReads ?? 0), - FAKE_NAME_QUERY_COUNT_FILE: nameQueryCount, - FAKE_PROVISIONED_SWAP_BYTES: String(options.provisionedSwapBytes ?? 1), - FAKE_SWAP_STATE_FILE: swapState, - LC_ALL: "C", - PATH: fakeBin, - }, - }, - ); - expect(result.error).toBeUndefined(); - const calls = existsSync(callLog) ? readFileSync(callLog, "utf8").trimEnd().split("\n") : []; - return { calls, status: result.status, stderr: result.stderr }; - } finally { - rmSync(fakeBin, { force: true, recursive: true }); - } -} describe("validateLiveProject (#6961)", () => { it("accepts the live project and defaults to it", () => { @@ -397,289 +245,3 @@ describe("runLiveVitestCommand (#6961)", () => { expect(result.stderr).toContain('expected "run"'); }); }); - -describe("runLiveVitestCommand Hermes resource setup (#7145)", () => { - it("provisions bounded idempotent swap before a hosted Hermes build", () => { - const calls: Array> = []; - const spawn: LiveVitestSpawner = (...args) => { - calls.push(args); - return { status: 0 }; - }; - - expect( - runLiveVitestCommand( - ["run", "--test-path", "test/e2e/live/hermes-inference-switch.test.ts"], - spawn, - { - GITHUB_ACTIONS: "true", - NEMOCLAW_E2E_EXPECTED_SHA: EXACT_HEAD_SHA, - }, - ), - ).toBe(0); - expect(calls).toHaveLength(2); - expect(calls[0]?.[0]).toBe("/usr/bin/sudo"); - expect(HERMES_E2E_SWAP_BYTES).toBe(34_359_738_368); - expect(HERMES_E2E_SWAP_FILE_BYTES).toBe(34_359_742_464); - expect(HERMES_E2E_SWAP_FILE).toBe("/mnt/nemoclaw-hermes-e2e.swap"); - expect(calls[0]?.[1].slice(0, 9)).toEqual([ - "-n", - "/usr/bin/env", - "-i", - "PATH=/usr/sbin:/usr/bin:/sbin:/bin", - "LC_ALL=C", - "/bin/bash", - "--noprofile", - "--norc", - "-c", - ]); - expect(calls[0]?.[1].slice(10)).toEqual([ - "nemoclaw-hermes-e2e-swap", - HERMES_E2E_SWAP_FILE, - String(HERMES_E2E_SWAP_BYTES), - String(HERMES_E2E_SWAP_FILE_BYTES), - ]); - const script = calls[0]?.[1][9] ?? ""; - expect( - spawnSync("/bin/bash", ["--noprofile", "--norc", "-n"], { - input: script, - }).status, - ).toBe(0); - expect(script).toContain("if (( active_swap_bytes >= required_swap_bytes )); then"); - expect(script).toContain('active_swap_names="$(swapon --show=NAME --noheadings --raw)"'); - expect(script).toContain('if [[ "$active_swap_name" == "$swap_file" ]]; then'); - expect(script).toContain( - "if (( cleanup_swap_active == 1 || swap_activation_succeeded == 1 )); then", - ); - expect(script).toContain( - 'if cleanup_swap_names="$(swapon --show=NAME --noheadings --raw 2>/dev/null)"; then', - ); - expect(script).toContain("swapon --show=SIZE --bytes --noheadings"); - expect(script).not.toContain("swapon --output"); - expect(script).toContain('if swapoff "$swap_file" 2>/dev/null; then'); - expect(script).toContain("Preserving active Hermes E2E swap after setup failure"); - expect(script).toContain("Preserving Hermes E2E swap because active swap could not be queried"); - expect(script).not.toContain('swapoff "$swap_file" 2>/dev/null || true'); - expect(script).not.toContain("swap_enabled"); - expect(script).toContain('fallocate -l "$swap_file_bytes" "$swap_file"'); - expect(script).toContain("activation_observation_attempts=5"); - expect(script).toContain("activation_observation_delay_seconds=1"); - expect(script).toContain("if ! observe_provisioned_swap; then"); - expect(calls[1]?.[0]).toBe("npx"); - }); - - it("fails closed before Vitest when Hermes swap provisioning fails", () => { - const calls: string[] = []; - const spawn: LiveVitestSpawner = (command) => { - calls.push(command); - return { status: 23 }; - }; - - expect( - runLiveVitestCommand( - ["run", "--test-path", "test/e2e/live/hermes-shields-config.test.ts"], - spawn, - { - GITHUB_ACTIONS: "true", - NEMOCLAW_E2E_EXPECTED_SHA: EXACT_HEAD_SHA, - }, - ), - ).toBe(23); - expect(calls).toEqual(["/usr/bin/sudo"]); - }); - - it.each([ - ["scheduled main", undefined], - ["manual main without a checkout SHA", ""], - ["a malformed checkout SHA", "A".repeat(40)], - ])("does not provision swap for %s", (_label, expectedSha) => { - const calls: string[] = []; - const spawn: LiveVitestSpawner = (command) => { - calls.push(command); - return { status: 0 }; - }; - - expect( - runLiveVitestCommand( - ["run", "--test-path", "test/e2e/live/hermes-inference-switch.test.ts"], - spawn, - { - GITHUB_ACTIONS: "true", - NEMOCLAW_E2E_EXPECTED_SHA: expectedSha, - }, - ), - ).toBe(0); - expect(calls).toEqual(["npx"]); - }); -}); - -describe("HERMES_E2E_SWAP_SCRIPT failure cleanup (#7145)", () => { - it("waits for delayed activation visibility before accepting the swap", () => { - const result = runHermesSwapScriptFailure({ - hiddenActivationReads: 2, - provisionedSwapBytes: HERMES_E2E_SWAP_BYTES, - }); - - expect(result.status).toBe(0); - expect(result.calls.filter((call) => call === "sleep:1")).toHaveLength(2); - expect( - result.calls.filter((call) => call === `swapon-activate:${HERMES_E2E_SWAP_FILE}`), - ).toHaveLength(1); - expect(result.calls.filter((call) => call.startsWith("swapoff:"))).toEqual([]); - }); - - it("removes the active fixed swap only after cleanup swapoff succeeds", () => { - const result = runHermesSwapScriptFailure(); - const swapoffIndex = result.calls.indexOf(`swapoff:${HERMES_E2E_SWAP_FILE}`); - const removeIndex = result.calls.lastIndexOf(`rm:-f -- ${HERMES_E2E_SWAP_FILE}`); - - expect(result.status).toBe(1); - expect(result.stderr).toContain("Hermes E2E swap provisioning failed"); - expect(result.calls.filter((call) => call === "sleep:1")).toHaveLength(4); - expect(swapoffIndex).toBeGreaterThan(-1); - expect(removeIndex).toBeGreaterThan(swapoffIndex); - }, 15_000); - - it("preserves the activated swap when visibility stays stale and cleanup swapoff fails", () => { - const result = runHermesSwapScriptFailure({ - failSwapoff: true, - hiddenActivationReads: 5, - }); - - expect(result.status).toBe(1); - expect(result.stderr).toContain("Preserving active Hermes E2E swap after setup failure"); - expect(result.calls.filter((call) => call === "sleep:1")).toHaveLength(4); - expect(result.calls).toContain(`swapoff:${HERMES_E2E_SWAP_FILE}`); - expect( - result.calls - .slice(result.calls.indexOf(`swapon-activate:${HERMES_E2E_SWAP_FILE}`) + 1) - .filter((call) => call.startsWith("rm:")), - ).toEqual([]); - }, 15_000); - - it("preserves the fixed swap when cleanup cannot query active swap", () => { - const result = runHermesSwapScriptFailure({ failCleanupQuery: true }); - - expect(result.status).toBe(1); - expect(result.stderr).toContain( - "Preserving Hermes E2E swap because active swap could not be queried", - ); - expect(result.calls).toContain("swapon-name-query:7:fail"); - expect( - result.calls - .slice(result.calls.indexOf(`swapon-activate:${HERMES_E2E_SWAP_FILE}`) + 1) - .filter((call) => call.startsWith("rm:")), - ).toEqual([]); - }, 15_000); -}); - -describe("needsHermesE2ESwap (#7145)", () => { - const exactHeadEnv = { - GITHUB_ACTIONS: "true", - NEMOCLAW_E2E_EXPECTED_SHA: EXACT_HEAD_SHA, - }; - - it.each([ - ["Hermes inference switch", "test/e2e/live/hermes-inference-switch.test.ts", {}], - ["Hermes shields", "test/e2e/live/hermes-shields-config.test.ts", {}], - ["agent turn latency", "test/e2e/live/agent-turn-latency.test.ts", {}], - [ - "Hermes Bedrock", - "test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts", - { NEMOCLAW_AGENT: "hermes" }, - ], - [ - "Hermes E2E", - "test/e2e/live/hermes-e2e.test.ts", - { E2E_TARGET_ID: "hermes-e2e", NEMOCLAW_AGENT: "hermes" }, - ], - [ - "Hermes dashboard", - "test/e2e/live/hermes-e2e.test.ts", - { E2E_TARGET_ID: "hermes-dashboard", NEMOCLAW_AGENT: "hermes" }, - ], - [ - "Hermes security posture", - "test/e2e/live/hermes-e2e.test.ts", - { E2E_TARGET_ID: "security-posture", NEMOCLAW_AGENT: "hermes" }, - ], - [ - "Hermes MCP", - "test/e2e/live/mcp-bridge.test.ts", - { E2E_TARGET_ID: "mcp-bridge", NEMOCLAW_MCP_BRIDGE_AGENT: "hermes" }, - ], - ])("selects the exact-head hosted %s build", (_label, testPath, env) => { - expect(needsHermesE2ESwap(testPath, { ...exactHeadEnv, ...env })).toBe(true); - }); - - it.each([ - [ - "a non-GitHub runner", - "test/e2e/live/hermes-inference-switch.test.ts", - { GITHUB_ACTIONS: "" }, - ], - [ - "a scheduled main run", - "test/e2e/live/hermes-inference-switch.test.ts", - { NEMOCLAW_E2E_EXPECTED_SHA: undefined }, - ], - [ - "a manual main run without a checkout SHA", - "test/e2e/live/hermes-inference-switch.test.ts", - { NEMOCLAW_E2E_EXPECTED_SHA: "" }, - ], - [ - "an uppercase checkout SHA", - "test/e2e/live/hermes-inference-switch.test.ts", - { NEMOCLAW_E2E_EXPECTED_SHA: "A".repeat(40) }, - ], - [ - "the OpenClaw Bedrock shard", - "test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts", - { GITHUB_ACTIONS: "true", NEMOCLAW_AGENT: "openclaw" }, - ], - [ - "the OpenClaw security posture shard", - "test/e2e/live/hermes-e2e.test.ts", - { - E2E_TARGET_ID: "security-posture", - GITHUB_ACTIONS: "true", - NEMOCLAW_AGENT: "openclaw", - }, - ], - [ - "an unlisted Hermes target sharing the test file", - "test/e2e/live/hermes-e2e.test.ts", - { E2E_TARGET_ID: "future-hermes-job", NEMOCLAW_AGENT: "hermes" }, - ], - [ - "a non-Hermes MCP shard", - "test/e2e/live/mcp-bridge.test.ts", - { - E2E_TARGET_ID: "mcp-bridge", - GITHUB_ACTIONS: "true", - NEMOCLAW_MCP_BRIDGE_AGENT: "deepagents", - }, - ], - [ - "the explicit-only MCP dev lane", - "test/e2e/live/mcp-bridge.test.ts", - { - E2E_TARGET_ID: "mcp-bridge-dev", - GITHUB_ACTIONS: "true", - NEMOCLAW_MCP_BRIDGE_AGENT: "hermes", - }, - ], - [ - "a rebuild lane with workflow-managed swap", - "test/e2e/live/rebuild-hermes.test.ts", - { GITHUB_ACTIONS: "true", NEMOCLAW_AGENT: "hermes" }, - ], - [ - "a self-hosted Hermes lane", - "test/e2e/live/hermes-slack-e2e.test.ts", - { GITHUB_ACTIONS: "true", NEMOCLAW_AGENT: "hermes" }, - ], - ])("does not select %s", (_label, testPath, env) => { - expect(needsHermesE2ESwap(testPath, { ...exactHeadEnv, ...env })).toBe(false); - }); -}); diff --git a/test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts b/test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts index 9b6af84d15a..e5240a20385 100644 --- a/test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts +++ b/test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts @@ -11,6 +11,7 @@ import { describe, expect, it } from "vitest"; import { TRUSTED_HERMES_SWAP_SCRIPT, TRUSTED_HERMES_SWAP_STEP_NAME, + validateTrustedHermesSwapHelperSource, validateTrustedHermesSwapWorkflow, } from "../../../tools/e2e/trusted-hermes-swap-workflow-boundary.mts"; import { readWorkflow } from "../../helpers/e2e-workflow-contract"; @@ -418,4 +419,17 @@ describe("trusted Hermes swap workflow boundary", () => { ]), ); }); + + it("rejects a candidate-side sudo payload without changing the trusted pre-checkout command (#7145)", () => { + const workflow = readWorkflow() as SwapWorkflow; + const helperPath = path.resolve("tools/e2e/live-vitest-invocation.mts"); + const maliciousCandidateHelper = `${readFileSync(helperPath, "utf8")} +void spawnSync("/usr/bin/sudo", ["-n", "/bin/bash", "-c", "id"]); +`; + + expect(validateTrustedHermesSwapHelperSource(maliciousCandidateHelper)).toContain( + "candidate live Vitest helper must not contain privileged swap fragment /usr/bin/sudo", + ); + expect(validateTrustedHermesSwapWorkflow(workflow)).toEqual([]); + }); }); diff --git a/tools/e2e/live-vitest-invocation.mts b/tools/e2e/live-vitest-invocation.mts index 1193d5bea30..8fd72a6e871 100644 --- a/tools/e2e/live-vitest-invocation.mts +++ b/tools/e2e/live-vitest-invocation.mts @@ -20,144 +20,9 @@ const { spawnExitCode } = processExit; export const LIVE_VITEST_PROJECT = "e2e-live"; export const LIVE_TEST_ROOT = "test/e2e/live/"; export const RISK_SIGNAL_REPORTER = "test/e2e/risk-signal-reporter.ts"; -// Keep this exact-head helper during the trusted workflow's first rollout -// phase. Once the pre-checkout workflow step is on main, it provisions enough -// swap for this helper to return before candidate-side privileged mutation. -export const HERMES_E2E_SWAP_BYTES = 32 * 1024 * 1024 * 1024; -export const HERMES_E2E_SWAP_FILE_BYTES = HERMES_E2E_SWAP_BYTES + 4096; -export const HERMES_E2E_SWAP_FILE = "/mnt/nemoclaw-hermes-e2e.swap"; const SHELL_METACHARACTER = /[^A-Za-z0-9_./^$=:@+-]/u; const TEST_PATH_PATTERN = /^[A-Za-z0-9_./-]+$/u; -const EXACT_HEAD_SHA_PATTERN = /^[0-9a-f]{40}$/u; -const ALWAYS_HERMES_BUILD_TESTS = new Set([ - "test/e2e/live/agent-turn-latency.test.ts", - "test/e2e/live/hermes-inference-switch.test.ts", - "test/e2e/live/hermes-shields-config.test.ts", -]); -const HERMES_BEDROCK_BUILD_TEST = "test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts"; -const HERMES_SHARED_E2E_TEST = "test/e2e/live/hermes-e2e.test.ts"; -const HERMES_SHARED_E2E_TARGETS = new Set(["hermes-dashboard", "hermes-e2e", "security-posture"]); -const HERMES_MCP_BUILD_TEST = "test/e2e/live/mcp-bridge.test.ts"; -export const HERMES_E2E_SWAP_SCRIPT = `set -euo pipefail -swap_file="$1" -required_swap_bytes="$2" -swap_file_bytes="$3" -activation_observation_attempts=5 -activation_observation_delay_seconds=1 -swap_activation_succeeded=0 - -case "$required_swap_bytes" in - ""|*[!0-9]*) - echo "Hermes E2E required swap size must be an integer byte count" >&2 - exit 2 - ;; -esac -case "$swap_file_bytes" in - ""|*[!0-9]*) - echo "Hermes E2E swap file size must be an integer byte count" >&2 - exit 2 - ;; -esac - -active_swap_bytes="$(swapon --show=SIZE --bytes --noheadings | awk '{ total += $1 } END { printf "%.0f", total }')" -active_swap_bytes="\${active_swap_bytes:-0}" -case "$active_swap_bytes" in - ""|*[!0-9]*) - echo "Unable to determine active swap capacity" >&2 - exit 2 - ;; -esac - -if (( active_swap_bytes >= required_swap_bytes )); then - printf 'Hermes E2E swap is already sufficient: %s bytes active\\n' "$active_swap_bytes" - exit 0 -fi - -active_swap_names="$(swapon --show=NAME --noheadings --raw)" -fixed_swap_active=0 -while IFS= read -r active_swap_name; do - if [[ "$active_swap_name" == "$swap_file" ]]; then - fixed_swap_active=1 - break - fi -done <<< "$active_swap_names" -if (( fixed_swap_active == 1 )); then - swapoff "$swap_file" -fi -rm -f -- "$swap_file" - -cleanup_partial_swap() { - status="$?" - if (( status != 0 )); then - if cleanup_swap_names="$(swapon --show=NAME --noheadings --raw 2>/dev/null)"; then - cleanup_swap_active=0 - while IFS= read -r cleanup_swap_name; do - if [[ "$cleanup_swap_name" == "$swap_file" ]]; then - cleanup_swap_active=1 - break - fi - done <<< "$cleanup_swap_names" - if (( cleanup_swap_active == 1 || swap_activation_succeeded == 1 )); then - if swapoff "$swap_file" 2>/dev/null; then - rm -f -- "$swap_file" || true - else - printf 'Preserving active Hermes E2E swap after setup failure: %s\\n' "$swap_file" >&2 - fi - else - rm -f -- "$swap_file" || true - fi - else - printf 'Preserving Hermes E2E swap because active swap could not be queried: %s\\n' "$swap_file" >&2 - fi - fi - trap - EXIT - exit "$status" -} -trap cleanup_partial_swap EXIT - -fallocate -l "$swap_file_bytes" "$swap_file" -chmod 0600 "$swap_file" -mkswap "$swap_file" -swapon "$swap_file" -swap_activation_succeeded=1 - -observe_provisioned_swap() { - activation_observation_attempt=1 - while (( activation_observation_attempt <= activation_observation_attempts )); do - provisioned_swap_active=0 - if active_swap_names="$(swapon --show=NAME --noheadings --raw 2>/dev/null)"; then - while IFS= read -r active_swap_name; do - if [[ "$active_swap_name" == "$swap_file" ]]; then - provisioned_swap_active=1 - break - fi - done <<< "$active_swap_names" - fi - if observed_swap_bytes="$(swapon --show=SIZE --bytes --noheadings 2>/dev/null | awk '{ total += $1 } END { printf "%.0f", total }')"; then - observed_swap_bytes="\${observed_swap_bytes:-0}" - if [[ "$observed_swap_bytes" != *[!0-9]* ]] && - (( provisioned_swap_active == 1 && observed_swap_bytes >= required_swap_bytes )); then - active_swap_bytes="$observed_swap_bytes" - return 0 - fi - fi - if (( activation_observation_attempt < activation_observation_attempts )); then - sleep "$activation_observation_delay_seconds" - fi - activation_observation_attempt=$((activation_observation_attempt + 1)) - done - return 1 -} - -if ! observe_provisioned_swap; then - printf 'Hermes E2E swap provisioning failed: required swap was not visible after %s attempts\\n' "$activation_observation_attempts" >&2 - exit 1 -fi - -trap - EXIT -printf 'Hermes E2E swap ready: %s bytes active\\n' "$active_swap_bytes" -swapon --show`; export interface LiveVitestInvocation { testPath: string | undefined; @@ -273,88 +138,26 @@ export function buildLiveVitestArgs(invocation: LiveVitestInvocation): string[] ]; } -export function needsHermesE2ESwap(testPath: string, env: NodeJS.ProcessEnv): boolean { - if ( - env.GITHUB_ACTIONS !== "true" || - !EXACT_HEAD_SHA_PATTERN.test(env.NEMOCLAW_E2E_EXPECTED_SHA ?? "") - ) { - return false; - } - if (ALWAYS_HERMES_BUILD_TESTS.has(testPath)) return true; - if (testPath === HERMES_BEDROCK_BUILD_TEST) return env.NEMOCLAW_AGENT === "hermes"; - if (testPath === HERMES_SHARED_E2E_TEST) { - return ( - env.NEMOCLAW_AGENT === "hermes" && HERMES_SHARED_E2E_TARGETS.has(env.E2E_TARGET_ID ?? "") - ); - } - return ( - testPath === HERMES_MCP_BUILD_TEST && - env.E2E_TARGET_ID === "mcp-bridge" && - env.NEMOCLAW_MCP_BRIDGE_AGENT === "hermes" - ); -} - function spawnResultExitCode(result: LiveVitestSpawnResult): number { if (result.error) throw result.error; return spawnExitCode(result); } -export function provisionHermesE2ESwap( - testPath: string, - env: NodeJS.ProcessEnv, - spawn: LiveVitestSpawner = spawnSync, -): number { - if (!needsHermesE2ESwap(testPath, env)) return 0; - - return spawnResultExitCode( - spawn( - "/usr/bin/sudo", - [ - "-n", - "/usr/bin/env", - "-i", - "PATH=/usr/sbin:/usr/bin:/sbin:/bin", - "LC_ALL=C", - "/bin/bash", - "--noprofile", - "--norc", - "-c", - HERMES_E2E_SWAP_SCRIPT, - "nemoclaw-hermes-e2e-swap", - HERMES_E2E_SWAP_FILE, - String(HERMES_E2E_SWAP_BYTES), - String(HERMES_E2E_SWAP_FILE_BYTES), - ], - { stdio: "inherit" }, - ), - ); -} - -export function runLiveVitestCli( - cliArgs: string[], - spawn: LiveVitestSpawner = spawnSync, - env: NodeJS.ProcessEnv = process.env, -): number { +export function runLiveVitestCli(cliArgs: string[], spawn: LiveVitestSpawner = spawnSync): number { const invocation = parseLiveVitestArgs(cliArgs); const testPath = validateLiveTestPath(invocation.testPath); const argv = buildLiveVitestArgs({ ...invocation, testPath }); - const swapExitCode = provisionHermesE2ESwap(testPath, env, spawn); - if (swapExitCode !== 0) return swapExitCode; return spawnResultExitCode(spawn("npx", argv, { stdio: "inherit" })); } -export function runLiveVitestCommand( - argv: string[], - spawn: LiveVitestSpawner = spawnSync, - env: NodeJS.ProcessEnv = process.env, -): number { +export function runLiveVitestCommand(argv: string[], spawn: LiveVitestSpawner = spawnSync): number { const [command, ...cliArgs] = argv; if (command !== "run") { throw new Error( `unsupported live Vitest command ${JSON.stringify(command ?? "")}; expected "run"`, ); } - return runLiveVitestCli(cliArgs, spawn, env); + return runLiveVitestCli(cliArgs, spawn); } if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { diff --git a/tools/e2e/trusted-hermes-swap-workflow-boundary.mts b/tools/e2e/trusted-hermes-swap-workflow-boundary.mts index 69ea06efe4e..954e49c1b01 100644 --- a/tools/e2e/trusted-hermes-swap-workflow-boundary.mts +++ b/tools/e2e/trusted-hermes-swap-workflow-boundary.mts @@ -289,3 +289,21 @@ export function validateTrustedHermesSwapWorkflow(workflowValue: unknown): strin return errors; } + +export function validateTrustedHermesSwapHelperSource(source: string): string[] { + const errors: string[] = []; + const forbidden = [ + "/usr/bin/sudo", + "HERMES_E2E_SWAP_SCRIPT", + "provisionHermesE2ESwap", + "needsHermesE2ESwap", + ]; + for (const fragment of forbidden) { + if (source.includes(fragment)) { + errors.push( + `candidate live Vitest helper must not contain privileged swap fragment ${fragment}`, + ); + } + } + return errors; +} diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index 57732a03570..c383a0de2a5 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -39,11 +39,20 @@ import { validateRunnerComparisonWorkflowBoundary } from "./runner-comparison-wo import { validateRunnerPressureWorkflow } from "./runner-pressure-workflow-boundary.mts"; import { validateSandboxOperationsWorkflow } from "./sandbox-operations-workflow-boundary.mts"; import { validateSecurityPostureWorkflow } from "./security-posture-workflow-boundary.mts"; -import { validateTrustedHermesSwapWorkflow } from "./trusted-hermes-swap-workflow-boundary.mts"; +import { + validateTrustedHermesSwapHelperSource, + validateTrustedHermesSwapWorkflow, +} from "./trusted-hermes-swap-workflow-boundary.mts"; import { validateUploadE2eArtifactsWorkflowBoundary } from "./upload-e2e-artifacts-workflow-boundary.mts"; const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); const DEFAULT_E2E_WORKFLOW_PATH = join(REPO_ROOT, ".github", "workflows", "e2e.yaml"); +const DEFAULT_LIVE_VITEST_INVOCATION_PATH = join( + REPO_ROOT, + "tools", + "e2e", + "live-vitest-invocation.mts", +); const DEFAULT_DOCKER_HUB_AUTH_ACTION_PATH = join( REPO_ROOT, ".github", @@ -4798,5 +4807,8 @@ export function validateE2eWorkflowBoundary(workflowPath = DEFAULT_E2E_WORKFLOW_ ...validateDockerHubAuthAction(), ...validateHostDependencyAction(), ...validateE2eWorkflow(readWorkflowRecord(workflowPath)), + ...validateTrustedHermesSwapHelperSource( + readFileSync(DEFAULT_LIVE_VITEST_INVOCATION_PATH, "utf8"), + ), ]; }