diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 530874a81b2..ae4d2c5cf04 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -144,6 +144,7 @@ jobs: test_matrix: ${{ steps.matrix.outputs.test_matrix }} hermes_selected: ${{ steps.matrix.outputs.hermes_selected }} explicit_only_jobs: ${{ steps.matrix.outputs.explicit_only_jobs }} + runner_routing: ${{ steps.runner_routing.outputs.runner_routing }} steps: - id: controller_matrix name: Build trusted controller target matrix @@ -167,6 +168,27 @@ jobs: esac printf 'matrix=%s\n' "${matrix}" >> "${GITHUB_OUTPUT}" + - id: runner_routing + name: Build trusted larger-runner routing + env: + CHECKOUT_SHA: ${{ inputs.checkout_sha }} + LARGER_RUNNER_LABEL: ${{ vars.E2E_LARGER_RUNNER_LABEL }} + REF: ${{ github.ref }} + REPOSITORY: ${{ github.repository }} + shell: bash + run: | + set -euo pipefail + larger_runner="ubuntu-latest" + if [[ "${REPOSITORY}" == "NVIDIA/NemoClaw" && "${REF}" == "refs/heads/main" && -z "${CHECKOUT_SHA}" && -n "${LARGER_RUNNER_LABEL}" ]]; then + if [[ ! "${LARGER_RUNNER_LABEL}" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$ ]]; then + echo "::error::E2E_LARGER_RUNNER_LABEL must be a 1-64 character workflow label using letters, digits, dots, underscores, or hyphens" >&2 + exit 1 + fi + larger_runner="${LARGER_RUNNER_LABEL}" + fi + runner_routing="$(jq -cn --arg standard "ubuntu-latest" --arg larger "${larger_runner}" '{"channels-stop-start-hermes":$larger,"channels-stop-start-openclaw":$standard,"common-egress-agent":$larger,"hermes-dashboard":$larger,"hermes-discord":$larger,"hermes-e2e":$larger,"hermes-inference-switch":$larger,"hermes-shields-config":$larger,"mcp-bridge-deepagents":$larger,"mcp-bridge-hermes":$larger,"mcp-bridge-openclaw":$standard,"rebuild-hermes":$larger,"rebuild-hermes-stale-base":$larger,"security-posture-hermes":$larger,"security-posture-openclaw":$standard}')" + printf 'runner_routing=%s\n' "${runner_routing}" >> "${GITHUB_OUTPUT}" + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: ref: ${{ inputs.checkout_sha || github.sha }} @@ -533,7 +555,7 @@ jobs: mcp-bridge: needs: generate-matrix if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',mcp-bridge,') || contains(format(',{0},', inputs.targets), ',mcp-bridge,') }} - runs-on: ubuntu-latest + runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)[format('mcp-bridge-{0}', matrix.agent)] }} permissions: contents: read # Keep each destructive agent lifecycle on a fresh runner. This bounds the @@ -1130,7 +1152,7 @@ jobs: hermes-inference-switch: needs: generate-matrix if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',hermes-inference-switch,') || contains(format(',{0},', inputs.targets), ',hermes-inference-switch,') }} - runs-on: ubuntu-latest + runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['hermes-inference-switch'] }} permissions: contents: read timeout-minutes: 55 @@ -1629,7 +1651,7 @@ jobs: hermes-e2e: needs: generate-matrix if: ${{ needs.generate-matrix.outputs.hermes_selected == 'true' }} - runs-on: ubuntu-latest + runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['hermes-e2e'] }} timeout-minutes: 75 env: E2E_JOB: "1" @@ -1936,7 +1958,7 @@ jobs: hermes-dashboard: needs: generate-matrix if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',hermes-dashboard,') || contains(format(',{0},', inputs.targets), ',hermes-dashboard,') }} - runs-on: ubuntu-latest + runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['hermes-dashboard'] }} timeout-minutes: 75 env: E2E_JOB: "1" @@ -1982,7 +2004,7 @@ jobs: hermes-discord: needs: generate-matrix if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',hermes-discord,') || contains(format(',{0},', inputs.targets), ',hermes-discord,') }} - runs-on: ubuntu-latest + runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['hermes-discord'] }} timeout-minutes: 75 env: E2E_JOB: "1" @@ -2096,7 +2118,7 @@ jobs: common-egress-agent: needs: generate-matrix if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',common-egress-agent,') || contains(format(',{0},', inputs.targets), ',common-egress-agent,') }} - runs-on: ubuntu-latest + runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['common-egress-agent'] }} timeout-minutes: 120 env: E2E_JOB: "1" @@ -2216,7 +2238,7 @@ jobs: hermes-shields-config: needs: generate-matrix if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',hermes-shields-config,') || contains(format(',{0},', inputs.targets), ',hermes-shields-config,') }} - runs-on: ubuntu-latest + runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['hermes-shields-config'] }} timeout-minutes: 45 env: E2E_JOB: "1" @@ -2318,7 +2340,7 @@ jobs: rebuild-hermes: needs: generate-matrix if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',rebuild-hermes,') || contains(format(',{0},', inputs.targets), ',rebuild-hermes,') }} - runs-on: ubuntu-latest + runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['rebuild-hermes'] }} timeout-minutes: 90 env: E2E_JOB: "1" @@ -2419,7 +2441,7 @@ jobs: rebuild-hermes-stale-base: needs: generate-matrix if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',rebuild-hermes-stale-base,') || contains(format(',{0},', inputs.targets), ',rebuild-hermes-stale-base,') }} - runs-on: ubuntu-latest + runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['rebuild-hermes-stale-base'] }} timeout-minutes: 90 env: E2E_JOB: "1" @@ -3108,7 +3130,7 @@ jobs: contents: read needs: generate-matrix if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',security-posture,') || contains(format(',{0},', inputs.targets), ',security-posture,') }} - runs-on: ubuntu-latest + runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)[format('security-posture-{0}', matrix.agent)] }} timeout-minutes: 75 strategy: fail-fast: false @@ -4585,7 +4607,7 @@ jobs: channels-stop-start: needs: generate-matrix if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',channels-stop-start,') || contains(format(',{0},', inputs.targets), ',channels-stop-start,') }} - runs-on: ubuntu-latest + runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)[format('channels-stop-start-{0}', matrix.agent)] }} timeout-minutes: 90 strategy: fail-fast: false diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index 02e52ccc642..f58d2aab64d 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -256,6 +256,21 @@ "test": "rejects free-standing E2E artifact uploads from raw temp paths", "category": "security" }, + { + "file": "test/e2e/support/larger-runner-routing-workflow-boundary.test.ts", + "test": "keeps every candidate on standard runners when $name (#7145)", + "category": "security" + }, + { + "file": "test/e2e/support/larger-runner-routing-workflow-boundary.test.ts", + "test": "rejects malformed administrator workflow labels (#7145)", + "category": "security" + }, + { + "file": "test/e2e/support/larger-runner-routing-workflow-boundary.test.ts", + "test": "routes only the measured heavy lanes on trusted main (#7145)", + "category": "security" + }, { "file": "test/fetch-guard-patch-regression.test.ts", "test": "requires classifier review and integrity evidence when the OpenClaw build pin changes", diff --git a/test/e2e/README.md b/test/e2e/README.md index 97210e223a1..3a7eaf56a7d 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -51,6 +51,72 @@ discovery command locally to inspect the generated test matrix: npx tsx tools/e2e/credential-free-tests.mts ``` +## Larger-runner routing + +The larger-runner experiment is inactive while the configuration variable +`E2E_LARGER_RUNNER_LABEL` is unset. In that state, every eligible lane continues +to use `ubuntu-latest`. The trusted `generate-matrix` job builds one runner map +before checking out test code, and it consumes the variable only when the +workflow repository is `NVIDIA/NemoClaw`, the ref is `refs/heads/main`, and +no alternate checkout SHA is requested. PR-gate dispatches therefore remain on +standard runners even though they use the trusted workflow definition from +`main`. + +Exact-head PR-gate dispatches use a bounded swap fallback for the hosted +Hermes image-building lanes that remain on those standard runners. The live +Vitest helper activates the fallback only when GitHub Actions supplies a +validated lowercase 40-hex checkout SHA. It reuses at least 32 GiB of active +swap when available; otherwise, it creates one fixed 32 GiB swap file under +`/mnt` before agent-turn latency, Hermes inference switch and shields, the +Hermes Bedrock and stable MCP shards, or the `hermes-e2e`, `hermes-dashboard`, +and Hermes security-posture tests. Setup failure stops before Vitest. Scheduled +and ordinary manual `main` runs, larger-runner executions, rebuild lanes with +workflow-managed swap, dedicated-runner lanes, `mcp-bridge-dev`, and non-Hermes +shards do not use this fallback. + +The fallback exists because the alternate-checkout trust boundary deliberately +keeps PR-authored code from selecting the administrator-managed larger-runner +label; changing the PR checkout cannot safely grant itself that capacity. +Remove the fallback only after the trusted controller routes exact-head PR +gates to an ephemeral GitHub-hosted runner with at least 32 GB RAM without +weakening the exact-SHA guard, and five consecutive runs of every protected +lane complete without runner loss while runner-pressure telemetry reports less +than 1 GiB of swap used. + +The eligible set is limited to the measured or repeatedly interrupted heavy +lanes: + +- `common-egress-agent`; +- `hermes-e2e`, `hermes-dashboard`, and `hermes-discord`; +- both `hermes-inference-switch` modes; +- `hermes-shields-config`; +- the Hermes shards of `security-posture` and `channels-stop-start`; +- `rebuild-hermes`; +- `rebuild-hermes-stale-base`; +- the `hermes` and `deepagents` shards of `mcp-bridge`. + +The OpenClaw shards of the matrix jobs, the `openclaw` MCP shard, and +`mcp-bridge-dev` remain on `ubuntu-latest`; unrelated jobs retain their +existing runner assignments. Before setting the variable, an organization +owner must: + +1. Create a GitHub-hosted Ubuntu x64 larger runner with 8 vCPU, 32 GB RAM, and + 300 GB SSD in a dedicated runner group. +2. Set the group maximum concurrency to 4 and restrict repository access to + `NVIDIA/NemoClaw` and workflow access to + `NVIDIA/NemoClaw/.github/workflows/e2e.yaml@refs/heads/main`. +3. Record at least five standard-runner samples for each eligible lane, + including queue time, execution time, peak CPU, memory and disk use, + infrastructure failures, and estimated cost. +4. Copy the larger runner's workflow label into the repository variable, then + repeat the same measurements for at least five representative executions + per migrated lane. + +Clearing `E2E_LARGER_RUNNER_LABEL` is the rollback. It sends the eligible lanes +back to `ubuntu-latest` without changing selectors, test setup, or test +semantics. Do not replace this experiment with a persistent self-hosted runner; +that requires a separate decision. + ## Scheduled operations The consolidated workflow keeps its operational reporting in the same job diff --git a/test/e2e/support/hermes-dashboard-workflow-boundary.test.ts b/test/e2e/support/hermes-dashboard-workflow-boundary.test.ts index c5410ebe453..236ecc03f84 100644 --- a/test/e2e/support/hermes-dashboard-workflow-boundary.test.ts +++ b/test/e2e/support/hermes-dashboard-workflow-boundary.test.ts @@ -13,7 +13,6 @@ describe("Hermes dashboard workflow boundary", () => { expect(validateHermesDashboardWorkflowBoundary()).toEqual([]); const dashboardMode = readHermesDashboardWorkflow(); const dashboardJob = dashboardMode.jobs["hermes-dashboard"]; - dashboardJob["runs-on"] = "self-hosted"; dashboardJob["timeout-minutes"] = 30; dashboardJob.env!.E2E_ARTIFACT_DIR = "/tmp/hermes-dashboard"; dashboardJob.env!.NEMOCLAW_E2E_HERMES_DASHBOARD = "0"; @@ -25,7 +24,6 @@ describe("Hermes dashboard workflow boundary", () => { checkout.with!["persist-credentials"] = true; expect(validateHermesDashboardWorkflow(dashboardMode)).toEqual( expect.arrayContaining([ - "hermes-dashboard must run on ubuntu-latest", "hermes-dashboard timeout must be 75 minutes", "hermes-dashboard must use its isolated artifact directory", "hermes-dashboard must enable Hermes dashboard coverage", diff --git a/test/e2e/support/larger-runner-routing-workflow-boundary.test.ts b/test/e2e/support/larger-runner-routing-workflow-boundary.test.ts new file mode 100644 index 00000000000..fa625230bdf --- /dev/null +++ b/test/e2e/support/larger-runner-routing-workflow-boundary.test.ts @@ -0,0 +1,215 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { validateE2eWorkflow } from "../../../tools/e2e/workflow-boundary.mts"; +import { readWorkflow } from "../../helpers/e2e-workflow-contract"; +import { requireFixture } from "./require-fixture"; + +type WorkflowStep = { + env?: Record; + id?: string; + name?: string; + run?: string; + uses?: string; +}; + +type RoutingWorkflow = { + jobs: Record< + string, + { + outputs?: Record; + steps?: WorkflowStep[]; + "runs-on"?: string; + } + >; +}; + +function routingStep(workflow: RoutingWorkflow): WorkflowStep { + const step = workflow.jobs["generate-matrix"]?.steps?.find( + (candidate) => candidate.id === "runner_routing", + ); + requireFixture(step?.run, "trusted larger-runner routing step is missing"); + return step; +} + +function evaluateRouting( + workflow: RoutingWorkflow, + env: { checkoutSha?: string; label: string; ref: string; repository: string }, +): Record { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-runner-routing-")); + const outputPath = path.join(directory, "github-output"); + try { + execFileSync("bash", ["-c", routingStep(workflow).run!], { + env: { + ...process.env, + CHECKOUT_SHA: env.checkoutSha ?? "", + GITHUB_OUTPUT: outputPath, + LARGER_RUNNER_LABEL: env.label, + REF: env.ref, + REPOSITORY: env.repository, + }, + }); + const output = fs.readFileSync(outputPath, "utf8").trim(); + requireFixture(output.startsWith("runner_routing="), "runner routing output is missing"); + return JSON.parse(output.slice("runner_routing=".length)) as Record; + } finally { + fs.rmSync(directory, { force: true, recursive: true }); + } +} + +const standardRouting = { + "channels-stop-start-hermes": "ubuntu-latest", + "channels-stop-start-openclaw": "ubuntu-latest", + "common-egress-agent": "ubuntu-latest", + "hermes-dashboard": "ubuntu-latest", + "hermes-discord": "ubuntu-latest", + "hermes-e2e": "ubuntu-latest", + "hermes-inference-switch": "ubuntu-latest", + "hermes-shields-config": "ubuntu-latest", + "mcp-bridge-deepagents": "ubuntu-latest", + "mcp-bridge-hermes": "ubuntu-latest", + "mcp-bridge-openclaw": "ubuntu-latest", + "rebuild-hermes": "ubuntu-latest", + "rebuild-hermes-stale-base": "ubuntu-latest", + "security-posture-hermes": "ubuntu-latest", + "security-posture-openclaw": "ubuntu-latest", +}; + +describe("larger-runner workflow routing boundary", () => { + // source-shape-contract: security -- Executes the shipped pre-checkout router to prove untrusted and unset configurations stay on standard runners + it.each([ + { + label: "", + name: "the administrator label is unset", + ref: "refs/heads/main", + repository: "NVIDIA/NemoClaw", + }, + { + label: "ubuntu-24.04-8core", + name: "the workflow is not running from main", + ref: "refs/heads/feature", + repository: "NVIDIA/NemoClaw", + }, + { + label: "ubuntu-24.04-8core", + name: "the workflow belongs to another repository", + ref: "refs/heads/main", + repository: "someone/NemoClaw", + }, + { + checkoutSha: "0123456789abcdef", + label: "ubuntu-24.04-8core", + name: "the workflow checks out a pull request revision", + ref: "refs/heads/main", + repository: "NVIDIA/NemoClaw", + }, + ])("keeps every candidate on standard runners when $name (#7145)", ({ + checkoutSha, + label, + ref, + repository, + }) => { + expect( + evaluateRouting(readWorkflow() as RoutingWorkflow, { + checkoutSha, + label, + ref, + repository, + }), + ).toEqual(standardRouting); + }); + + // source-shape-contract: security -- Executes the shipped pre-checkout router to prove trusted main can reach only the reviewed heavy lanes + it("routes only the measured heavy lanes on trusted main (#7145)", () => { + const largerRunner = "ubuntu-24.04-8core"; + expect( + evaluateRouting(readWorkflow() as RoutingWorkflow, { + label: largerRunner, + ref: "refs/heads/main", + repository: "NVIDIA/NemoClaw", + }), + ).toEqual({ + ...standardRouting, + "channels-stop-start-hermes": largerRunner, + "common-egress-agent": largerRunner, + "hermes-dashboard": largerRunner, + "hermes-discord": largerRunner, + "hermes-e2e": largerRunner, + "hermes-inference-switch": largerRunner, + "hermes-shields-config": largerRunner, + "mcp-bridge-deepagents": largerRunner, + "mcp-bridge-hermes": largerRunner, + "rebuild-hermes": largerRunner, + "rebuild-hermes-stale-base": largerRunner, + "security-posture-hermes": largerRunner, + }); + }); + + // source-shape-contract: security -- Executes the shipped pre-checkout router to prove malformed administrator labels fail before job routing + it("rejects malformed administrator workflow labels (#7145)", () => { + expect(() => + evaluateRouting(readWorkflow() as RoutingWorkflow, { + label: "invalid runner\nlabel", + ref: "refs/heads/main", + repository: "NVIDIA/NemoClaw", + }), + ).toThrow(); + }); + + it("rejects moving or weakening the trusted pre-checkout map (#7145)", () => { + const workflow = readWorkflow() as RoutingWorkflow; + const generate = workflow.jobs["generate-matrix"]; + const steps = generate.steps!; + const routing = routingStep(workflow); + generate.outputs!.runner_routing = "${{ steps.matrix.outputs.runner_routing }}"; + routing.env!.CHECKOUT_SHA = "${{ github.sha }}"; + routing.env!.REF = "${{ inputs.base_sha }}"; + routing.run = routing.run!.replace('"${REF}" == "refs/heads/main" && ', ""); + routing.run = routing.run!.replace('-z "${CHECKOUT_SHA}" && ', ""); + steps.splice(steps.indexOf(routing), 1); + steps.push(routing); + + expect(validateE2eWorkflow(workflow)).toEqual( + expect.arrayContaining([ + "generate-matrix job must expose the trusted larger-runner routing output", + "trusted larger-runner routing step must bind only the administrator label and trusted repository identity", + "trusted larger-runner routing step must preserve the exact main-only map and ubuntu-latest fallback", + "trusted larger-runner routing step must run before PR checkout", + ]), + ); + }); + + it("rejects routing any lane outside the centralized eligible set (#7145)", () => { + const workflow = readWorkflow() as RoutingWorkflow; + workflow.jobs["common-egress-agent"]["runs-on"] = "ubuntu-latest"; + workflow.jobs["hermes-e2e"]["runs-on"] = "ubuntu-latest"; + workflow.jobs["mcp-bridge"]["runs-on"] = "ubuntu-latest"; + workflow.jobs["security-posture"]["runs-on"] = "ubuntu-latest"; + workflow.jobs["channels-stop-start"]["runs-on"] = "ubuntu-latest"; + workflow.jobs["mcp-bridge-dev"]["runs-on"] = + "${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['mcp-bridge-deepagents'] }}"; + workflow.jobs["network-policy"]["runs-on"] = "${{ vars.E2E_LARGER_RUNNER_LABEL }}"; + workflow.jobs["shields-config"]["runs-on"] = + "${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['common-egress-agent'] }}"; + + expect(validateE2eWorkflow(workflow)).toEqual( + expect.arrayContaining([ + "common-egress-agent job must use the trusted larger-runner routing map", + "hermes-e2e job must use the trusted larger-runner routing map", + "mcp-bridge job must route each matrix entry through the trusted runner map", + "security-posture job must route each matrix entry through the trusted runner map", + "channels-stop-start job must route each matrix entry through the trusted runner map", + "mcp-bridge-dev job must remain on ubuntu-latest", + "network-policy job must not consume E2E_LARGER_RUNNER_LABEL directly", + "shields-config job must not use the larger-runner routing map", + ]), + ); + }); +}); diff --git a/test/e2e/support/live-vitest-invocation.test.ts b/test/e2e/support/live-vitest-invocation.test.ts index ae51ec2f5a0..82151de6e0a 100644 --- a/test/e2e/support/live-vitest-invocation.test.ts +++ b/test/e2e/support/live-vitest-invocation.test.ts @@ -2,14 +2,20 @@ // SPDX-License-Identifier: Apache-2.0 import { spawnSync } from "node:child_process"; +import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; import { buildLiveVitestArgs, + HERMES_E2E_SWAP_BYTES, + HERMES_E2E_SWAP_FILE, + HERMES_E2E_SWAP_SCRIPT, LIVE_VITEST_PROJECT, type LiveVitestSpawner, + needsHermesE2ESwap, RISK_SIGNAL_REPORTER, runLiveVitestCommand, validateLiveProject, @@ -19,6 +25,110 @@ import { const LIVE_VITEST_TOOL = path.resolve("tools/e2e/live-vitest-invocation.mts"); const TSX = path.resolve("node_modules", ".bin", "tsx"); +const EXACT_HEAD_SHA = "a".repeat(40); + +interface FakeSwapScriptOptions { + failCleanupQuery?: boolean; + failSwapoff?: boolean; +} + +interface FakeSwapScriptResult { + calls: string[]; + status: number | null; + stderr: string; +} + +function writeFakeCommand(directory: string, name: string, lines: string[]): void { + const commandPath = path.join(directory, name); + writeFileSync(commandPath, `${["#!/bin/sh", "set -eu", ...lines].join("\n")}\n`); + chmodSync(commandPath, 0o755); +} + +function runHermesSwapScriptFailure(options: FakeSwapScriptOptions = {}): FakeSwapScriptResult { + const fakeBin = mkdtempSync(path.join(tmpdir(), "nemoclaw-hermes-swap-")); + const callLog = path.join(fakeBin, "calls.log"); + const swapState = path.join(fakeBin, "swap-state"); + const nameQueryCount = path.join(fakeBin, "name-query-count"); + writeFileSync(swapState, "inactive\n"); + + writeFakeCommand(fakeBin, "swapon", [ + `printf 'swapon:%s\\n' "$*" >> "$FAKE_CALL_LOG"`, + 'case "$*" in', + ' *"--output SIZE"*)', + ` printf '1\\n'`, + " ;;", + ' *"--output NAME"*)', + " query_count=0", + ' if [ -f "$FAKE_NAME_QUERY_COUNT_FILE" ]; then', + ' IFS= read -r query_count < "$FAKE_NAME_QUERY_COUNT_FILE" || query_count=0', + " fi", + " query_count=$((query_count + 1))", + ` printf '%s\\n' "$query_count" > "$FAKE_NAME_QUERY_COUNT_FILE"`, + ' if [ "${FAKE_FAIL_NAME_QUERY_AT:-0}" -eq "$query_count" ]; then', + ` printf 'swapon-name-query:%s:fail\\n' "$query_count" >> "$FAKE_CALL_LOG"`, + " exit 41", + " fi", + ' swap_state="inactive"', + ' if [ -f "$FAKE_SWAP_STATE_FILE" ]; then', + ' IFS= read -r swap_state < "$FAKE_SWAP_STATE_FILE" || swap_state="inactive"', + " fi", + ` printf 'swapon-name-query:%s:%s\\n' "$query_count" "$swap_state" >> "$FAKE_CALL_LOG"`, + ' if [ "$swap_state" = "active" ]; then', + ` printf '%s\\n' "$FAKE_FIXED_SWAP"`, + " fi", + " ;;", + " *)", + ` printf 'active\\n' > "$FAKE_SWAP_STATE_FILE"`, + ` printf 'swapon-activate:%s\\n' "$1" >> "$FAKE_CALL_LOG"`, + " ;;", + "esac", + ]); + writeFakeCommand(fakeBin, "awk", ["while IFS= read -r _line; do :; done", `printf '1\\n'`]); + writeFakeCommand(fakeBin, "swapoff", [ + `printf 'swapoff:%s\\n' "$1" >> "$FAKE_CALL_LOG"`, + 'if [ "${FAKE_FAIL_SWAPOFF:-0}" = "1" ]; then', + " exit 42", + "fi", + `printf 'inactive\\n' > "$FAKE_SWAP_STATE_FILE"`, + ]); + writeFakeCommand(fakeBin, "rm", [`printf 'rm:%s\\n' "$*" >> "$FAKE_CALL_LOG"`]); + writeFakeCommand(fakeBin, "fallocate", [`printf 'fallocate:%s\\n' "$*" >> "$FAKE_CALL_LOG"`]); + writeFakeCommand(fakeBin, "chmod", [`printf 'chmod:%s\\n' "$*" >> "$FAKE_CALL_LOG"`]); + writeFakeCommand(fakeBin, "mkswap", [`printf 'mkswap:%s\\n' "$*" >> "$FAKE_CALL_LOG"`]); + + try { + const result = spawnSync( + "/bin/bash", + [ + "--noprofile", + "--norc", + "-c", + HERMES_E2E_SWAP_SCRIPT, + "hermes-e2e-swap-test", + HERMES_E2E_SWAP_FILE, + String(HERMES_E2E_SWAP_BYTES), + ], + { + encoding: "utf8", + env: { + FAKE_CALL_LOG: callLog, + FAKE_FAIL_NAME_QUERY_AT: options.failCleanupQuery ? "2" : "0", + FAKE_FAIL_SWAPOFF: options.failSwapoff ? "1" : "0", + FAKE_FIXED_SWAP: HERMES_E2E_SWAP_FILE, + FAKE_NAME_QUERY_COUNT_FILE: nameQueryCount, + FAKE_SWAP_STATE_FILE: swapState, + LC_ALL: "C", + PATH: fakeBin, + }, + }, + ); + expect(result.error).toBeUndefined(); + const calls = existsSync(callLog) ? readFileSync(callLog, "utf8").trimEnd().split("\n") : []; + return { calls, status: result.status, stderr: result.stderr }; + } finally { + rmSync(fakeBin, { force: true, recursive: true }); + } +} describe("validateLiveProject (#6961)", () => { it("accepts the live project and defaults to it", () => { @@ -245,3 +355,286 @@ describe("runLiveVitestCommand (#6961)", () => { expect(result.stderr).toContain('expected "run"'); }); }); + +describe("runLiveVitestCommand Hermes resource setup (#7145)", () => { + it("provisions bounded idempotent swap before a hosted Hermes build", () => { + const calls: Array> = []; + const spawn: LiveVitestSpawner = (...args) => { + calls.push(args); + return { status: 0 }; + }; + + expect( + runLiveVitestCommand( + ["run", "--test-path", "test/e2e/live/hermes-inference-switch.test.ts"], + spawn, + { + GITHUB_ACTIONS: "true", + NEMOCLAW_E2E_EXPECTED_SHA: EXACT_HEAD_SHA, + }, + ), + ).toBe(0); + expect(calls).toHaveLength(2); + expect(calls[0]?.[0]).toBe("/usr/bin/sudo"); + expect(HERMES_E2E_SWAP_BYTES).toBe(34_359_738_368); + expect(HERMES_E2E_SWAP_FILE).toBe("/mnt/nemoclaw-hermes-e2e.swap"); + expect(calls[0]?.[1].slice(0, 9)).toEqual([ + "-n", + "/usr/bin/env", + "-i", + "PATH=/usr/sbin:/usr/bin:/sbin:/bin", + "LC_ALL=C", + "/bin/bash", + "--noprofile", + "--norc", + "-c", + ]); + expect(calls[0]?.[1].slice(10)).toEqual([ + "nemoclaw-hermes-e2e-swap", + HERMES_E2E_SWAP_FILE, + String(HERMES_E2E_SWAP_BYTES), + ]); + const script = calls[0]?.[1][9] ?? ""; + expect( + spawnSync("/bin/bash", ["--noprofile", "--norc", "-n"], { + input: script, + }).status, + ).toBe(0); + expect(script).toContain("if (( active_swap_bytes >= swap_size_bytes )); then"); + expect(script).toContain( + 'active_swap_names="$(swapon --show --noheadings --raw --output NAME)"', + ); + expect(script).toContain('if [[ "$active_swap_name" == "$swap_file" ]]; then'); + expect(script).toContain( + 'if (( fixed_swap_active == 1 )); then\n swapoff "$swap_file"\nfi\nrm -f -- "$swap_file"', + ); + expect(script).toContain( + 'if cleanup_swap_names="$(swapon --show --noheadings --raw --output NAME 2>/dev/null)"; then', + ); + expect(script).toContain('if swapoff "$swap_file" 2>/dev/null; then'); + expect(script).toContain("Preserving active Hermes E2E swap after setup failure"); + expect(script).toContain("Preserving Hermes E2E swap because active swap could not be queried"); + expect(script).not.toContain('swapoff "$swap_file" 2>/dev/null || true'); + expect(script).not.toContain("swap_enabled"); + expect(script).toContain('fallocate -l "$swap_size_bytes" "$swap_file"'); + expect(script).toContain("if (( active_swap_bytes < swap_size_bytes )); then"); + expect(calls[1]?.[0]).toBe("npx"); + }); + + it("fails closed before Vitest when Hermes swap provisioning fails", () => { + const calls: string[] = []; + const spawn: LiveVitestSpawner = (command) => { + calls.push(command); + return { status: 23 }; + }; + + expect( + runLiveVitestCommand( + ["run", "--test-path", "test/e2e/live/hermes-shields-config.test.ts"], + spawn, + { + GITHUB_ACTIONS: "true", + NEMOCLAW_E2E_EXPECTED_SHA: EXACT_HEAD_SHA, + }, + ), + ).toBe(23); + expect(calls).toEqual(["/usr/bin/sudo"]); + }); + + it.each([ + ["scheduled main", undefined], + ["manual main without a checkout SHA", ""], + ["a malformed checkout SHA", "A".repeat(40)], + ])("does not provision swap for %s", (_label, expectedSha) => { + const calls: string[] = []; + const spawn: LiveVitestSpawner = (command) => { + calls.push(command); + return { status: 0 }; + }; + + expect( + runLiveVitestCommand( + ["run", "--test-path", "test/e2e/live/hermes-inference-switch.test.ts"], + spawn, + { + GITHUB_ACTIONS: "true", + NEMOCLAW_E2E_EXPECTED_SHA: expectedSha, + }, + ), + ).toBe(0); + expect(calls).toEqual(["npx"]); + }); +}); + +describe("HERMES_E2E_SWAP_SCRIPT failure cleanup (#7145)", () => { + const provisioningFailureCalls = [ + "swapon:--show --bytes --noheadings --output SIZE", + "swapon:--show --noheadings --raw --output NAME", + "swapon-name-query:1:inactive", + `rm:-f -- ${HERMES_E2E_SWAP_FILE}`, + `fallocate:-l ${HERMES_E2E_SWAP_BYTES} ${HERMES_E2E_SWAP_FILE}`, + `chmod:0600 ${HERMES_E2E_SWAP_FILE}`, + `mkswap:${HERMES_E2E_SWAP_FILE}`, + `swapon:${HERMES_E2E_SWAP_FILE}`, + `swapon-activate:${HERMES_E2E_SWAP_FILE}`, + "swapon:--show --bytes --noheadings --output SIZE", + "swapon:--show --noheadings --raw --output NAME", + ]; + + it("removes the active fixed swap only after cleanup swapoff succeeds", () => { + const result = runHermesSwapScriptFailure(); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("Hermes E2E swap provisioning failed"); + expect(result.calls).toEqual([ + ...provisioningFailureCalls, + "swapon-name-query:2:active", + `swapoff:${HERMES_E2E_SWAP_FILE}`, + `rm:-f -- ${HERMES_E2E_SWAP_FILE}`, + ]); + }, 15_000); + + it("preserves the active fixed swap when cleanup swapoff fails", () => { + const result = runHermesSwapScriptFailure({ failSwapoff: true }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("Preserving active Hermes E2E swap after setup failure"); + expect(result.calls).toEqual([ + ...provisioningFailureCalls, + "swapon-name-query:2:active", + `swapoff:${HERMES_E2E_SWAP_FILE}`, + ]); + expect( + result.calls + .slice(result.calls.indexOf(`swapon-activate:${HERMES_E2E_SWAP_FILE}`) + 1) + .filter((call) => call.startsWith("rm:")), + ).toEqual([]); + }, 15_000); + + it("preserves the fixed swap when cleanup cannot query active swap", () => { + const result = runHermesSwapScriptFailure({ failCleanupQuery: true }); + + expect(result.status).toBe(1); + expect(result.stderr).toContain( + "Preserving Hermes E2E swap because active swap could not be queried", + ); + expect(result.calls).toEqual([...provisioningFailureCalls, "swapon-name-query:2:fail"]); + expect( + result.calls + .slice(result.calls.indexOf(`swapon-activate:${HERMES_E2E_SWAP_FILE}`) + 1) + .filter((call) => call.startsWith("rm:")), + ).toEqual([]); + }, 15_000); +}); + +describe("needsHermesE2ESwap (#7145)", () => { + const exactHeadEnv = { + GITHUB_ACTIONS: "true", + NEMOCLAW_E2E_EXPECTED_SHA: EXACT_HEAD_SHA, + }; + + it.each([ + ["Hermes inference switch", "test/e2e/live/hermes-inference-switch.test.ts", {}], + ["Hermes shields", "test/e2e/live/hermes-shields-config.test.ts", {}], + ["agent turn latency", "test/e2e/live/agent-turn-latency.test.ts", {}], + [ + "Hermes Bedrock", + "test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts", + { NEMOCLAW_AGENT: "hermes" }, + ], + [ + "Hermes E2E", + "test/e2e/live/hermes-e2e.test.ts", + { E2E_TARGET_ID: "hermes-e2e", NEMOCLAW_AGENT: "hermes" }, + ], + [ + "Hermes dashboard", + "test/e2e/live/hermes-e2e.test.ts", + { E2E_TARGET_ID: "hermes-dashboard", NEMOCLAW_AGENT: "hermes" }, + ], + [ + "Hermes security posture", + "test/e2e/live/hermes-e2e.test.ts", + { E2E_TARGET_ID: "security-posture", NEMOCLAW_AGENT: "hermes" }, + ], + [ + "Hermes MCP", + "test/e2e/live/mcp-bridge.test.ts", + { E2E_TARGET_ID: "mcp-bridge", NEMOCLAW_MCP_BRIDGE_AGENT: "hermes" }, + ], + ])("selects the exact-head hosted %s build", (_label, testPath, env) => { + expect(needsHermesE2ESwap(testPath, { ...exactHeadEnv, ...env })).toBe(true); + }); + + it.each([ + [ + "a non-GitHub runner", + "test/e2e/live/hermes-inference-switch.test.ts", + { GITHUB_ACTIONS: "" }, + ], + [ + "a scheduled main run", + "test/e2e/live/hermes-inference-switch.test.ts", + { NEMOCLAW_E2E_EXPECTED_SHA: undefined }, + ], + [ + "a manual main run without a checkout SHA", + "test/e2e/live/hermes-inference-switch.test.ts", + { NEMOCLAW_E2E_EXPECTED_SHA: "" }, + ], + [ + "an uppercase checkout SHA", + "test/e2e/live/hermes-inference-switch.test.ts", + { NEMOCLAW_E2E_EXPECTED_SHA: "A".repeat(40) }, + ], + [ + "the OpenClaw Bedrock shard", + "test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts", + { GITHUB_ACTIONS: "true", NEMOCLAW_AGENT: "openclaw" }, + ], + [ + "the OpenClaw security posture shard", + "test/e2e/live/hermes-e2e.test.ts", + { + E2E_TARGET_ID: "security-posture", + GITHUB_ACTIONS: "true", + NEMOCLAW_AGENT: "openclaw", + }, + ], + [ + "an unlisted Hermes target sharing the test file", + "test/e2e/live/hermes-e2e.test.ts", + { E2E_TARGET_ID: "future-hermes-job", NEMOCLAW_AGENT: "hermes" }, + ], + [ + "a non-Hermes MCP shard", + "test/e2e/live/mcp-bridge.test.ts", + { + E2E_TARGET_ID: "mcp-bridge", + GITHUB_ACTIONS: "true", + NEMOCLAW_MCP_BRIDGE_AGENT: "deepagents", + }, + ], + [ + "the explicit-only MCP dev lane", + "test/e2e/live/mcp-bridge.test.ts", + { + E2E_TARGET_ID: "mcp-bridge-dev", + GITHUB_ACTIONS: "true", + NEMOCLAW_MCP_BRIDGE_AGENT: "hermes", + }, + ], + [ + "a rebuild lane with workflow-managed swap", + "test/e2e/live/rebuild-hermes.test.ts", + { GITHUB_ACTIONS: "true", NEMOCLAW_AGENT: "hermes" }, + ], + [ + "a self-hosted Hermes lane", + "test/e2e/live/hermes-slack-e2e.test.ts", + { GITHUB_ACTIONS: "true", NEMOCLAW_AGENT: "hermes" }, + ], + ])("does not select %s", (_label, testPath, env) => { + expect(needsHermesE2ESwap(testPath, { ...exactHeadEnv, ...env })).toBe(false); + }); +}); diff --git a/test/e2e/support/security-posture-workflow-boundary.test.ts b/test/e2e/support/security-posture-workflow-boundary.test.ts index 7ccf597ce1f..2f2a2434c0d 100644 --- a/test/e2e/support/security-posture-workflow-boundary.test.ts +++ b/test/e2e/support/security-posture-workflow-boundary.test.ts @@ -66,7 +66,6 @@ describe("security posture workflow boundary", () => { const workflow = readSecurityPostureWorkflow(); const job = (workflow.jobs as Record>)["security-posture"]; const env = job.env as Record; - job["runs-on"] = "self-hosted"; job["timeout-minutes"] = 30; (job.strategy as Record)["fail-fast"] = true; delete env.NEMOCLAW_E2E_SECURITY_POSTURE; @@ -88,7 +87,6 @@ describe("security posture workflow boundary", () => { expect(install).toBeTruthy(); install!.run = "bash scripts/install-openshell.sh"; const errors = validateSecurityPostureWorkflow(workflow); - expect(errors).toContain("security-posture must run on ubuntu-latest"); expect(errors).toContain("security-posture must retain its 75 minute two-agent budget"); expect(errors).toContain("security-posture matrix must keep fail-fast disabled"); expect(errors).toContain("security-posture must set NEMOCLAW_E2E_SECURITY_POSTURE=1"); diff --git a/tools/e2e/hermes-dashboard-workflow-boundary.mts b/tools/e2e/hermes-dashboard-workflow-boundary.mts index fb226d2d161..0e6534cce70 100644 --- a/tools/e2e/hermes-dashboard-workflow-boundary.mts +++ b/tools/e2e/hermes-dashboard-workflow-boundary.mts @@ -67,7 +67,6 @@ export function validateHermesDashboardWorkflow(workflow: HermesDashboardWorkflo } requireEqual(errors, job.needs, "generate-matrix", `${JOB_NAME} must depend on generate-matrix`); - requireEqual(errors, job["runs-on"], "ubuntu-latest", `${JOB_NAME} must run on ubuntu-latest`); requireEqual(errors, job["timeout-minutes"], 75, `${JOB_NAME} timeout must be 75 minutes`); requireEqual(errors, env.E2E_JOB, "1", `${JOB_NAME} must be free-standing`); requireEqual( diff --git a/tools/e2e/live-vitest-invocation.mts b/tools/e2e/live-vitest-invocation.mts index 981f22c0609..feea5c2717a 100644 --- a/tools/e2e/live-vitest-invocation.mts +++ b/tools/e2e/live-vitest-invocation.mts @@ -20,9 +20,111 @@ const { spawnExitCode } = processExit; export const LIVE_VITEST_PROJECT = "e2e-live"; export const LIVE_TEST_ROOT = "test/e2e/live/"; export const RISK_SIGNAL_REPORTER = "test/e2e/risk-signal-reporter.ts"; +// Credentialed E2E trusts the workflow from main but executes this helper from +// the reviewed PR checkout, so exact-head resource setup must live here. +export const HERMES_E2E_SWAP_BYTES = 32 * 1024 * 1024 * 1024; +export const HERMES_E2E_SWAP_FILE = "/mnt/nemoclaw-hermes-e2e.swap"; const SHELL_METACHARACTER = /[^A-Za-z0-9_./^$=:@+-]/u; const TEST_PATH_PATTERN = /^[A-Za-z0-9_./-]+$/u; +const EXACT_HEAD_SHA_PATTERN = /^[0-9a-f]{40}$/u; +const ALWAYS_HERMES_BUILD_TESTS = new Set([ + "test/e2e/live/agent-turn-latency.test.ts", + "test/e2e/live/hermes-inference-switch.test.ts", + "test/e2e/live/hermes-shields-config.test.ts", +]); +const HERMES_BEDROCK_BUILD_TEST = "test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts"; +const HERMES_SHARED_E2E_TEST = "test/e2e/live/hermes-e2e.test.ts"; +const HERMES_SHARED_E2E_TARGETS = new Set(["hermes-dashboard", "hermes-e2e", "security-posture"]); +const HERMES_MCP_BUILD_TEST = "test/e2e/live/mcp-bridge.test.ts"; +export const HERMES_E2E_SWAP_SCRIPT = `set -euo pipefail +swap_file="$1" +swap_size_bytes="$2" + +case "$swap_size_bytes" in + ""|*[!0-9]*) + echo "Hermes E2E swap size must be an integer byte count" >&2 + exit 2 + ;; +esac + +active_swap_bytes="$(swapon --show --bytes --noheadings --output SIZE | awk '{ total += $1 } END { printf "%.0f", total }')" +active_swap_bytes="\${active_swap_bytes:-0}" +case "$active_swap_bytes" in + ""|*[!0-9]*) + echo "Unable to determine active swap capacity" >&2 + exit 2 + ;; +esac + +if (( active_swap_bytes >= swap_size_bytes )); then + printf 'Hermes E2E swap is already sufficient: %s bytes active\\n' "$active_swap_bytes" + exit 0 +fi + +active_swap_names="$(swapon --show --noheadings --raw --output NAME)" +fixed_swap_active=0 +while IFS= read -r active_swap_name; do + if [[ "$active_swap_name" == "$swap_file" ]]; then + fixed_swap_active=1 + break + fi +done <<< "$active_swap_names" +if (( fixed_swap_active == 1 )); then + swapoff "$swap_file" +fi +rm -f -- "$swap_file" + +cleanup_partial_swap() { + status="$?" + if (( status != 0 )); then + if cleanup_swap_names="$(swapon --show --noheadings --raw --output NAME 2>/dev/null)"; then + cleanup_swap_active=0 + while IFS= read -r cleanup_swap_name; do + if [[ "$cleanup_swap_name" == "$swap_file" ]]; then + cleanup_swap_active=1 + break + fi + done <<< "$cleanup_swap_names" + if (( cleanup_swap_active == 1 )); then + if swapoff "$swap_file" 2>/dev/null; then + rm -f -- "$swap_file" || true + else + printf 'Preserving active Hermes E2E swap after setup failure: %s\\n' "$swap_file" >&2 + fi + else + rm -f -- "$swap_file" || true + fi + else + printf 'Preserving Hermes E2E swap because active swap could not be queried: %s\\n' "$swap_file" >&2 + fi + fi + trap - EXIT + exit "$status" +} +trap cleanup_partial_swap EXIT + +fallocate -l "$swap_size_bytes" "$swap_file" +chmod 0600 "$swap_file" +mkswap "$swap_file" +swapon "$swap_file" + +active_swap_bytes="$(swapon --show --bytes --noheadings --output SIZE | awk '{ total += $1 } END { printf "%.0f", total }')" +active_swap_bytes="\${active_swap_bytes:-0}" +case "$active_swap_bytes" in + ""|*[!0-9]*) + echo "Unable to verify active swap capacity" >&2 + exit 2 + ;; +esac +if (( active_swap_bytes < swap_size_bytes )); then + printf 'Hermes E2E swap provisioning failed: %s of %s bytes active\\n' "$active_swap_bytes" "$swap_size_bytes" >&2 + exit 1 +fi + +trap - EXIT +printf 'Hermes E2E swap ready: %s bytes active\\n' "$active_swap_bytes" +swapon --show`; export interface LiveVitestInvocation { testPath: string | undefined; @@ -138,23 +240,87 @@ export function buildLiveVitestArgs(invocation: LiveVitestInvocation): string[] ]; } -export function runLiveVitestCli(cliArgs: string[], spawn: LiveVitestSpawner = spawnSync): number { - const argv = buildLiveVitestArgs(parseLiveVitestArgs(cliArgs)); - const result = spawn("npx", argv, { stdio: "inherit" }); - if (result.error) { - throw result.error; +export function needsHermesE2ESwap(testPath: string, env: NodeJS.ProcessEnv): boolean { + if ( + env.GITHUB_ACTIONS !== "true" || + !EXACT_HEAD_SHA_PATTERN.test(env.NEMOCLAW_E2E_EXPECTED_SHA ?? "") + ) { + return false; } + if (ALWAYS_HERMES_BUILD_TESTS.has(testPath)) return true; + if (testPath === HERMES_BEDROCK_BUILD_TEST) return env.NEMOCLAW_AGENT === "hermes"; + if (testPath === HERMES_SHARED_E2E_TEST) { + return ( + env.NEMOCLAW_AGENT === "hermes" && HERMES_SHARED_E2E_TARGETS.has(env.E2E_TARGET_ID ?? "") + ); + } + return ( + testPath === HERMES_MCP_BUILD_TEST && + env.E2E_TARGET_ID === "mcp-bridge" && + env.NEMOCLAW_MCP_BRIDGE_AGENT === "hermes" + ); +} + +function spawnResultExitCode(result: LiveVitestSpawnResult): number { + if (result.error) throw result.error; return spawnExitCode(result); } -export function runLiveVitestCommand(argv: string[], spawn: LiveVitestSpawner = spawnSync): number { +export function provisionHermesE2ESwap( + testPath: string, + env: NodeJS.ProcessEnv, + spawn: LiveVitestSpawner = spawnSync, +): number { + if (!needsHermesE2ESwap(testPath, env)) return 0; + + return spawnResultExitCode( + spawn( + "/usr/bin/sudo", + [ + "-n", + "/usr/bin/env", + "-i", + "PATH=/usr/sbin:/usr/bin:/sbin:/bin", + "LC_ALL=C", + "/bin/bash", + "--noprofile", + "--norc", + "-c", + HERMES_E2E_SWAP_SCRIPT, + "nemoclaw-hermes-e2e-swap", + HERMES_E2E_SWAP_FILE, + String(HERMES_E2E_SWAP_BYTES), + ], + { stdio: "inherit" }, + ), + ); +} + +export function runLiveVitestCli( + cliArgs: string[], + spawn: LiveVitestSpawner = spawnSync, + env: NodeJS.ProcessEnv = process.env, +): number { + const invocation = parseLiveVitestArgs(cliArgs); + const testPath = validateLiveTestPath(invocation.testPath); + const argv = buildLiveVitestArgs({ ...invocation, testPath }); + const swapExitCode = provisionHermesE2ESwap(testPath, env, spawn); + if (swapExitCode !== 0) return swapExitCode; + return spawnResultExitCode(spawn("npx", argv, { stdio: "inherit" })); +} + +export function runLiveVitestCommand( + argv: string[], + spawn: LiveVitestSpawner = spawnSync, + env: NodeJS.ProcessEnv = process.env, +): number { const [command, ...cliArgs] = argv; if (command !== "run") { throw new Error( `unsupported live Vitest command ${JSON.stringify(command ?? "")}; expected "run"`, ); } - return runLiveVitestCli(cliArgs, spawn); + return runLiveVitestCli(cliArgs, spawn, env); } if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { diff --git a/tools/e2e/security-posture-workflow-boundary.mts b/tools/e2e/security-posture-workflow-boundary.mts index 9a648d5966b..24d99c8377e 100644 --- a/tools/e2e/security-posture-workflow-boundary.mts +++ b/tools/e2e/security-posture-workflow-boundary.mts @@ -59,9 +59,6 @@ export function validateSecurityPostureWorkflow(workflow: WorkflowRecord): strin if (job.needs !== "generate-matrix") { errors.push(`${JOB_NAME} must depend on generate-matrix`); } - if (job["runs-on"] !== "ubuntu-latest") { - errors.push(`${JOB_NAME} must run on ubuntu-latest`); - } if (job["timeout-minutes"] !== 75) { errors.push(`${JOB_NAME} must retain its 75 minute two-agent budget`); } diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index f3473ad57c1..caec2915edd 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -144,6 +144,50 @@ const GUARDED_DOCKER_HUB_AUTH_REQUIRED = `\${{ ${TRUSTED_DOCKER_HUB_PREDICATE} & const GUARDED_DOCKER_HUB_USERNAME = `\${{ ${TRUSTED_DOCKER_HUB_PREDICATE} && secrets.DOCKERHUB_USERNAME || '' }}`; const GUARDED_DOCKER_HUB_TOKEN = `\${{ ${TRUSTED_DOCKER_HUB_PREDICATE} && secrets.DOCKERHUB_TOKEN || '' }}`; const GUARDED_HERMES_E2E_INFERENCE_KEY = `\${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '' && (inputs.inference_mode || 'mock') != 'mock' && secrets.NVIDIA_INFERENCE_API_KEY || '' }}`; +const RUNNER_ROUTING_OUTPUT = "${{ steps.runner_routing.outputs.runner_routing }}"; +const RUNNER_ROUTING_STEP_NAME = "Build trusted larger-runner routing"; +const RUNNER_ROUTING_SCRIPT = [ + "set -euo pipefail", + 'larger_runner="ubuntu-latest"', + 'if [[ "${REPOSITORY}" == "NVIDIA/NemoClaw" && "${REF}" == "refs/heads/main" && -z "${CHECKOUT_SHA}" && -n "${LARGER_RUNNER_LABEL}" ]]; then', + ' if [[ ! "${LARGER_RUNNER_LABEL}" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$ ]]; then', + ' echo "::error::E2E_LARGER_RUNNER_LABEL must be a 1-64 character workflow label using letters, digits, dots, underscores, or hyphens" >&2', + " exit 1", + " fi", + ' larger_runner="${LARGER_RUNNER_LABEL}"', + "fi", + 'runner_routing="$(jq -cn --arg standard "ubuntu-latest" --arg larger "${larger_runner}" \'{"channels-stop-start-hermes":$larger,"channels-stop-start-openclaw":$standard,"common-egress-agent":$larger,"hermes-dashboard":$larger,"hermes-discord":$larger,"hermes-e2e":$larger,"hermes-inference-switch":$larger,"hermes-shields-config":$larger,"mcp-bridge-deepagents":$larger,"mcp-bridge-hermes":$larger,"mcp-bridge-openclaw":$standard,"rebuild-hermes":$larger,"rebuild-hermes-stale-base":$larger,"security-posture-hermes":$larger,"security-posture-openclaw":$standard}\')"', + 'printf \'runner_routing=%s\\n\' "${runner_routing}" >> "${GITHUB_OUTPUT}"', +].join("\n"); +const ROUTED_JOB_RUNNER_EXPRESSIONS = { + "common-egress-agent": + "${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['common-egress-agent'] }}", + "hermes-dashboard": + "${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['hermes-dashboard'] }}", + "hermes-discord": + "${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['hermes-discord'] }}", + "hermes-e2e": "${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['hermes-e2e'] }}", + "hermes-inference-switch": + "${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['hermes-inference-switch'] }}", + "hermes-shields-config": + "${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['hermes-shields-config'] }}", + "rebuild-hermes": + "${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['rebuild-hermes'] }}", + "rebuild-hermes-stale-base": + "${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['rebuild-hermes-stale-base'] }}", +} as const; +const MATRIX_ROUTED_JOB_RUNNER_EXPRESSIONS = { + "channels-stop-start": + "${{ fromJSON(needs.generate-matrix.outputs.runner_routing)[format('channels-stop-start-{0}', matrix.agent)] }}", + "mcp-bridge": + "${{ fromJSON(needs.generate-matrix.outputs.runner_routing)[format('mcp-bridge-{0}', matrix.agent)] }}", + "security-posture": + "${{ fromJSON(needs.generate-matrix.outputs.runner_routing)[format('security-posture-{0}', matrix.agent)] }}", +} as const; +const ROUTED_JOB_NAMES = new Set([ + ...Object.keys(ROUTED_JOB_RUNNER_EXPRESSIONS), + ...Object.keys(MATRIX_ROUTED_JOB_RUNNER_EXPRESSIONS), +]); function asRecord(value: unknown): WorkflowRecord { return value && typeof value === "object" && !Array.isArray(value) @@ -679,6 +723,95 @@ function requireRunDoesNotContain( } } +function validateLargerRunnerRouting( + errors: string[], + jobs: WorkflowRecord, + generateMatrix: WorkflowRecord, + generateSteps: readonly WorkflowStep[], + generateCheckout: WorkflowStep | undefined, +): void { + const generateOutputs = asRecord(generateMatrix.outputs); + if (generateOutputs.runner_routing !== RUNNER_ROUTING_OUTPUT) { + errors.push("generate-matrix job must expose the trusted larger-runner routing output"); + } + + const routing = requireJobStep( + errors, + "generate-matrix", + generateSteps, + RUNNER_ROUTING_STEP_NAME, + ); + if (!routing) return; + if (routing.id !== "runner_routing") { + errors.push("trusted larger-runner routing step must use id runner_routing"); + } + if (routing.if !== undefined) { + errors.push("trusted larger-runner routing step must always publish the fallback map"); + } + if (routing.shell !== "bash") { + errors.push("trusted larger-runner routing step must use bash"); + } + const expectedEnv = { + CHECKOUT_SHA: "${{ inputs.checkout_sha }}", + LARGER_RUNNER_LABEL: "${{ vars.E2E_LARGER_RUNNER_LABEL }}", + REF: "${{ github.ref }}", + REPOSITORY: "${{ github.repository }}", + }; + if (!isDeepStrictEqual(asRecord(routing.env), expectedEnv)) { + errors.push( + "trusted larger-runner routing step must bind only the administrator label and trusted repository identity", + ); + } + if (stringValue(routing.run).trimEnd() !== RUNNER_ROUTING_SCRIPT) { + errors.push( + "trusted larger-runner routing step must preserve the exact main-only map and ubuntu-latest fallback", + ); + } + if ( + generateCheckout && + generateSteps.indexOf(routing) >= generateSteps.indexOf(generateCheckout) + ) { + errors.push("trusted larger-runner routing step must run before PR checkout"); + } + + for (const [jobName, expected] of Object.entries(ROUTED_JOB_RUNNER_EXPRESSIONS)) { + if (asRecord(jobs[jobName])["runs-on"] !== expected) { + errors.push(`${jobName} job must use the trusted larger-runner routing map`); + } + } + for (const [jobName, expected] of Object.entries(MATRIX_ROUTED_JOB_RUNNER_EXPRESSIONS)) { + if (asRecord(jobs[jobName])["runs-on"] !== expected) { + errors.push(`${jobName} job must route each matrix entry through the trusted runner map`); + } + } + if (asRecord(jobs["mcp-bridge-dev"])["runs-on"] !== "ubuntu-latest") { + errors.push("mcp-bridge-dev job must remain on ubuntu-latest"); + } + + for (const [jobName, jobValue] of Object.entries(jobs)) { + const runsOn = stringValue(asRecord(jobValue)["runs-on"]); + if ( + runsOn.includes("needs.generate-matrix.outputs.runner_routing") && + !ROUTED_JOB_NAMES.has(jobName) + ) { + errors.push(`${jobName} job must not use the larger-runner routing map`); + } + if ( + jobName !== "generate-matrix" && + JSON.stringify(jobValue).includes("vars.E2E_LARGER_RUNNER_LABEL") + ) { + errors.push(`${jobName} job must not consume E2E_LARGER_RUNNER_LABEL directly`); + } + } + for (const step of generateSteps) { + if (step !== routing && JSON.stringify(step).includes("vars.E2E_LARGER_RUNNER_LABEL")) { + errors.push( + "only the trusted larger-runner routing step may consume E2E_LARGER_RUNNER_LABEL", + ); + } + } +} + function requireUploadPathContains(errors: string[], uploadPath: string, expected: string): void { if (!uploadPath.includes(expected)) { errors.push(`artifact upload path must include ${expected}`); @@ -1257,9 +1390,6 @@ function validateCommonEgressAgentJob(errors: string[], jobs: WorkflowRecord): v return; } - if (job["runs-on"] !== "ubuntu-latest") { - errors.push("common-egress-agent job must run on ubuntu-latest"); - } validateFreeStandingJobSelector(errors, jobs, jobName, "common-egress-agent"); if (job["timeout-minutes"] !== 120) { errors.push("common-egress-agent job must keep the legacy 120 minute timeout"); @@ -1601,9 +1731,6 @@ function validateRebuildHermesJob( } errors.push(...validateRebuildHermesBootstrapBoundary(jobName, job)); - if (job["runs-on"] !== "ubuntu-latest") { - errors.push(`${jobName} job must run on ubuntu-latest`); - } validateFreeStandingJobSelector(errors, jobs, jobName, targetName); if (job["timeout-minutes"] !== 90) { errors.push(`${jobName} job must keep the legacy 90 minute timeout`); @@ -1683,7 +1810,6 @@ function validateRebuildHermesJob( if (asRecord(checkout?.with)["persist-credentials"] !== false) { errors.push(`${jobName} checkout step must set persist-credentials=false`); } - } function validateSandboxRebuildJob(errors: string[], jobs: WorkflowRecord): void { @@ -2485,9 +2611,6 @@ function validateHermesE2EJob(errors: string[], jobs: WorkflowRecord): void { return; } - if (job["runs-on"] !== "ubuntu-latest") { - errors.push("hermes-e2e job must run on ubuntu-latest"); - } if (job.needs !== "generate-matrix") { errors.push("hermes-e2e job must depend on generate-matrix validation"); } @@ -3400,9 +3523,6 @@ function validateChannelsStopStartJob(errors: string[], jobs: WorkflowRecord): v return; } - if (job["runs-on"] !== "ubuntu-latest") { - errors.push("channels-stop-start job must run on ubuntu-latest"); - } validateFreeStandingJobSelector(errors, jobs, jobName, targetName); if (job["timeout-minutes"] !== 90) { errors.push("channels-stop-start job must keep the 90 minute timeout"); @@ -4003,6 +4123,7 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { if (asRecord(generateCheckout?.with)["persist-credentials"] !== false) { errors.push("generate-matrix checkout step must set persist-credentials=false"); } + validateLargerRunnerRouting(errors, jobs, generateMatrix, generateSteps, generateCheckout); const generate = requireStep(errors, generateSteps, "Generate E2E target matrix"); const generateEnv = asRecord(generate?.env); if (generateEnv.CHECKOUT_SHA !== "${{ inputs.checkout_sha }}") {