diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 8f5f9adbd97..d158ea160a0 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -653,7 +653,7 @@ jobs: steps: - id: trusted_hermes_swap name: Provision trusted Hermes E2E swap - if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' && matrix.agent == 'hermes' }} + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && matrix.agent == 'hermes' }} shell: /bin/bash --noprofile --norc -e -o pipefail {0} env: BASH_ENV: /dev/null @@ -685,14 +685,28 @@ jobs: exit 1 } - if [[ "${REPOSITORY}" != "NVIDIA/NemoClaw" || "${EVENT_NAME}" != "workflow_dispatch" || "${REF}" != "refs/heads/main" ]]; then + if [[ "${REPOSITORY}" != "NVIDIA/NemoClaw" || "${REF}" != "refs/heads/main" ]]; then fail "workflow must run from NVIDIA/NemoClaw main" fi - if [[ ! "${CHECKOUT_SHA}" =~ ^[0-9a-f]{40}$ ]]; then - fail "checkout SHA must be lowercase 40-hex" + if [[ "${EVENT_NAME}" != "schedule" && "${EVENT_NAME}" != "workflow_dispatch" ]]; then + fail "workflow event must be schedule or workflow_dispatch" fi - if [[ ! "${EXPECTED_WORKFLOW_SHA}" =~ ^[0-9a-f]{40}$ || "${WORKFLOW_SHA}" != "${EXPECTED_WORKFLOW_SHA}" || "${WORKFLOW_SHA}" != "${DISPATCH_SHA}" ]]; then - fail "workflow source must match the trusted dispatch revision" + # Exact-head mode: controller-dispatched PR revision. + if [[ "${EVENT_NAME}" == "workflow_dispatch" && -n "${CHECKOUT_SHA}" ]]; then + if [[ ! "${CHECKOUT_SHA}" =~ ^[0-9a-f]{40}$ ]]; then + fail "checkout SHA must be lowercase 40-hex" + fi + if [[ ! "${EXPECTED_WORKFLOW_SHA}" =~ ^[0-9a-f]{40}$ || "${WORKFLOW_SHA}" != "${EXPECTED_WORKFLOW_SHA}" || "${WORKFLOW_SHA}" != "${DISPATCH_SHA}" ]]; then + fail "workflow source must match the trusted dispatch revision" + fi + else + # Direct-main mode: schedule or manual trigger on main. + if [[ -n "${CHECKOUT_SHA}" || -n "${EXPECTED_WORKFLOW_SHA}" ]]; then + fail "direct main runs must not request an alternate checkout or workflow revision" + fi + if [[ ! "${WORKFLOW_SHA}" =~ ^[0-9a-f]{40}$ || "${WORKFLOW_SHA}" != "${DISPATCH_SHA}" ]]; then + fail "direct main workflow source must match the run revision" + fi fi if [[ "${RUNNER_ENVIRONMENT_KIND}" != "github-hosted" || "${RUNNER_OS_KIND}" != "Linux" || "${RUNNER_ARCH_KIND}" != "X64" ]]; then fail "swap fallback requires an ephemeral GitHub-hosted Linux x64 runner" @@ -1332,7 +1346,7 @@ jobs: steps: - id: trusted_hermes_swap name: Provision trusted Hermes E2E swap - if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' }} + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} shell: /bin/bash --noprofile --norc -e -o pipefail {0} env: BASH_ENV: /dev/null @@ -1474,7 +1488,7 @@ jobs: steps: - id: trusted_hermes_swap name: Provision trusted Hermes E2E swap - if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' }} + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} shell: /bin/bash --noprofile --norc -e -o pipefail {0} env: BASH_ENV: /dev/null @@ -1981,7 +1995,7 @@ jobs: steps: - id: trusted_hermes_swap name: Provision trusted Hermes E2E swap - if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' && (contains(format(',{0},', inputs.jobs), ',hermes-e2e,') || contains(format(',{0},', inputs.targets), ',hermes-e2e,')) }} + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && (github.event_name == 'schedule' || inputs.checkout_sha == '' || (github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && (contains(format(',{0},', inputs.jobs), ',hermes-e2e,') || contains(format(',{0},', inputs.targets), ',hermes-e2e,')))) }} shell: /bin/bash --noprofile --norc -e -o pipefail {0} env: BASH_ENV: /dev/null @@ -2321,7 +2335,7 @@ jobs: steps: - id: trusted_hermes_swap name: Provision trusted Hermes E2E swap - if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' }} + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} shell: /bin/bash --noprofile --norc -e -o pipefail {0} env: BASH_ENV: /dev/null @@ -2394,6 +2408,26 @@ jobs: NEMOCLAW_POLICY_TIER: open OPENSHELL_GATEWAY: nemoclaw steps: + - id: trusted_hermes_swap + name: Provision trusted Hermes E2E swap + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} + shell: /bin/bash --noprofile --norc -e -o pipefail {0} + env: + BASH_ENV: /dev/null + CHECKOUT_SHA: ${{ inputs.checkout_sha }} + DISPATCH_SHA: ${{ github.sha }} + ENV: /dev/null + EVENT_NAME: ${{ github.event_name }} + EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }} + LC_ALL: C + REF: ${{ github.ref }} + REPOSITORY: ${{ github.repository }} + RUNNER_ARCH_KIND: ${{ runner.arch }} + RUNNER_ENVIRONMENT_KIND: ${{ runner.environment }} + RUNNER_OS_KIND: ${{ runner.os }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + run: *trusted-hermes-e2e-swap + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: ref: ${{ inputs.checkout_sha || github.sha }} @@ -2550,6 +2584,26 @@ jobs: NEMOCLAW_RECREATE_SANDBOX: "1" OPENSHELL_GATEWAY: nemoclaw steps: + - id: trusted_hermes_swap + name: Provision trusted Hermes E2E swap + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && matrix.scenario == 'hermes-open-reference' }} + shell: /bin/bash --noprofile --norc -e -o pipefail {0} + env: + BASH_ENV: /dev/null + CHECKOUT_SHA: ${{ inputs.checkout_sha }} + DISPATCH_SHA: ${{ github.sha }} + ENV: /dev/null + EVENT_NAME: ${{ github.event_name }} + EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }} + LC_ALL: C + REF: ${{ github.ref }} + REPOSITORY: ${{ github.repository }} + RUNNER_ARCH_KIND: ${{ runner.arch }} + RUNNER_ENVIRONMENT_KIND: ${{ runner.environment }} + RUNNER_OS_KIND: ${{ runner.os }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + run: *trusted-hermes-e2e-swap + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: ref: ${{ inputs.checkout_sha || github.sha }} @@ -2687,7 +2741,7 @@ jobs: steps: - id: trusted_hermes_swap name: Provision trusted Hermes E2E swap - if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' }} + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} shell: /bin/bash --noprofile --norc -e -o pipefail {0} env: BASH_ENV: /dev/null @@ -2836,6 +2890,16 @@ jobs: - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 + - name: Add swap for Hermes image rebuild + shell: bash + run: | + set -euo pipefail + sudo fallocate -l 32G /mnt/nemoclaw-hermes-rebuild.swap + sudo chmod 0600 /mnt/nemoclaw-hermes-rebuild.swap + sudo mkswap /mnt/nemoclaw-hermes-rebuild.swap + sudo swapon /mnt/nemoclaw-hermes-rebuild.swap + swapon --show + - name: Initialize runner comparison telemetry if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' }} continue-on-error: true @@ -2849,16 +2913,6 @@ jobs: set -euo pipefail env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u NVIDIA_INFERENCE_API_KEY -u GITHUB_TOKEN -u GH_TOKEN bash scripts/install-openshell.sh - - name: Add swap for Hermes image rebuild - shell: bash - run: | - set -euo pipefail - sudo fallocate -l 32G /mnt/nemoclaw-hermes-rebuild.swap - sudo chmod 0600 /mnt/nemoclaw-hermes-rebuild.swap - sudo mkswap /mnt/nemoclaw-hermes-rebuild.swap - sudo swapon /mnt/nemoclaw-hermes-rebuild.swap - swapon --show - - name: Run Hermes rebuild live test # Preserves the checked-out CLI, Docker/OpenShell, Hermes base-image # rebuild, registry, messaging-placeholder, and backup hygiene boundaries. @@ -2950,6 +3004,16 @@ jobs: - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 + - name: Add swap for Hermes image rebuild + shell: bash + run: | + set -euo pipefail + sudo fallocate -l 32G /mnt/nemoclaw-hermes-rebuild.swap + sudo chmod 0600 /mnt/nemoclaw-hermes-rebuild.swap + sudo mkswap /mnt/nemoclaw-hermes-rebuild.swap + sudo swapon /mnt/nemoclaw-hermes-rebuild.swap + swapon --show + - name: Initialize runner comparison telemetry if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' }} continue-on-error: true @@ -2963,16 +3027,6 @@ jobs: set -euo pipefail env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u NVIDIA_INFERENCE_API_KEY -u GITHUB_TOKEN -u GH_TOKEN bash scripts/install-openshell.sh - - name: Add swap for Hermes image rebuild - shell: bash - run: | - set -euo pipefail - sudo fallocate -l 32G /mnt/nemoclaw-hermes-rebuild.swap - sudo chmod 0600 /mnt/nemoclaw-hermes-rebuild.swap - sudo mkswap /mnt/nemoclaw-hermes-rebuild.swap - sudo swapon /mnt/nemoclaw-hermes-rebuild.swap - swapon --show - - name: Run Hermes stale-base rebuild live test # Uses NEMOCLAW_HERMES_STALE_BASE_REBUILD_E2E=1, preserving issue #3025's # stale cached base-image regression boundary. @@ -3655,7 +3709,7 @@ jobs: steps: - id: trusted_hermes_swap name: Provision trusted Hermes E2E swap - if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' && matrix.agent == 'hermes' }} + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && matrix.agent == 'hermes' }} shell: /bin/bash --noprofile --norc -e -o pipefail {0} env: BASH_ENV: /dev/null @@ -4938,7 +4992,7 @@ jobs: steps: - id: trusted_hermes_swap name: Provision trusted Hermes E2E swap - if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' && matrix.agent == 'hermes' }} + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && matrix.agent == 'hermes' }} shell: /bin/bash --noprofile --norc -e -o pipefail {0} env: BASH_ENV: /dev/null @@ -5239,6 +5293,26 @@ jobs: NEMOCLAW_SANDBOX_NAME: e2e-channels-stop-start-${{ matrix.agent }} OPENSHELL_GATEWAY: "nemoclaw" steps: + - id: trusted_hermes_swap + name: Provision trusted Hermes E2E swap + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && matrix.agent == 'hermes' }} + shell: /bin/bash --noprofile --norc -e -o pipefail {0} + env: + BASH_ENV: /dev/null + CHECKOUT_SHA: ${{ inputs.checkout_sha }} + DISPATCH_SHA: ${{ github.sha }} + ENV: /dev/null + EVENT_NAME: ${{ github.event_name }} + EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }} + LC_ALL: C + REF: ${{ github.ref }} + REPOSITORY: ${{ github.repository }} + RUNNER_ARCH_KIND: ${{ runner.arch }} + RUNNER_ENVIRONMENT_KIND: ${{ runner.environment }} + RUNNER_OS_KIND: ${{ runner.os }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + run: *trusted-hermes-e2e-swap + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: ref: ${{ inputs.checkout_sha || github.sha }} diff --git a/test/e2e/README.md b/test/e2e/README.md index 587467d28ae..e5db3a8093d 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -62,13 +62,16 @@ no alternate checkout SHA is requested. PR-gate dispatches therefore remain on standard runners even though they use the trusted workflow definition from `main`. -Exact-head PR-gate dispatches use a bounded swap fallback for the hosted -Hermes image-building lanes that remain on those standard runners. The trusted -workflow provisions the fallback as the first job step, before checking out or -executing the candidate revision. It requires a controller-supplied lowercase -40-hex checkout SHA, matching trusted workflow and dispatch revisions, and an -ephemeral GitHub-hosted Linux x64 runner. Candidate code cannot supply the -program or arguments passed to `sudo`. +Exact-head PR-gate dispatches and direct scheduled or manual `main` runs use a +bounded swap fallback for eligible hosted Hermes image-building lanes. The +fallback does not change runner routing. The trusted workflow provisions the +fallback as the first job step, before checking out or executing the selected +revision. Exact-head mode requires a controller-supplied lowercase 40-hex +checkout SHA plus matching trusted workflow and dispatch revisions. Direct-main +mode rejects alternate checkout and workflow revisions and requires the +workflow source to match the run revision. Both modes require an ephemeral +GitHub-hosted Linux x64 runner. Candidate code cannot supply the program or +arguments passed to `sudo`. The trusted step requires at least 32 GiB (34,359,738,368 bytes) of usable swap. It reuses active swap that meets this requirement. @@ -87,20 +90,20 @@ Cleanup removes it only after `swapoff` succeeds. Successful state is discarded with the ephemeral runner. The fallback covers agent-turn latency, Hermes inference switch and shields, -the Hermes Bedrock and stable MCP shards, and the `hermes-e2e`, -`hermes-dashboard`, and Hermes security-posture tests. Scheduled and ordinary -manual `main` runs, larger-runner executions, rebuild lanes with -workflow-managed swap, dedicated-runner lanes, `mcp-bridge-dev`, and non-Hermes -shards do not use it. +the Hermes Bedrock and stable MCP shards, the Hermes common-egress and channel +stop/start shards, and the `hermes-e2e`, `hermes-dashboard`, `hermes-discord`, +and Hermes security-posture tests. Rebuild lanes with workflow-managed swap, +dedicated-runner lanes, `mcp-bridge-dev`, and non-Hermes shards do not use it. +Candidate-authored workflow definitions and fork-owned runs cannot reach it. The fallback exists because the alternate-checkout trust boundary deliberately keeps PR-authored code from selecting the administrator-managed larger-runner label; changing the PR checkout cannot safely grant itself that capacity. -Remove the fallback only after the trusted controller routes exact-head PR -gates to an ephemeral GitHub-hosted runner with at least 32 GB RAM without -weakening the exact-SHA guard, and five consecutive runs of every protected -lane complete without runner loss while runner-pressure telemetry reports less -than 1 GiB of swap used. +Remove the fallback only after trusted main and exact-head PR runs use +ephemeral GitHub-hosted runners with at least 32 GB RAM without weakening the +source guards, and five consecutive runs of every protected lane complete +without runner loss while runner-pressure telemetry reports less than 1 GiB of +swap used. The eligible set is limited to the measured or repeatedly interrupted heavy lanes: @@ -183,7 +186,8 @@ The OpenClaw matrix entries for `mcp-bridge`, Each execution writes one bounded, ordered v2 time series to the canonical `runner-comparison.jsonl` ledger. It contains: -- an `initialize` endpoint after workspace preparation; +- an `initialize` endpoint after workspace preparation and any fixed-capacity + rebuild swap; - a distinct `scenario-start` for every test handled by the execution; - a `periodic` sample on an approximately 60-second fixed cadence; - a `phase` sample before each semantic phase transition and when the final @@ -202,7 +206,9 @@ records exist to reserve the last slot for `finalize`. A missing, historical-v1, already-finalized, full, or invalid ledger permanently disables comparison sampling for that test progress instance. In `rebuild-hermes` and `rebuild-hermes-stale-base`, where legacy phase resource evidence is configured, -the existing five-minute full snapshot then becomes the best-effort fallback. +the workflow establishes its 32 GiB swap before `initialize` so the ledger sees +one stable swap capacity. If canonical sampling becomes unavailable, the +existing five-minute full snapshot becomes the best-effort fallback. That full profile may run `ps`, `docker stats`, and `docker system df` sequentially with a 15-second timeout each, or 45 seconds in the worst case; canonical sampling suppresses this heavier collection while it remains active. @@ -801,8 +807,8 @@ memory-heavy image build. The rebuild fixture verifies that floor and provisions the same swap file on GitHub Actions when a trusted control-plane run uses the workflow definition from `main`. Those paths build large Hermes image layers and can otherwise exhaust the runner's default memory and swap -during Docker layer export. Other E2E jobs keep the standard runner memory -configuration except for the exact-head Hermes PR-gate fallback described in +during Docker layer export. Apart from those rebuild and export paths, E2E jobs +add swap only through the trusted Hermes main-workflow fallback described in [Larger-runner routing](#larger-runner-routing). These assertions run inside the existing `full-e2e` lifecycle instead of a diff --git a/test/e2e/support/runner-comparison-workflow-boundary.test.ts b/test/e2e/support/runner-comparison-workflow-boundary.test.ts index 3afed2d27c1..beb3a75e5ee 100644 --- a/test/e2e/support/runner-comparison-workflow-boundary.test.ts +++ b/test/e2e/support/runner-comparison-workflow-boundary.test.ts @@ -4,6 +4,7 @@ import { describe, expect, it } from "vitest"; import { + HERMES_REBUILD_SWAP_STEP, RUNNER_COMPARISON_COMMAND, RUNNER_COMPARISON_FINALIZE_STEP, RUNNER_COMPARISON_INITIALIZE_STEP, @@ -40,6 +41,7 @@ const JOBS = [ "rebuild-hermes-stale-base", "security-posture", ] as const; +const REBUILD_JOBS = ["rebuild-hermes", "rebuild-hermes-stale-base"] as const; function loadWorkflow(): Workflow { return structuredClone(readWorkflow()) as Workflow; @@ -156,9 +158,10 @@ describe("runner comparison E2E workflow boundary (#7145)", () => { lateSteps[initializeIndex + 1]!, lateSteps[initializeIndex]!, ]; - expect(validateRunnerComparisonWorkflow(lateInitialize)).toContain( - `${jobId} must initialize runner comparison telemetry immediately after prepare-e2e`, - ); + const expectedInitializeError = REBUILD_JOBS.includes(jobId as (typeof REBUILD_JOBS)[number]) + ? `${jobId} must establish rebuild swap before initializing runner comparison telemetry` + : `${jobId} must initialize runner comparison telemetry immediately after prepare-e2e`; + expect(validateRunnerComparisonWorkflow(lateInitialize)).toContain(expectedInitializeError); const afterPublication = loadWorkflow(); const publicationSteps = afterPublication.jobs[jobId]!.steps; @@ -174,6 +177,35 @@ describe("runner comparison E2E workflow boundary (#7145)", () => { ); }); + it.each( + REBUILD_JOBS, + )("initializes %s telemetry only after workflow-managed swap reaches its final capacity", (jobId) => { + const workflow = loadWorkflow(); + const jobSteps = workflow.jobs[jobId]!.steps; + const swap = step(workflow, jobId, HERMES_REBUILD_SWAP_STEP); + const initialize = step(workflow, jobId, RUNNER_COMPARISON_INITIALIZE_STEP); + const swapIndex = jobSteps.indexOf(swap); + const initializeIndex = jobSteps.indexOf(initialize); + + expect(initializeIndex).toBe(swapIndex + 1); + + [jobSteps[swapIndex], jobSteps[initializeIndex]] = [initialize, swap]; + expect(validateRunnerComparisonWorkflow(workflow)).toContain( + `${jobId} must establish rebuild swap before initializing runner comparison telemetry`, + ); + }); + + it.each(REBUILD_JOBS)("rejects %s telemetry when rebuild swap is delayed", (jobId) => { + const workflow = loadWorkflow(); + const jobSteps = workflow.jobs[jobId]!.steps; + const swapIndex = jobSteps.indexOf(step(workflow, jobId, HERMES_REBUILD_SWAP_STEP)); + jobSteps.splice(swapIndex, 0, { name: "Unexpected step before rebuild swap" }); + + expect(validateRunnerComparisonWorkflow(workflow)).toContain( + `${jobId} must establish rebuild swap before initializing runner comparison telemetry`, + ); + }); + it("rejects weakened trusted-main and always-run guards", () => { const workflow = loadWorkflow(); step(workflow, "common-egress-agent", RUNNER_COMPARISON_INITIALIZE_STEP).if = diff --git a/test/e2e/support/runner-pressure-workflow-boundary.test.ts b/test/e2e/support/runner-pressure-workflow-boundary.test.ts index f2a497bdfa6..550746b6432 100644 --- a/test/e2e/support/runner-pressure-workflow-boundary.test.ts +++ b/test/e2e/support/runner-pressure-workflow-boundary.test.ts @@ -26,6 +26,12 @@ function swapStep(workflow: Workflow, jobId: (typeof JOBS)[number]): WorkflowSte )!; } +function comparisonInitializeStep(workflow: Workflow, jobId: (typeof JOBS)[number]): WorkflowStep { + return workflow.jobs[jobId]!.steps.find( + (step) => step.name === "Initialize runner comparison telemetry", + )!; +} + describe("runner-pressure E2E workflow boundary (#7146)", () => { it("accepts the canonical Hermes heartbeat and terminal-consumer wiring", () => { expect(validateRunnerPressureWorkflow(loadWorkflow())).toEqual([]); @@ -37,9 +43,11 @@ describe("runner-pressure E2E workflow boundary (#7146)", () => { const workflow = loadWorkflow(); const jobSteps = workflow.jobs[jobId]!.steps; const provision = swapStep(workflow, jobId); + const comparisonInitialize = comparisonInitializeStep(workflow, jobId); const run = runStep(workflow, jobId); expect(provision).toBeDefined(); + expect(jobSteps.indexOf(provision)).toBeLessThan(jobSteps.indexOf(comparisonInitialize)); expect(jobSteps.indexOf(provision)).toBeLessThan(jobSteps.indexOf(run)); expect(provision.run).toContain("fallocate -l 32G /mnt/nemoclaw-hermes-rebuild.swap"); expect(provision.run).toContain("chmod 0600 /mnt/nemoclaw-hermes-rebuild.swap"); diff --git a/test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts b/test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts index e5240a20385..d5e1afffcb8 100644 --- a/test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts +++ b/test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts @@ -34,7 +34,10 @@ type SwapWorkflow = { const PROTECTED_JOBS = [ "agent-turn-latency", "bedrock-runtime-compatible-anthropic", + "channels-stop-start", + "common-egress-agent", "hermes-dashboard", + "hermes-discord", "hermes-e2e", "hermes-inference-switch", "hermes-shields-config", @@ -52,12 +55,22 @@ function trustedSwapStep(workflow: SwapWorkflow, jobName: string): WorkflowStep type SwapHarnessOptions = { activeSwapBytes?: number; + checkoutSha?: string; + dispatchSha?: string; diskBytes?: number; + eventName?: string; + expectedWorkflowSha?: string; failCleanupQuery?: boolean; failMkswap?: boolean; failSwapoff?: boolean; hiddenActivationReads?: number; provisionedSwapBytes?: number; + ref?: string; + repository?: string; + runnerArch?: string; + runnerEnvironment?: string; + runnerOs?: string; + workflowSha?: string; }; type SwapHarnessResult = { @@ -79,6 +92,7 @@ function runTrustedSwapHarness(options: SwapHarnessOptions = {}): SwapHarnessRes const queryCount = path.join(fakeBin, "query-count"); const swapState = path.join(fakeBin, "swap-state"); const swapFile = "/mnt/nemoclaw-hermes-e2e-swap/nemoclaw-hermes.fake.swap"; + writeFileSync(callLog, ""); writeFileSync(swapState, "inactive\n"); const commands = new Map(); @@ -196,16 +210,18 @@ function runTrustedSwapHarness(options: SwapHarnessOptions = {}): SwapHarnessRes } try { - const workflowSha = "b".repeat(40); + const workflowSha = options.workflowSha ?? "b".repeat(40); + const checkoutSha = options.checkoutSha ?? "a".repeat(40); const result = spawnSync("/bin/bash", ["--noprofile", "--norc", "-c", script], { encoding: "utf8", env: { BASH_ENV: "/dev/null", - CHECKOUT_SHA: "a".repeat(40), - DISPATCH_SHA: workflowSha, + CHECKOUT_SHA: checkoutSha, + DISPATCH_SHA: options.dispatchSha ?? workflowSha, ENV: "/dev/null", - EVENT_NAME: "workflow_dispatch", - EXPECTED_WORKFLOW_SHA: workflowSha, + EVENT_NAME: options.eventName ?? "workflow_dispatch", + EXPECTED_WORKFLOW_SHA: + options.expectedWorkflowSha ?? (checkoutSha === "" ? "" : workflowSha), FAKE_ACTIVE_SWAP_BYTES: String(options.activeSwapBytes ?? 0), FAKE_CALL_LOG: callLog, FAKE_DISK_BYTES: String(options.diskBytes ?? 100_000_000_000), @@ -219,15 +235,16 @@ function runTrustedSwapHarness(options: SwapHarnessOptions = {}): SwapHarnessRes FAKE_SWAP_STATE: swapState, LC_ALL: "C", PATH: "/usr/bin:/bin", - REF: "refs/heads/main", - REPOSITORY: "NVIDIA/NemoClaw", - RUNNER_ARCH_KIND: "X64", - RUNNER_ENVIRONMENT_KIND: "github-hosted", - RUNNER_OS_KIND: "Linux", + REF: options.ref ?? "refs/heads/main", + REPOSITORY: options.repository ?? "NVIDIA/NemoClaw", + RUNNER_ARCH_KIND: options.runnerArch ?? "X64", + RUNNER_ENVIRONMENT_KIND: options.runnerEnvironment ?? "github-hosted", + RUNNER_OS_KIND: options.runnerOs ?? "Linux", WORKFLOW_SHA: workflowSha, }, }); - const calls = readFileSync(callLog, "utf8").trimEnd().split("\n"); + const callContents = readFileSync(callLog, "utf8").trimEnd(); + const calls = callContents === "" ? [] : callContents.split("\n"); return { calls, status: result.status, stderr: result.stderr }; } finally { rmSync(fakeBin, { force: true, recursive: true }); @@ -281,6 +298,116 @@ describe("trusted Hermes swap workflow boundary", () => { expect(TRUSTED_HERMES_SWAP_SCRIPT).not.toContain("${{"); }); + it.each([ + "schedule", + "workflow_dispatch", + ])("accepts the trusted direct main source for %s runs (#7145)", (eventName) => { + const result = runTrustedSwapHarness({ + activeSwapBytes: 34_359_738_368, + checkoutSha: "", + eventName, + }); + + expect(result.status).toBe(0); + expect(result.calls).toEqual([ + "stat:-c %F:%u:%g -- /mnt", + "swapon:--show=SIZE --bytes --noheadings", + ]); + }); + + it.each([ + { + expected: "direct main runs must not request an alternate checkout or workflow revision", + name: "a scheduled run supplies an alternate checkout", + options: { checkoutSha: "a".repeat(40), eventName: "schedule" }, + }, + { + expected: "direct main runs must not request an alternate checkout or workflow revision", + name: "a direct dispatch supplies an alternate workflow revision", + options: { + checkoutSha: "", + expectedWorkflowSha: "b".repeat(40), + }, + }, + { + expected: "direct main workflow source must match the run revision", + name: "the workflow and run revisions diverge", + options: { + checkoutSha: "", + dispatchSha: "c".repeat(40), + }, + }, + { + expected: "direct main workflow source must match the run revision", + name: "the workflow source is malformed", + options: { + checkoutSha: "", + dispatchSha: "b".repeat(40), + workflowSha: "not-a-sha", + }, + }, + ])("rejects trusted direct main mode when $name (#7145)", ({ expected, options }) => { + const result = runTrustedSwapHarness(options); + + expect(result.status).toBe(1); + expect(result.stderr).toContain(expected); + expect(result.calls).toEqual([]); + }); + + it.each([ + { + expected: "workflow must run from NVIDIA/NemoClaw main", + name: "the repository is not canonical", + options: { repository: "example/NemoClaw" }, + }, + { + expected: "workflow must run from NVIDIA/NemoClaw main", + name: "the ref is not main", + options: { ref: "refs/heads/candidate" }, + }, + { + expected: "workflow event must be schedule or workflow_dispatch", + name: "the event is not trusted", + options: { eventName: "pull_request" }, + }, + { + expected: "checkout SHA must be lowercase 40-hex", + name: "the exact-head checkout SHA is malformed", + options: { checkoutSha: "A".repeat(40) }, + }, + { + expected: "workflow source must match the trusted dispatch revision", + name: "the exact-head workflow SHA is missing", + options: { expectedWorkflowSha: "" }, + }, + { + expected: "workflow source must match the trusted dispatch revision", + name: "the exact-head workflow SHA differs", + options: { expectedWorkflowSha: "c".repeat(40) }, + }, + { + expected: "swap fallback requires an ephemeral GitHub-hosted Linux x64 runner", + name: "the runner is self-hosted", + options: { runnerEnvironment: "self-hosted" }, + }, + { + expected: "swap fallback requires an ephemeral GitHub-hosted Linux x64 runner", + name: "the runner OS is not Linux", + options: { runnerOs: "Windows" }, + }, + { + expected: "swap fallback requires an ephemeral GitHub-hosted Linux x64 runner", + name: "the runner architecture is not x64", + options: { runnerArch: "ARM64" }, + }, + ])("rejects identity drift when $name (#7145)", ({ expected, options }) => { + const result = runTrustedSwapHarness(options); + + expect(result.status).toBe(1); + expect(result.stderr).toContain(expected); + expect(result.calls).toEqual([]); + }); + it("exits before privileged allocation when enough swap is already active (#7145)", () => { const result = runTrustedSwapHarness({ activeSwapBytes: 34_359_738_368 }); @@ -396,9 +523,18 @@ describe("trusted Hermes swap workflow boundary", () => { const bedrockProvision = trustedSwapStep(workflow, "bedrock-runtime-compatible-anthropic"); bedrockProvision.run = "sudo bash tools/e2e/live-vitest-invocation.mts"; + const channelsProvision = trustedSwapStep(workflow, "channels-stop-start"); + channelsProvision.if = channelsProvision.if!.replace(" && matrix.agent == 'hermes'", ""); + + const commonEgressProvision = trustedSwapStep(workflow, "common-egress-agent"); + commonEgressProvision.if = commonEgressProvision.if!.replace( + " && matrix.scenario == 'hermes-open-reference'", + "", + ); + const hermesE2eProvision = trustedSwapStep(workflow, "hermes-e2e"); hermesE2eProvision.if = hermesE2eProvision.if!.replace( - " && (contains(format(',{0},', inputs.jobs), ',hermes-e2e,') || contains(format(',{0},', inputs.targets), ',hermes-e2e,'))", + " && (github.event_name == 'schedule' || inputs.checkout_sha == '' || (github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && (contains(format(',{0},', inputs.jobs), ',hermes-e2e,') || contains(format(',{0},', inputs.targets), ',hermes-e2e,')))", "", ); @@ -411,8 +547,10 @@ describe("trusted Hermes swap workflow boundary", () => { "agent-turn-latency trusted Hermes swap job must depend on controller validation", "agent-turn-latency trusted Hermes swap step must preserve its fail-closed shape", "agent-turn-latency trusted Hermes swap step must run before candidate checkout", - "hermes-e2e trusted Hermes swap step must preserve the exact-head main guard", - "security-posture trusted Hermes swap step must preserve the exact-head main guard", + "hermes-e2e trusted Hermes swap step must preserve the trusted main guard", + "channels-stop-start trusted Hermes swap step must preserve the trusted main guard", + "common-egress-agent trusted Hermes swap step must preserve the trusted main guard", + "security-posture trusted Hermes swap step must preserve the trusted main guard", "security-posture trusted Hermes swap step must bind only trusted workflow, checkout, and runner identity", "bedrock-runtime-compatible-anthropic trusted Hermes swap step must preserve the fixed privileged program", "mcp-bridge-dev job must not provision trusted Hermes swap", diff --git a/tools/e2e/runner-comparison-workflow-boundary.mts b/tools/e2e/runner-comparison-workflow-boundary.mts index d43c100f02c..df2e87d3081 100644 --- a/tools/e2e/runner-comparison-workflow-boundary.mts +++ b/tools/e2e/runner-comparison-workflow-boundary.mts @@ -8,6 +8,7 @@ import { UPLOAD_E2E_ARTIFACTS_ACTION } from "./upload-e2e-artifacts-workflow-bou export const RUNNER_COMPARISON_INITIALIZE_STEP = "Initialize runner comparison telemetry"; export const RUNNER_COMPARISON_FINALIZE_STEP = "Finalize runner comparison telemetry"; export const RUNNER_COMPARISON_COMMAND = "npx tsx tools/e2e/runner-comparison.mts"; +export const HERMES_REBUILD_SWAP_STEP = "Add swap for Hermes image rebuild"; const TRUSTED_MAIN_GUARD = "github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == ''"; @@ -60,6 +61,7 @@ const COMPARISON_JOBS: ReadonlyMap; type WorkflowStep = WorkflowRecord & { @@ -136,8 +138,9 @@ function publicationIndex(jobSteps: readonly WorkflowStep[]): number { /** * Keep the #7145 comparison to 12 routed workflow lane identities / 15 * concrete trusted-main job executions. Telemetry is best-effort, but it must - * span the complete post-prepare job and finish before evidence is scanned or - * uploaded. + * span the complete stable-capacity job and finish before evidence is scanned + * or uploaded. Rebuild jobs establish their fixed swap capacity first because + * the v2 ledger rejects capacity changes after initialization. */ export function validateRunnerComparisonWorkflow(workflowValue: unknown): string[] { const jobs = record(record(workflowValue).jobs); @@ -196,7 +199,14 @@ export function validateRunnerComparisonWorkflow(workflowValue: unknown): string const initializeIndex = jobSteps.indexOf(initialize); const finalizeIndex = jobSteps.indexOf(finalize); const publish = publicationIndex(jobSteps); - if (prepare < 0 || initializeIndex !== prepare + 1) { + if (HERMES_REBUILD_SWAP_JOBS.has(jobId)) { + const swapIndex = jobSteps.findIndex((step) => step.name === HERMES_REBUILD_SWAP_STEP); + if (prepare < 0 || swapIndex !== prepare + 1 || initializeIndex !== swapIndex + 1) { + errors.push( + `${jobId} must establish rebuild swap before initializing runner comparison telemetry`, + ); + } + } else if (prepare < 0 || initializeIndex !== prepare + 1) { errors.push( `${jobId} must initialize runner comparison telemetry immediately after prepare-e2e`, ); diff --git a/tools/e2e/trusted-hermes-swap-workflow-boundary.mts b/tools/e2e/trusted-hermes-swap-workflow-boundary.mts index 954e49c1b01..09ee502b6a2 100644 --- a/tools/e2e/trusted-hermes-swap-workflow-boundary.mts +++ b/tools/e2e/trusted-hermes-swap-workflow-boundary.mts @@ -18,9 +18,10 @@ export const TRUSTED_HERMES_SWAP_STEP_NAME = "Provision trusted Hermes E2E swap" export const TRUSTED_HERMES_SWAP_STEP_ID = "trusted_hermes_swap"; const TRUSTED_HERMES_SWAP_IF = - "github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != ''"; + "github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')"; const TRUSTED_HERMES_E2E_SELECTION = "(contains(format(',{0},', inputs.jobs), ',hermes-e2e,') || contains(format(',{0},', inputs.targets), ',hermes-e2e,'))"; +const TRUSTED_HERMES_E2E_ELIGIBILITY = `(github.event_name == 'schedule' || inputs.checkout_sha == '' || (github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && ${TRUSTED_HERMES_E2E_SELECTION}))`; const TRUSTED_HERMES_SWAP_SHELL = "/bin/bash --noprofile --norc -e -o pipefail {0}"; const TRUSTED_HERMES_SWAP_ENV = { BASH_ENV: "/dev/null", @@ -54,14 +55,28 @@ export const TRUSTED_HERMES_SWAP_SCRIPT = [ " exit 1", "}", "", - 'if [[ "${REPOSITORY}" != "NVIDIA/NemoClaw" || "${EVENT_NAME}" != "workflow_dispatch" || "${REF}" != "refs/heads/main" ]]; then', + 'if [[ "${REPOSITORY}" != "NVIDIA/NemoClaw" || "${REF}" != "refs/heads/main" ]]; then', ' fail "workflow must run from NVIDIA/NemoClaw main"', "fi", - 'if [[ ! "${CHECKOUT_SHA}" =~ ^[0-9a-f]{40}$ ]]; then', - ' fail "checkout SHA must be lowercase 40-hex"', + 'if [[ "${EVENT_NAME}" != "schedule" && "${EVENT_NAME}" != "workflow_dispatch" ]]; then', + ' fail "workflow event must be schedule or workflow_dispatch"', "fi", - 'if [[ ! "${EXPECTED_WORKFLOW_SHA}" =~ ^[0-9a-f]{40}$ || "${WORKFLOW_SHA}" != "${EXPECTED_WORKFLOW_SHA}" || "${WORKFLOW_SHA}" != "${DISPATCH_SHA}" ]]; then', - ' fail "workflow source must match the trusted dispatch revision"', + "# Exact-head mode: controller-dispatched PR revision.", + 'if [[ "${EVENT_NAME}" == "workflow_dispatch" && -n "${CHECKOUT_SHA}" ]]; then', + ' if [[ ! "${CHECKOUT_SHA}" =~ ^[0-9a-f]{40}$ ]]; then', + ' fail "checkout SHA must be lowercase 40-hex"', + " fi", + ' if [[ ! "${EXPECTED_WORKFLOW_SHA}" =~ ^[0-9a-f]{40}$ || "${WORKFLOW_SHA}" != "${EXPECTED_WORKFLOW_SHA}" || "${WORKFLOW_SHA}" != "${DISPATCH_SHA}" ]]; then', + ' fail "workflow source must match the trusted dispatch revision"', + " fi", + "else", + " # Direct-main mode: schedule or manual trigger on main.", + ' if [[ -n "${CHECKOUT_SHA}" || -n "${EXPECTED_WORKFLOW_SHA}" ]]; then', + ' fail "direct main runs must not request an alternate checkout or workflow revision"', + " fi", + ' if [[ ! "${WORKFLOW_SHA}" =~ ^[0-9a-f]{40}$ || "${WORKFLOW_SHA}" != "${DISPATCH_SHA}" ]]; then', + ' fail "direct main workflow source must match the run revision"', + " fi", "fi", 'if [[ "${RUNNER_ENVIRONMENT_KIND}" != "github-hosted" || "${RUNNER_OS_KIND}" != "Linux" || "${RUNNER_ARCH_KIND}" != "X64" ]]; then', ' fail "swap fallback requires an ephemeral GitHub-hosted Linux x64 runner"', @@ -200,8 +215,11 @@ export const TRUSTED_HERMES_SWAP_SCRIPT = [ const JOB_CONDITIONS = { "agent-turn-latency": `\${{ ${TRUSTED_HERMES_SWAP_IF} }}`, "bedrock-runtime-compatible-anthropic": `\${{ ${TRUSTED_HERMES_SWAP_IF} && matrix.agent == 'hermes' }}`, + "channels-stop-start": `\${{ ${TRUSTED_HERMES_SWAP_IF} && matrix.agent == 'hermes' }}`, + "common-egress-agent": `\${{ ${TRUSTED_HERMES_SWAP_IF} && matrix.scenario == 'hermes-open-reference' }}`, "hermes-dashboard": `\${{ ${TRUSTED_HERMES_SWAP_IF} }}`, - "hermes-e2e": `\${{ ${TRUSTED_HERMES_SWAP_IF} && ${TRUSTED_HERMES_E2E_SELECTION} }}`, + "hermes-discord": `\${{ ${TRUSTED_HERMES_SWAP_IF} }}`, + "hermes-e2e": `\${{ ${TRUSTED_HERMES_SWAP_IF} && ${TRUSTED_HERMES_E2E_ELIGIBILITY} }}`, "hermes-inference-switch": `\${{ ${TRUSTED_HERMES_SWAP_IF} }}`, "hermes-shields-config": `\${{ ${TRUSTED_HERMES_SWAP_IF} }}`, "mcp-bridge": `\${{ ${TRUSTED_HERMES_SWAP_IF} && matrix.agent == 'hermes' }}`, @@ -259,7 +277,7 @@ export function validateTrustedHermesSwapWorkflow(workflowValue: unknown): strin errors.push(`${jobName} trusted Hermes swap step must preserve its fixed name`); } if (provision.if !== expectedCondition) { - errors.push(`${jobName} trusted Hermes swap step must preserve the exact-head main guard`); + errors.push(`${jobName} trusted Hermes swap step must preserve the trusted main guard`); } if (provision.shell !== TRUSTED_HERMES_SWAP_SHELL) { errors.push(`${jobName} trusted Hermes swap step must use the isolated Bash shell`);