From af0942053a87fb34a23a1df099b777104efee9c7 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Fri, 24 Jul 2026 00:51:34 -0700 Subject: [PATCH 1/6] fix(ci): provision Hermes swap for direct main runs Signed-off-by: Apurv Kumaria --- .github/workflows/e2e.yaml | 38 +++-- test/e2e/README.md | 37 +++-- ...sted-hermes-swap-workflow-boundary.test.ts | 152 ++++++++++++++++-- .../trusted-hermes-swap-workflow-boundary.mts | 29 +++- 4 files changed, 204 insertions(+), 52 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 8f5f9adbd97..643ef420386 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -653,7 +653,7 @@ jobs: steps: - id: trusted_hermes_swap name: Provision trusted Hermes E2E swap - if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' && matrix.agent == 'hermes' }} + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && matrix.agent == 'hermes' }} shell: /bin/bash --noprofile --norc -e -o pipefail {0} env: BASH_ENV: /dev/null @@ -685,14 +685,26 @@ jobs: exit 1 } - if [[ "${REPOSITORY}" != "NVIDIA/NemoClaw" || "${EVENT_NAME}" != "workflow_dispatch" || "${REF}" != "refs/heads/main" ]]; then + if [[ "${REPOSITORY}" != "NVIDIA/NemoClaw" || "${REF}" != "refs/heads/main" ]]; then fail "workflow must run from NVIDIA/NemoClaw main" fi - if [[ ! "${CHECKOUT_SHA}" =~ ^[0-9a-f]{40}$ ]]; then - fail "checkout SHA must be lowercase 40-hex" + if [[ "${EVENT_NAME}" != "schedule" && "${EVENT_NAME}" != "workflow_dispatch" ]]; then + fail "workflow event must be schedule or workflow_dispatch" fi - if [[ ! "${EXPECTED_WORKFLOW_SHA}" =~ ^[0-9a-f]{40}$ || "${WORKFLOW_SHA}" != "${EXPECTED_WORKFLOW_SHA}" || "${WORKFLOW_SHA}" != "${DISPATCH_SHA}" ]]; then - fail "workflow source must match the trusted dispatch revision" + if [[ "${EVENT_NAME}" == "workflow_dispatch" && -n "${CHECKOUT_SHA}" ]]; then + if [[ ! "${CHECKOUT_SHA}" =~ ^[0-9a-f]{40}$ ]]; then + fail "checkout SHA must be lowercase 40-hex" + fi + if [[ ! "${EXPECTED_WORKFLOW_SHA}" =~ ^[0-9a-f]{40}$ || "${WORKFLOW_SHA}" != "${EXPECTED_WORKFLOW_SHA}" || "${WORKFLOW_SHA}" != "${DISPATCH_SHA}" ]]; then + fail "workflow source must match the trusted dispatch revision" + fi + else + if [[ -n "${CHECKOUT_SHA}" || -n "${EXPECTED_WORKFLOW_SHA}" ]]; then + fail "direct main runs must not request an alternate checkout or workflow revision" + fi + if [[ ! "${WORKFLOW_SHA}" =~ ^[0-9a-f]{40}$ || "${WORKFLOW_SHA}" != "${DISPATCH_SHA}" ]]; then + fail "direct main workflow source must match the run revision" + fi fi if [[ "${RUNNER_ENVIRONMENT_KIND}" != "github-hosted" || "${RUNNER_OS_KIND}" != "Linux" || "${RUNNER_ARCH_KIND}" != "X64" ]]; then fail "swap fallback requires an ephemeral GitHub-hosted Linux x64 runner" @@ -1332,7 +1344,7 @@ jobs: steps: - id: trusted_hermes_swap name: Provision trusted Hermes E2E swap - if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' }} + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} shell: /bin/bash --noprofile --norc -e -o pipefail {0} env: BASH_ENV: /dev/null @@ -1474,7 +1486,7 @@ jobs: steps: - id: trusted_hermes_swap name: Provision trusted Hermes E2E swap - if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' }} + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} shell: /bin/bash --noprofile --norc -e -o pipefail {0} env: BASH_ENV: /dev/null @@ -1981,7 +1993,7 @@ jobs: steps: - id: trusted_hermes_swap name: Provision trusted Hermes E2E swap - if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' && (contains(format(',{0},', inputs.jobs), ',hermes-e2e,') || contains(format(',{0},', inputs.targets), ',hermes-e2e,')) }} + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && (github.event_name == 'schedule' || inputs.checkout_sha == '' || (github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && (contains(format(',{0},', inputs.jobs), ',hermes-e2e,') || contains(format(',{0},', inputs.targets), ',hermes-e2e,')))) }} shell: /bin/bash --noprofile --norc -e -o pipefail {0} env: BASH_ENV: /dev/null @@ -2321,7 +2333,7 @@ jobs: steps: - id: trusted_hermes_swap name: Provision trusted Hermes E2E swap - if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' }} + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} shell: /bin/bash --noprofile --norc -e -o pipefail {0} env: BASH_ENV: /dev/null @@ -2687,7 +2699,7 @@ jobs: steps: - id: trusted_hermes_swap name: Provision trusted Hermes E2E swap - if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' }} + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} shell: /bin/bash --noprofile --norc -e -o pipefail {0} env: BASH_ENV: /dev/null @@ -3655,7 +3667,7 @@ jobs: steps: - id: trusted_hermes_swap name: Provision trusted Hermes E2E swap - if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' && matrix.agent == 'hermes' }} + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && matrix.agent == 'hermes' }} shell: /bin/bash --noprofile --norc -e -o pipefail {0} env: BASH_ENV: /dev/null @@ -4938,7 +4950,7 @@ jobs: steps: - id: trusted_hermes_swap name: Provision trusted Hermes E2E swap - if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' && matrix.agent == 'hermes' }} + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && matrix.agent == 'hermes' }} shell: /bin/bash --noprofile --norc -e -o pipefail {0} env: BASH_ENV: /dev/null diff --git a/test/e2e/README.md b/test/e2e/README.md index 587467d28ae..208a7c4f29c 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -62,13 +62,16 @@ no alternate checkout SHA is requested. PR-gate dispatches therefore remain on standard runners even though they use the trusted workflow definition from `main`. -Exact-head PR-gate dispatches use a bounded swap fallback for the hosted -Hermes image-building lanes that remain on those standard runners. The trusted -workflow provisions the fallback as the first job step, before checking out or -executing the candidate revision. It requires a controller-supplied lowercase -40-hex checkout SHA, matching trusted workflow and dispatch revisions, and an -ephemeral GitHub-hosted Linux x64 runner. Candidate code cannot supply the -program or arguments passed to `sudo`. +Exact-head PR-gate dispatches and direct scheduled or manual `main` runs use a +bounded swap fallback for eligible hosted Hermes image-building lanes. The +fallback does not change runner routing. The trusted workflow provisions the +fallback as the first job step, before checking out or executing the selected +revision. Exact-head mode requires a controller-supplied lowercase 40-hex +checkout SHA plus matching trusted workflow and dispatch revisions. Direct-main +mode rejects alternate checkout and workflow revisions and requires the +workflow source to match the run revision. Both modes require an ephemeral +GitHub-hosted Linux x64 runner. Candidate code cannot supply the program or +arguments passed to `sudo`. The trusted step requires at least 32 GiB (34,359,738,368 bytes) of usable swap. It reuses active swap that meets this requirement. @@ -88,19 +91,19 @@ Successful state is discarded with the ephemeral runner. The fallback covers agent-turn latency, Hermes inference switch and shields, the Hermes Bedrock and stable MCP shards, and the `hermes-e2e`, -`hermes-dashboard`, and Hermes security-posture tests. Scheduled and ordinary -manual `main` runs, larger-runner executions, rebuild lanes with +`hermes-dashboard`, and Hermes security-posture tests. Rebuild lanes with workflow-managed swap, dedicated-runner lanes, `mcp-bridge-dev`, and non-Hermes -shards do not use it. +shards do not use it. Candidate-authored workflow definitions and fork-owned +runs cannot reach it. The fallback exists because the alternate-checkout trust boundary deliberately keeps PR-authored code from selecting the administrator-managed larger-runner label; changing the PR checkout cannot safely grant itself that capacity. -Remove the fallback only after the trusted controller routes exact-head PR -gates to an ephemeral GitHub-hosted runner with at least 32 GB RAM without -weakening the exact-SHA guard, and five consecutive runs of every protected -lane complete without runner loss while runner-pressure telemetry reports less -than 1 GiB of swap used. +Remove the fallback only after trusted main and exact-head PR runs use +ephemeral GitHub-hosted runners with at least 32 GB RAM without weakening the +source guards, and five consecutive runs of every protected lane complete +without runner loss while runner-pressure telemetry reports less than 1 GiB of +swap used. The eligible set is limited to the measured or repeatedly interrupted heavy lanes: @@ -801,8 +804,8 @@ memory-heavy image build. The rebuild fixture verifies that floor and provisions the same swap file on GitHub Actions when a trusted control-plane run uses the workflow definition from `main`. Those paths build large Hermes image layers and can otherwise exhaust the runner's default memory and swap -during Docker layer export. Other E2E jobs keep the standard runner memory -configuration except for the exact-head Hermes PR-gate fallback described in +during Docker layer export. Other E2E jobs do not add workflow-managed swap +except for the trusted Hermes main-workflow fallback described in [Larger-runner routing](#larger-runner-routing). These assertions run inside the existing `full-e2e` lifecycle instead of a diff --git a/test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts b/test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts index e5240a20385..6628bbecc9e 100644 --- a/test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts +++ b/test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts @@ -52,12 +52,22 @@ function trustedSwapStep(workflow: SwapWorkflow, jobName: string): WorkflowStep type SwapHarnessOptions = { activeSwapBytes?: number; + checkoutSha?: string; + dispatchSha?: string; diskBytes?: number; + eventName?: string; + expectedWorkflowSha?: string; failCleanupQuery?: boolean; failMkswap?: boolean; failSwapoff?: boolean; hiddenActivationReads?: number; provisionedSwapBytes?: number; + ref?: string; + repository?: string; + runnerArch?: string; + runnerEnvironment?: string; + runnerOs?: string; + workflowSha?: string; }; type SwapHarnessResult = { @@ -79,6 +89,7 @@ function runTrustedSwapHarness(options: SwapHarnessOptions = {}): SwapHarnessRes const queryCount = path.join(fakeBin, "query-count"); const swapState = path.join(fakeBin, "swap-state"); const swapFile = "/mnt/nemoclaw-hermes-e2e-swap/nemoclaw-hermes.fake.swap"; + writeFileSync(callLog, ""); writeFileSync(swapState, "inactive\n"); const commands = new Map(); @@ -196,16 +207,18 @@ function runTrustedSwapHarness(options: SwapHarnessOptions = {}): SwapHarnessRes } try { - const workflowSha = "b".repeat(40); + const workflowSha = options.workflowSha ?? "b".repeat(40); + const checkoutSha = options.checkoutSha ?? "a".repeat(40); const result = spawnSync("/bin/bash", ["--noprofile", "--norc", "-c", script], { encoding: "utf8", env: { BASH_ENV: "/dev/null", - CHECKOUT_SHA: "a".repeat(40), - DISPATCH_SHA: workflowSha, + CHECKOUT_SHA: checkoutSha, + DISPATCH_SHA: options.dispatchSha ?? workflowSha, ENV: "/dev/null", - EVENT_NAME: "workflow_dispatch", - EXPECTED_WORKFLOW_SHA: workflowSha, + EVENT_NAME: options.eventName ?? "workflow_dispatch", + EXPECTED_WORKFLOW_SHA: + options.expectedWorkflowSha ?? (checkoutSha === "" ? "" : workflowSha), FAKE_ACTIVE_SWAP_BYTES: String(options.activeSwapBytes ?? 0), FAKE_CALL_LOG: callLog, FAKE_DISK_BYTES: String(options.diskBytes ?? 100_000_000_000), @@ -219,15 +232,16 @@ function runTrustedSwapHarness(options: SwapHarnessOptions = {}): SwapHarnessRes FAKE_SWAP_STATE: swapState, LC_ALL: "C", PATH: "/usr/bin:/bin", - REF: "refs/heads/main", - REPOSITORY: "NVIDIA/NemoClaw", - RUNNER_ARCH_KIND: "X64", - RUNNER_ENVIRONMENT_KIND: "github-hosted", - RUNNER_OS_KIND: "Linux", + REF: options.ref ?? "refs/heads/main", + REPOSITORY: options.repository ?? "NVIDIA/NemoClaw", + RUNNER_ARCH_KIND: options.runnerArch ?? "X64", + RUNNER_ENVIRONMENT_KIND: options.runnerEnvironment ?? "github-hosted", + RUNNER_OS_KIND: options.runnerOs ?? "Linux", WORKFLOW_SHA: workflowSha, }, }); - const calls = readFileSync(callLog, "utf8").trimEnd().split("\n"); + const callContents = readFileSync(callLog, "utf8").trimEnd(); + const calls = callContents === "" ? [] : callContents.split("\n"); return { calls, status: result.status, stderr: result.stderr }; } finally { rmSync(fakeBin, { force: true, recursive: true }); @@ -281,6 +295,116 @@ describe("trusted Hermes swap workflow boundary", () => { expect(TRUSTED_HERMES_SWAP_SCRIPT).not.toContain("${{"); }); + it.each([ + "schedule", + "workflow_dispatch", + ])("accepts the trusted direct main source for %s runs (#7145)", (eventName) => { + const result = runTrustedSwapHarness({ + activeSwapBytes: 34_359_738_368, + checkoutSha: "", + eventName, + }); + + expect(result.status).toBe(0); + expect(result.calls).toEqual([ + "stat:-c %F:%u:%g -- /mnt", + "swapon:--show=SIZE --bytes --noheadings", + ]); + }); + + it.each([ + { + expected: "direct main runs must not request an alternate checkout or workflow revision", + name: "a scheduled run supplies an alternate checkout", + options: { checkoutSha: "a".repeat(40), eventName: "schedule" }, + }, + { + expected: "direct main runs must not request an alternate checkout or workflow revision", + name: "a direct dispatch supplies an alternate workflow revision", + options: { + checkoutSha: "", + expectedWorkflowSha: "b".repeat(40), + }, + }, + { + expected: "direct main workflow source must match the run revision", + name: "the workflow and run revisions diverge", + options: { + checkoutSha: "", + dispatchSha: "c".repeat(40), + }, + }, + { + expected: "direct main workflow source must match the run revision", + name: "the workflow source is malformed", + options: { + checkoutSha: "", + dispatchSha: "b".repeat(40), + workflowSha: "not-a-sha", + }, + }, + ])("rejects trusted direct main mode when $name (#7145)", ({ expected, options }) => { + const result = runTrustedSwapHarness(options); + + expect(result.status).toBe(1); + expect(result.stderr).toContain(expected); + expect(result.calls).toEqual([]); + }); + + it.each([ + { + expected: "workflow must run from NVIDIA/NemoClaw main", + name: "the repository is not canonical", + options: { repository: "example/NemoClaw" }, + }, + { + expected: "workflow must run from NVIDIA/NemoClaw main", + name: "the ref is not main", + options: { ref: "refs/heads/candidate" }, + }, + { + expected: "workflow event must be schedule or workflow_dispatch", + name: "the event is not trusted", + options: { eventName: "pull_request" }, + }, + { + expected: "checkout SHA must be lowercase 40-hex", + name: "the exact-head checkout SHA is malformed", + options: { checkoutSha: "A".repeat(40) }, + }, + { + expected: "workflow source must match the trusted dispatch revision", + name: "the exact-head workflow SHA is missing", + options: { expectedWorkflowSha: "" }, + }, + { + expected: "workflow source must match the trusted dispatch revision", + name: "the exact-head workflow SHA differs", + options: { expectedWorkflowSha: "c".repeat(40) }, + }, + { + expected: "swap fallback requires an ephemeral GitHub-hosted Linux x64 runner", + name: "the runner is self-hosted", + options: { runnerEnvironment: "self-hosted" }, + }, + { + expected: "swap fallback requires an ephemeral GitHub-hosted Linux x64 runner", + name: "the runner OS is not Linux", + options: { runnerOs: "Windows" }, + }, + { + expected: "swap fallback requires an ephemeral GitHub-hosted Linux x64 runner", + name: "the runner architecture is not x64", + options: { runnerArch: "ARM64" }, + }, + ])("rejects identity drift when $name (#7145)", ({ expected, options }) => { + const result = runTrustedSwapHarness(options); + + expect(result.status).toBe(1); + expect(result.stderr).toContain(expected); + expect(result.calls).toEqual([]); + }); + it("exits before privileged allocation when enough swap is already active (#7145)", () => { const result = runTrustedSwapHarness({ activeSwapBytes: 34_359_738_368 }); @@ -398,7 +522,7 @@ describe("trusted Hermes swap workflow boundary", () => { const hermesE2eProvision = trustedSwapStep(workflow, "hermes-e2e"); hermesE2eProvision.if = hermesE2eProvision.if!.replace( - " && (contains(format(',{0},', inputs.jobs), ',hermes-e2e,') || contains(format(',{0},', inputs.targets), ',hermes-e2e,'))", + " && (github.event_name == 'schedule' || inputs.checkout_sha == '' || (github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && (contains(format(',{0},', inputs.jobs), ',hermes-e2e,') || contains(format(',{0},', inputs.targets), ',hermes-e2e,')))", "", ); @@ -411,8 +535,8 @@ describe("trusted Hermes swap workflow boundary", () => { "agent-turn-latency trusted Hermes swap job must depend on controller validation", "agent-turn-latency trusted Hermes swap step must preserve its fail-closed shape", "agent-turn-latency trusted Hermes swap step must run before candidate checkout", - "hermes-e2e trusted Hermes swap step must preserve the exact-head main guard", - "security-posture trusted Hermes swap step must preserve the exact-head main guard", + "hermes-e2e trusted Hermes swap step must preserve the trusted main guard", + "security-posture trusted Hermes swap step must preserve the trusted main guard", "security-posture trusted Hermes swap step must bind only trusted workflow, checkout, and runner identity", "bedrock-runtime-compatible-anthropic trusted Hermes swap step must preserve the fixed privileged program", "mcp-bridge-dev job must not provision trusted Hermes swap", diff --git a/tools/e2e/trusted-hermes-swap-workflow-boundary.mts b/tools/e2e/trusted-hermes-swap-workflow-boundary.mts index 954e49c1b01..d8f33999f90 100644 --- a/tools/e2e/trusted-hermes-swap-workflow-boundary.mts +++ b/tools/e2e/trusted-hermes-swap-workflow-boundary.mts @@ -18,9 +18,10 @@ export const TRUSTED_HERMES_SWAP_STEP_NAME = "Provision trusted Hermes E2E swap" export const TRUSTED_HERMES_SWAP_STEP_ID = "trusted_hermes_swap"; const TRUSTED_HERMES_SWAP_IF = - "github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != ''"; + "github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')"; const TRUSTED_HERMES_E2E_SELECTION = "(contains(format(',{0},', inputs.jobs), ',hermes-e2e,') || contains(format(',{0},', inputs.targets), ',hermes-e2e,'))"; +const TRUSTED_HERMES_E2E_ELIGIBILITY = `(github.event_name == 'schedule' || inputs.checkout_sha == '' || (github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && ${TRUSTED_HERMES_E2E_SELECTION}))`; const TRUSTED_HERMES_SWAP_SHELL = "/bin/bash --noprofile --norc -e -o pipefail {0}"; const TRUSTED_HERMES_SWAP_ENV = { BASH_ENV: "/dev/null", @@ -54,14 +55,26 @@ export const TRUSTED_HERMES_SWAP_SCRIPT = [ " exit 1", "}", "", - 'if [[ "${REPOSITORY}" != "NVIDIA/NemoClaw" || "${EVENT_NAME}" != "workflow_dispatch" || "${REF}" != "refs/heads/main" ]]; then', + 'if [[ "${REPOSITORY}" != "NVIDIA/NemoClaw" || "${REF}" != "refs/heads/main" ]]; then', ' fail "workflow must run from NVIDIA/NemoClaw main"', "fi", - 'if [[ ! "${CHECKOUT_SHA}" =~ ^[0-9a-f]{40}$ ]]; then', - ' fail "checkout SHA must be lowercase 40-hex"', + 'if [[ "${EVENT_NAME}" != "schedule" && "${EVENT_NAME}" != "workflow_dispatch" ]]; then', + ' fail "workflow event must be schedule or workflow_dispatch"', "fi", - 'if [[ ! "${EXPECTED_WORKFLOW_SHA}" =~ ^[0-9a-f]{40}$ || "${WORKFLOW_SHA}" != "${EXPECTED_WORKFLOW_SHA}" || "${WORKFLOW_SHA}" != "${DISPATCH_SHA}" ]]; then', - ' fail "workflow source must match the trusted dispatch revision"', + 'if [[ "${EVENT_NAME}" == "workflow_dispatch" && -n "${CHECKOUT_SHA}" ]]; then', + ' if [[ ! "${CHECKOUT_SHA}" =~ ^[0-9a-f]{40}$ ]]; then', + ' fail "checkout SHA must be lowercase 40-hex"', + " fi", + ' if [[ ! "${EXPECTED_WORKFLOW_SHA}" =~ ^[0-9a-f]{40}$ || "${WORKFLOW_SHA}" != "${EXPECTED_WORKFLOW_SHA}" || "${WORKFLOW_SHA}" != "${DISPATCH_SHA}" ]]; then', + ' fail "workflow source must match the trusted dispatch revision"', + " fi", + "else", + ' if [[ -n "${CHECKOUT_SHA}" || -n "${EXPECTED_WORKFLOW_SHA}" ]]; then', + ' fail "direct main runs must not request an alternate checkout or workflow revision"', + " fi", + ' if [[ ! "${WORKFLOW_SHA}" =~ ^[0-9a-f]{40}$ || "${WORKFLOW_SHA}" != "${DISPATCH_SHA}" ]]; then', + ' fail "direct main workflow source must match the run revision"', + " fi", "fi", 'if [[ "${RUNNER_ENVIRONMENT_KIND}" != "github-hosted" || "${RUNNER_OS_KIND}" != "Linux" || "${RUNNER_ARCH_KIND}" != "X64" ]]; then', ' fail "swap fallback requires an ephemeral GitHub-hosted Linux x64 runner"', @@ -201,7 +214,7 @@ const JOB_CONDITIONS = { "agent-turn-latency": `\${{ ${TRUSTED_HERMES_SWAP_IF} }}`, "bedrock-runtime-compatible-anthropic": `\${{ ${TRUSTED_HERMES_SWAP_IF} && matrix.agent == 'hermes' }}`, "hermes-dashboard": `\${{ ${TRUSTED_HERMES_SWAP_IF} }}`, - "hermes-e2e": `\${{ ${TRUSTED_HERMES_SWAP_IF} && ${TRUSTED_HERMES_E2E_SELECTION} }}`, + "hermes-e2e": `\${{ ${TRUSTED_HERMES_SWAP_IF} && ${TRUSTED_HERMES_E2E_ELIGIBILITY} }}`, "hermes-inference-switch": `\${{ ${TRUSTED_HERMES_SWAP_IF} }}`, "hermes-shields-config": `\${{ ${TRUSTED_HERMES_SWAP_IF} }}`, "mcp-bridge": `\${{ ${TRUSTED_HERMES_SWAP_IF} && matrix.agent == 'hermes' }}`, @@ -259,7 +272,7 @@ export function validateTrustedHermesSwapWorkflow(workflowValue: unknown): strin errors.push(`${jobName} trusted Hermes swap step must preserve its fixed name`); } if (provision.if !== expectedCondition) { - errors.push(`${jobName} trusted Hermes swap step must preserve the exact-head main guard`); + errors.push(`${jobName} trusted Hermes swap step must preserve the trusted main guard`); } if (provision.shell !== TRUSTED_HERMES_SWAP_SHELL) { errors.push(`${jobName} trusted Hermes swap step must use the isolated Bash shell`); From 2bc1936e7065caa21fc7db2f1821612ef5b2d81d Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Fri, 24 Jul 2026 01:29:04 -0700 Subject: [PATCH 2/6] chore(ci): label Hermes swap trust modes Signed-off-by: Apurv Kumaria --- .github/workflows/e2e.yaml | 2 ++ tools/e2e/trusted-hermes-swap-workflow-boundary.mts | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 643ef420386..c670cfcc0fd 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -691,6 +691,7 @@ jobs: if [[ "${EVENT_NAME}" != "schedule" && "${EVENT_NAME}" != "workflow_dispatch" ]]; then fail "workflow event must be schedule or workflow_dispatch" fi + # Exact-head mode: controller-dispatched PR revision. if [[ "${EVENT_NAME}" == "workflow_dispatch" && -n "${CHECKOUT_SHA}" ]]; then if [[ ! "${CHECKOUT_SHA}" =~ ^[0-9a-f]{40}$ ]]; then fail "checkout SHA must be lowercase 40-hex" @@ -699,6 +700,7 @@ jobs: fail "workflow source must match the trusted dispatch revision" fi else + # Direct-main mode: schedule or manual trigger on main. if [[ -n "${CHECKOUT_SHA}" || -n "${EXPECTED_WORKFLOW_SHA}" ]]; then fail "direct main runs must not request an alternate checkout or workflow revision" fi diff --git a/tools/e2e/trusted-hermes-swap-workflow-boundary.mts b/tools/e2e/trusted-hermes-swap-workflow-boundary.mts index d8f33999f90..4a6106e22bd 100644 --- a/tools/e2e/trusted-hermes-swap-workflow-boundary.mts +++ b/tools/e2e/trusted-hermes-swap-workflow-boundary.mts @@ -61,6 +61,7 @@ export const TRUSTED_HERMES_SWAP_SCRIPT = [ 'if [[ "${EVENT_NAME}" != "schedule" && "${EVENT_NAME}" != "workflow_dispatch" ]]; then', ' fail "workflow event must be schedule or workflow_dispatch"', "fi", + "# Exact-head mode: controller-dispatched PR revision.", 'if [[ "${EVENT_NAME}" == "workflow_dispatch" && -n "${CHECKOUT_SHA}" ]]; then', ' if [[ ! "${CHECKOUT_SHA}" =~ ^[0-9a-f]{40}$ ]]; then', ' fail "checkout SHA must be lowercase 40-hex"', @@ -69,6 +70,7 @@ export const TRUSTED_HERMES_SWAP_SCRIPT = [ ' fail "workflow source must match the trusted dispatch revision"', " fi", "else", + " # Direct-main mode: schedule or manual trigger on main.", ' if [[ -n "${CHECKOUT_SHA}" || -n "${EXPECTED_WORKFLOW_SHA}" ]]; then', ' fail "direct main runs must not request an alternate checkout or workflow revision"', " fi", From 3c0bce633310e374bfdffed41a9b7e489633aa53 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Fri, 24 Jul 2026 10:37:00 -0700 Subject: [PATCH 3/6] fix(ci): protect Hermes Discord with trusted swap Signed-off-by: Apurv Kumaria --- .github/workflows/e2e.yaml | 20 +++++++++++++++++++ test/e2e/README.md | 8 ++++---- ...sted-hermes-swap-workflow-boundary.test.ts | 1 + .../trusted-hermes-swap-workflow-boundary.mts | 1 + 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index c670cfcc0fd..99f8f95c217 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -2408,6 +2408,26 @@ jobs: NEMOCLAW_POLICY_TIER: open OPENSHELL_GATEWAY: nemoclaw steps: + - id: trusted_hermes_swap + name: Provision trusted Hermes E2E swap + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} + shell: /bin/bash --noprofile --norc -e -o pipefail {0} + env: + BASH_ENV: /dev/null + CHECKOUT_SHA: ${{ inputs.checkout_sha }} + DISPATCH_SHA: ${{ github.sha }} + ENV: /dev/null + EVENT_NAME: ${{ github.event_name }} + EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }} + LC_ALL: C + REF: ${{ github.ref }} + REPOSITORY: ${{ github.repository }} + RUNNER_ARCH_KIND: ${{ runner.arch }} + RUNNER_ENVIRONMENT_KIND: ${{ runner.environment }} + RUNNER_OS_KIND: ${{ runner.os }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + run: *trusted-hermes-e2e-swap + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: ref: ${{ inputs.checkout_sha || github.sha }} diff --git a/test/e2e/README.md b/test/e2e/README.md index 208a7c4f29c..02d8e8176bd 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -91,10 +91,10 @@ Successful state is discarded with the ephemeral runner. The fallback covers agent-turn latency, Hermes inference switch and shields, the Hermes Bedrock and stable MCP shards, and the `hermes-e2e`, -`hermes-dashboard`, and Hermes security-posture tests. Rebuild lanes with -workflow-managed swap, dedicated-runner lanes, `mcp-bridge-dev`, and non-Hermes -shards do not use it. Candidate-authored workflow definitions and fork-owned -runs cannot reach it. +`hermes-dashboard`, `hermes-discord`, and Hermes security-posture tests. Rebuild +lanes with workflow-managed swap, dedicated-runner lanes, `mcp-bridge-dev`, and +non-Hermes shards do not use it. Candidate-authored workflow definitions and +fork-owned runs cannot reach it. The fallback exists because the alternate-checkout trust boundary deliberately keeps PR-authored code from selecting the administrator-managed larger-runner diff --git a/test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts b/test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts index 6628bbecc9e..6dc5b64a960 100644 --- a/test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts +++ b/test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts @@ -35,6 +35,7 @@ const PROTECTED_JOBS = [ "agent-turn-latency", "bedrock-runtime-compatible-anthropic", "hermes-dashboard", + "hermes-discord", "hermes-e2e", "hermes-inference-switch", "hermes-shields-config", diff --git a/tools/e2e/trusted-hermes-swap-workflow-boundary.mts b/tools/e2e/trusted-hermes-swap-workflow-boundary.mts index 4a6106e22bd..cb3a0a3141d 100644 --- a/tools/e2e/trusted-hermes-swap-workflow-boundary.mts +++ b/tools/e2e/trusted-hermes-swap-workflow-boundary.mts @@ -216,6 +216,7 @@ const JOB_CONDITIONS = { "agent-turn-latency": `\${{ ${TRUSTED_HERMES_SWAP_IF} }}`, "bedrock-runtime-compatible-anthropic": `\${{ ${TRUSTED_HERMES_SWAP_IF} && matrix.agent == 'hermes' }}`, "hermes-dashboard": `\${{ ${TRUSTED_HERMES_SWAP_IF} }}`, + "hermes-discord": `\${{ ${TRUSTED_HERMES_SWAP_IF} }}`, "hermes-e2e": `\${{ ${TRUSTED_HERMES_SWAP_IF} && ${TRUSTED_HERMES_E2E_ELIGIBILITY} }}`, "hermes-inference-switch": `\${{ ${TRUSTED_HERMES_SWAP_IF} }}`, "hermes-shields-config": `\${{ ${TRUSTED_HERMES_SWAP_IF} }}`, From 96bb6152f9054afe89db7e33e1b743e090596d16 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Fri, 24 Jul 2026 10:47:14 -0700 Subject: [PATCH 4/6] fix(ci): initialize rebuild telemetry after swap Signed-off-by: Apurv Kumaria --- .github/workflows/e2e.yaml | 40 +++++++++---------- test/e2e/README.md | 7 +++- ...unner-comparison-workflow-boundary.test.ts | 27 +++++++++++-- .../runner-pressure-workflow-boundary.test.ts | 8 ++++ .../runner-comparison-workflow-boundary.mts | 16 ++++++-- 5 files changed, 70 insertions(+), 28 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 99f8f95c217..454aa32539a 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -2870,6 +2870,16 @@ jobs: - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 + - name: Add swap for Hermes image rebuild + shell: bash + run: | + set -euo pipefail + sudo fallocate -l 32G /mnt/nemoclaw-hermes-rebuild.swap + sudo chmod 0600 /mnt/nemoclaw-hermes-rebuild.swap + sudo mkswap /mnt/nemoclaw-hermes-rebuild.swap + sudo swapon /mnt/nemoclaw-hermes-rebuild.swap + swapon --show + - name: Initialize runner comparison telemetry if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' }} continue-on-error: true @@ -2883,16 +2893,6 @@ jobs: set -euo pipefail env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u NVIDIA_INFERENCE_API_KEY -u GITHUB_TOKEN -u GH_TOKEN bash scripts/install-openshell.sh - - name: Add swap for Hermes image rebuild - shell: bash - run: | - set -euo pipefail - sudo fallocate -l 32G /mnt/nemoclaw-hermes-rebuild.swap - sudo chmod 0600 /mnt/nemoclaw-hermes-rebuild.swap - sudo mkswap /mnt/nemoclaw-hermes-rebuild.swap - sudo swapon /mnt/nemoclaw-hermes-rebuild.swap - swapon --show - - name: Run Hermes rebuild live test # Preserves the checked-out CLI, Docker/OpenShell, Hermes base-image # rebuild, registry, messaging-placeholder, and backup hygiene boundaries. @@ -2984,6 +2984,16 @@ jobs: - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 + - name: Add swap for Hermes image rebuild + shell: bash + run: | + set -euo pipefail + sudo fallocate -l 32G /mnt/nemoclaw-hermes-rebuild.swap + sudo chmod 0600 /mnt/nemoclaw-hermes-rebuild.swap + sudo mkswap /mnt/nemoclaw-hermes-rebuild.swap + sudo swapon /mnt/nemoclaw-hermes-rebuild.swap + swapon --show + - name: Initialize runner comparison telemetry if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' }} continue-on-error: true @@ -2997,16 +3007,6 @@ jobs: set -euo pipefail env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u NVIDIA_INFERENCE_API_KEY -u GITHUB_TOKEN -u GH_TOKEN bash scripts/install-openshell.sh - - name: Add swap for Hermes image rebuild - shell: bash - run: | - set -euo pipefail - sudo fallocate -l 32G /mnt/nemoclaw-hermes-rebuild.swap - sudo chmod 0600 /mnt/nemoclaw-hermes-rebuild.swap - sudo mkswap /mnt/nemoclaw-hermes-rebuild.swap - sudo swapon /mnt/nemoclaw-hermes-rebuild.swap - swapon --show - - name: Run Hermes stale-base rebuild live test # Uses NEMOCLAW_HERMES_STALE_BASE_REBUILD_E2E=1, preserving issue #3025's # stale cached base-image regression boundary. diff --git a/test/e2e/README.md b/test/e2e/README.md index 02d8e8176bd..8323f02784a 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -186,7 +186,8 @@ The OpenClaw matrix entries for `mcp-bridge`, Each execution writes one bounded, ordered v2 time series to the canonical `runner-comparison.jsonl` ledger. It contains: -- an `initialize` endpoint after workspace preparation; +- an `initialize` endpoint after workspace preparation and any fixed-capacity + rebuild swap; - a distinct `scenario-start` for every test handled by the execution; - a `periodic` sample on an approximately 60-second fixed cadence; - a `phase` sample before each semantic phase transition and when the final @@ -205,7 +206,9 @@ records exist to reserve the last slot for `finalize`. A missing, historical-v1, already-finalized, full, or invalid ledger permanently disables comparison sampling for that test progress instance. In `rebuild-hermes` and `rebuild-hermes-stale-base`, where legacy phase resource evidence is configured, -the existing five-minute full snapshot then becomes the best-effort fallback. +the workflow establishes its 32 GiB swap before `initialize` so the ledger sees +one stable swap capacity. If canonical sampling becomes unavailable, the +existing five-minute full snapshot becomes the best-effort fallback. That full profile may run `ps`, `docker stats`, and `docker system df` sequentially with a 15-second timeout each, or 45 seconds in the worst case; canonical sampling suppresses this heavier collection while it remains active. diff --git a/test/e2e/support/runner-comparison-workflow-boundary.test.ts b/test/e2e/support/runner-comparison-workflow-boundary.test.ts index 3afed2d27c1..83380787182 100644 --- a/test/e2e/support/runner-comparison-workflow-boundary.test.ts +++ b/test/e2e/support/runner-comparison-workflow-boundary.test.ts @@ -4,6 +4,7 @@ import { describe, expect, it } from "vitest"; import { + HERMES_REBUILD_SWAP_STEP, RUNNER_COMPARISON_COMMAND, RUNNER_COMPARISON_FINALIZE_STEP, RUNNER_COMPARISON_INITIALIZE_STEP, @@ -40,6 +41,7 @@ const JOBS = [ "rebuild-hermes-stale-base", "security-posture", ] as const; +const REBUILD_JOBS = ["rebuild-hermes", "rebuild-hermes-stale-base"] as const; function loadWorkflow(): Workflow { return structuredClone(readWorkflow()) as Workflow; @@ -156,9 +158,10 @@ describe("runner comparison E2E workflow boundary (#7145)", () => { lateSteps[initializeIndex + 1]!, lateSteps[initializeIndex]!, ]; - expect(validateRunnerComparisonWorkflow(lateInitialize)).toContain( - `${jobId} must initialize runner comparison telemetry immediately after prepare-e2e`, - ); + const expectedInitializeError = REBUILD_JOBS.includes(jobId as (typeof REBUILD_JOBS)[number]) + ? `${jobId} must establish rebuild swap before initializing runner comparison telemetry` + : `${jobId} must initialize runner comparison telemetry immediately after prepare-e2e`; + expect(validateRunnerComparisonWorkflow(lateInitialize)).toContain(expectedInitializeError); const afterPublication = loadWorkflow(); const publicationSteps = afterPublication.jobs[jobId]!.steps; @@ -174,6 +177,24 @@ describe("runner comparison E2E workflow boundary (#7145)", () => { ); }); + it.each( + REBUILD_JOBS, + )("initializes %s telemetry only after workflow-managed swap reaches its final capacity", (jobId) => { + const workflow = loadWorkflow(); + const jobSteps = workflow.jobs[jobId]!.steps; + const swap = step(workflow, jobId, HERMES_REBUILD_SWAP_STEP); + const initialize = step(workflow, jobId, RUNNER_COMPARISON_INITIALIZE_STEP); + const swapIndex = jobSteps.indexOf(swap); + const initializeIndex = jobSteps.indexOf(initialize); + + expect(initializeIndex).toBe(swapIndex + 1); + + [jobSteps[swapIndex], jobSteps[initializeIndex]] = [initialize, swap]; + expect(validateRunnerComparisonWorkflow(workflow)).toContain( + `${jobId} must establish rebuild swap before initializing runner comparison telemetry`, + ); + }); + it("rejects weakened trusted-main and always-run guards", () => { const workflow = loadWorkflow(); step(workflow, "common-egress-agent", RUNNER_COMPARISON_INITIALIZE_STEP).if = diff --git a/test/e2e/support/runner-pressure-workflow-boundary.test.ts b/test/e2e/support/runner-pressure-workflow-boundary.test.ts index f2a497bdfa6..550746b6432 100644 --- a/test/e2e/support/runner-pressure-workflow-boundary.test.ts +++ b/test/e2e/support/runner-pressure-workflow-boundary.test.ts @@ -26,6 +26,12 @@ function swapStep(workflow: Workflow, jobId: (typeof JOBS)[number]): WorkflowSte )!; } +function comparisonInitializeStep(workflow: Workflow, jobId: (typeof JOBS)[number]): WorkflowStep { + return workflow.jobs[jobId]!.steps.find( + (step) => step.name === "Initialize runner comparison telemetry", + )!; +} + describe("runner-pressure E2E workflow boundary (#7146)", () => { it("accepts the canonical Hermes heartbeat and terminal-consumer wiring", () => { expect(validateRunnerPressureWorkflow(loadWorkflow())).toEqual([]); @@ -37,9 +43,11 @@ describe("runner-pressure E2E workflow boundary (#7146)", () => { const workflow = loadWorkflow(); const jobSteps = workflow.jobs[jobId]!.steps; const provision = swapStep(workflow, jobId); + const comparisonInitialize = comparisonInitializeStep(workflow, jobId); const run = runStep(workflow, jobId); expect(provision).toBeDefined(); + expect(jobSteps.indexOf(provision)).toBeLessThan(jobSteps.indexOf(comparisonInitialize)); expect(jobSteps.indexOf(provision)).toBeLessThan(jobSteps.indexOf(run)); expect(provision.run).toContain("fallocate -l 32G /mnt/nemoclaw-hermes-rebuild.swap"); expect(provision.run).toContain("chmod 0600 /mnt/nemoclaw-hermes-rebuild.swap"); diff --git a/tools/e2e/runner-comparison-workflow-boundary.mts b/tools/e2e/runner-comparison-workflow-boundary.mts index d43c100f02c..df2e87d3081 100644 --- a/tools/e2e/runner-comparison-workflow-boundary.mts +++ b/tools/e2e/runner-comparison-workflow-boundary.mts @@ -8,6 +8,7 @@ import { UPLOAD_E2E_ARTIFACTS_ACTION } from "./upload-e2e-artifacts-workflow-bou export const RUNNER_COMPARISON_INITIALIZE_STEP = "Initialize runner comparison telemetry"; export const RUNNER_COMPARISON_FINALIZE_STEP = "Finalize runner comparison telemetry"; export const RUNNER_COMPARISON_COMMAND = "npx tsx tools/e2e/runner-comparison.mts"; +export const HERMES_REBUILD_SWAP_STEP = "Add swap for Hermes image rebuild"; const TRUSTED_MAIN_GUARD = "github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == ''"; @@ -60,6 +61,7 @@ const COMPARISON_JOBS: ReadonlyMap; type WorkflowStep = WorkflowRecord & { @@ -136,8 +138,9 @@ function publicationIndex(jobSteps: readonly WorkflowStep[]): number { /** * Keep the #7145 comparison to 12 routed workflow lane identities / 15 * concrete trusted-main job executions. Telemetry is best-effort, but it must - * span the complete post-prepare job and finish before evidence is scanned or - * uploaded. + * span the complete stable-capacity job and finish before evidence is scanned + * or uploaded. Rebuild jobs establish their fixed swap capacity first because + * the v2 ledger rejects capacity changes after initialization. */ export function validateRunnerComparisonWorkflow(workflowValue: unknown): string[] { const jobs = record(record(workflowValue).jobs); @@ -196,7 +199,14 @@ export function validateRunnerComparisonWorkflow(workflowValue: unknown): string const initializeIndex = jobSteps.indexOf(initialize); const finalizeIndex = jobSteps.indexOf(finalize); const publish = publicationIndex(jobSteps); - if (prepare < 0 || initializeIndex !== prepare + 1) { + if (HERMES_REBUILD_SWAP_JOBS.has(jobId)) { + const swapIndex = jobSteps.findIndex((step) => step.name === HERMES_REBUILD_SWAP_STEP); + if (prepare < 0 || swapIndex !== prepare + 1 || initializeIndex !== swapIndex + 1) { + errors.push( + `${jobId} must establish rebuild swap before initializing runner comparison telemetry`, + ); + } + } else if (prepare < 0 || initializeIndex !== prepare + 1) { errors.push( `${jobId} must initialize runner comparison telemetry immediately after prepare-e2e`, ); From c147c7a8eb649d2fec2d475daf45648abdfafcce Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Fri, 24 Jul 2026 10:53:09 -0700 Subject: [PATCH 5/6] fix(ci): protect remaining Hermes matrix lanes Signed-off-by: Apurv Kumaria --- .github/workflows/e2e.yaml | 40 +++++++++++++++++++ test/e2e/README.md | 10 ++--- ...sted-hermes-swap-workflow-boundary.test.ts | 13 ++++++ .../trusted-hermes-swap-workflow-boundary.mts | 2 + 4 files changed, 60 insertions(+), 5 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 454aa32539a..d158ea160a0 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -2584,6 +2584,26 @@ jobs: NEMOCLAW_RECREATE_SANDBOX: "1" OPENSHELL_GATEWAY: nemoclaw steps: + - id: trusted_hermes_swap + name: Provision trusted Hermes E2E swap + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && matrix.scenario == 'hermes-open-reference' }} + shell: /bin/bash --noprofile --norc -e -o pipefail {0} + env: + BASH_ENV: /dev/null + CHECKOUT_SHA: ${{ inputs.checkout_sha }} + DISPATCH_SHA: ${{ github.sha }} + ENV: /dev/null + EVENT_NAME: ${{ github.event_name }} + EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }} + LC_ALL: C + REF: ${{ github.ref }} + REPOSITORY: ${{ github.repository }} + RUNNER_ARCH_KIND: ${{ runner.arch }} + RUNNER_ENVIRONMENT_KIND: ${{ runner.environment }} + RUNNER_OS_KIND: ${{ runner.os }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + run: *trusted-hermes-e2e-swap + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: ref: ${{ inputs.checkout_sha || github.sha }} @@ -5273,6 +5293,26 @@ jobs: NEMOCLAW_SANDBOX_NAME: e2e-channels-stop-start-${{ matrix.agent }} OPENSHELL_GATEWAY: "nemoclaw" steps: + - id: trusted_hermes_swap + name: Provision trusted Hermes E2E swap + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && matrix.agent == 'hermes' }} + shell: /bin/bash --noprofile --norc -e -o pipefail {0} + env: + BASH_ENV: /dev/null + CHECKOUT_SHA: ${{ inputs.checkout_sha }} + DISPATCH_SHA: ${{ github.sha }} + ENV: /dev/null + EVENT_NAME: ${{ github.event_name }} + EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }} + LC_ALL: C + REF: ${{ github.ref }} + REPOSITORY: ${{ github.repository }} + RUNNER_ARCH_KIND: ${{ runner.arch }} + RUNNER_ENVIRONMENT_KIND: ${{ runner.environment }} + RUNNER_OS_KIND: ${{ runner.os }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + run: *trusted-hermes-e2e-swap + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: ref: ${{ inputs.checkout_sha || github.sha }} diff --git a/test/e2e/README.md b/test/e2e/README.md index 8323f02784a..df3cb02b808 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -90,11 +90,11 @@ Cleanup removes it only after `swapoff` succeeds. Successful state is discarded with the ephemeral runner. The fallback covers agent-turn latency, Hermes inference switch and shields, -the Hermes Bedrock and stable MCP shards, and the `hermes-e2e`, -`hermes-dashboard`, `hermes-discord`, and Hermes security-posture tests. Rebuild -lanes with workflow-managed swap, dedicated-runner lanes, `mcp-bridge-dev`, and -non-Hermes shards do not use it. Candidate-authored workflow definitions and -fork-owned runs cannot reach it. +the Hermes Bedrock and stable MCP shards, the Hermes common-egress and channel +stop/start shards, and the `hermes-e2e`, `hermes-dashboard`, `hermes-discord`, +and Hermes security-posture tests. Rebuild lanes with workflow-managed swap, +dedicated-runner lanes, `mcp-bridge-dev`, and non-Hermes shards do not use it. +Candidate-authored workflow definitions and fork-owned runs cannot reach it. The fallback exists because the alternate-checkout trust boundary deliberately keeps PR-authored code from selecting the administrator-managed larger-runner diff --git a/test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts b/test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts index 6dc5b64a960..d5e1afffcb8 100644 --- a/test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts +++ b/test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts @@ -34,6 +34,8 @@ type SwapWorkflow = { const PROTECTED_JOBS = [ "agent-turn-latency", "bedrock-runtime-compatible-anthropic", + "channels-stop-start", + "common-egress-agent", "hermes-dashboard", "hermes-discord", "hermes-e2e", @@ -521,6 +523,15 @@ describe("trusted Hermes swap workflow boundary", () => { const bedrockProvision = trustedSwapStep(workflow, "bedrock-runtime-compatible-anthropic"); bedrockProvision.run = "sudo bash tools/e2e/live-vitest-invocation.mts"; + const channelsProvision = trustedSwapStep(workflow, "channels-stop-start"); + channelsProvision.if = channelsProvision.if!.replace(" && matrix.agent == 'hermes'", ""); + + const commonEgressProvision = trustedSwapStep(workflow, "common-egress-agent"); + commonEgressProvision.if = commonEgressProvision.if!.replace( + " && matrix.scenario == 'hermes-open-reference'", + "", + ); + const hermesE2eProvision = trustedSwapStep(workflow, "hermes-e2e"); hermesE2eProvision.if = hermesE2eProvision.if!.replace( " && (github.event_name == 'schedule' || inputs.checkout_sha == '' || (github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && (contains(format(',{0},', inputs.jobs), ',hermes-e2e,') || contains(format(',{0},', inputs.targets), ',hermes-e2e,')))", @@ -537,6 +548,8 @@ describe("trusted Hermes swap workflow boundary", () => { "agent-turn-latency trusted Hermes swap step must preserve its fail-closed shape", "agent-turn-latency trusted Hermes swap step must run before candidate checkout", "hermes-e2e trusted Hermes swap step must preserve the trusted main guard", + "channels-stop-start trusted Hermes swap step must preserve the trusted main guard", + "common-egress-agent trusted Hermes swap step must preserve the trusted main guard", "security-posture trusted Hermes swap step must preserve the trusted main guard", "security-posture trusted Hermes swap step must bind only trusted workflow, checkout, and runner identity", "bedrock-runtime-compatible-anthropic trusted Hermes swap step must preserve the fixed privileged program", diff --git a/tools/e2e/trusted-hermes-swap-workflow-boundary.mts b/tools/e2e/trusted-hermes-swap-workflow-boundary.mts index cb3a0a3141d..09ee502b6a2 100644 --- a/tools/e2e/trusted-hermes-swap-workflow-boundary.mts +++ b/tools/e2e/trusted-hermes-swap-workflow-boundary.mts @@ -215,6 +215,8 @@ export const TRUSTED_HERMES_SWAP_SCRIPT = [ const JOB_CONDITIONS = { "agent-turn-latency": `\${{ ${TRUSTED_HERMES_SWAP_IF} }}`, "bedrock-runtime-compatible-anthropic": `\${{ ${TRUSTED_HERMES_SWAP_IF} && matrix.agent == 'hermes' }}`, + "channels-stop-start": `\${{ ${TRUSTED_HERMES_SWAP_IF} && matrix.agent == 'hermes' }}`, + "common-egress-agent": `\${{ ${TRUSTED_HERMES_SWAP_IF} && matrix.scenario == 'hermes-open-reference' }}`, "hermes-dashboard": `\${{ ${TRUSTED_HERMES_SWAP_IF} }}`, "hermes-discord": `\${{ ${TRUSTED_HERMES_SWAP_IF} }}`, "hermes-e2e": `\${{ ${TRUSTED_HERMES_SWAP_IF} && ${TRUSTED_HERMES_E2E_ELIGIBILITY} }}`, From ee734c5a833d1f3ed0711107b8033df0cd72c4b1 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Fri, 24 Jul 2026 11:27:20 -0700 Subject: [PATCH 6/6] test(ci): cover delayed Hermes rebuild swap Signed-off-by: Prekshi Vyas --- test/e2e/README.md | 4 ++-- .../runner-comparison-workflow-boundary.test.ts | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index df3cb02b808..e5db3a8093d 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -807,8 +807,8 @@ memory-heavy image build. The rebuild fixture verifies that floor and provisions the same swap file on GitHub Actions when a trusted control-plane run uses the workflow definition from `main`. Those paths build large Hermes image layers and can otherwise exhaust the runner's default memory and swap -during Docker layer export. Other E2E jobs do not add workflow-managed swap -except for the trusted Hermes main-workflow fallback described in +during Docker layer export. Apart from those rebuild and export paths, E2E jobs +add swap only through the trusted Hermes main-workflow fallback described in [Larger-runner routing](#larger-runner-routing). These assertions run inside the existing `full-e2e` lifecycle instead of a diff --git a/test/e2e/support/runner-comparison-workflow-boundary.test.ts b/test/e2e/support/runner-comparison-workflow-boundary.test.ts index 83380787182..beb3a75e5ee 100644 --- a/test/e2e/support/runner-comparison-workflow-boundary.test.ts +++ b/test/e2e/support/runner-comparison-workflow-boundary.test.ts @@ -195,6 +195,17 @@ describe("runner comparison E2E workflow boundary (#7145)", () => { ); }); + it.each(REBUILD_JOBS)("rejects %s telemetry when rebuild swap is delayed", (jobId) => { + const workflow = loadWorkflow(); + const jobSteps = workflow.jobs[jobId]!.steps; + const swapIndex = jobSteps.indexOf(step(workflow, jobId, HERMES_REBUILD_SWAP_STEP)); + jobSteps.splice(swapIndex, 0, { name: "Unexpected step before rebuild swap" }); + + expect(validateRunnerComparisonWorkflow(workflow)).toContain( + `${jobId} must establish rebuild swap before initializing runner comparison telemetry`, + ); + }); + it("rejects weakened trusted-main and always-run guards", () => { const workflow = loadWorkflow(); step(workflow, "common-egress-agent", RUNNER_COMPARISON_INITIALIZE_STEP).if =