Skip to content
Merged
140 changes: 107 additions & 33 deletions .github/workflows/e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -653,7 +653,7 @@ jobs:
steps:
- id: trusted_hermes_swap
name: Provision trusted Hermes E2E swap
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' && matrix.agent == 'hermes' }}
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && matrix.agent == 'hermes' }}
shell: /bin/bash --noprofile --norc -e -o pipefail {0}
env:
BASH_ENV: /dev/null
Expand Down Expand Up @@ -685,14 +685,28 @@ jobs:
exit 1
}

if [[ "${REPOSITORY}" != "NVIDIA/NemoClaw" || "${EVENT_NAME}" != "workflow_dispatch" || "${REF}" != "refs/heads/main" ]]; then
if [[ "${REPOSITORY}" != "NVIDIA/NemoClaw" || "${REF}" != "refs/heads/main" ]]; then
fail "workflow must run from NVIDIA/NemoClaw main"
fi
if [[ ! "${CHECKOUT_SHA}" =~ ^[0-9a-f]{40}$ ]]; then
fail "checkout SHA must be lowercase 40-hex"
if [[ "${EVENT_NAME}" != "schedule" && "${EVENT_NAME}" != "workflow_dispatch" ]]; then
fail "workflow event must be schedule or workflow_dispatch"
fi
if [[ ! "${EXPECTED_WORKFLOW_SHA}" =~ ^[0-9a-f]{40}$ || "${WORKFLOW_SHA}" != "${EXPECTED_WORKFLOW_SHA}" || "${WORKFLOW_SHA}" != "${DISPATCH_SHA}" ]]; then
fail "workflow source must match the trusted dispatch revision"
# Exact-head mode: controller-dispatched PR revision.
if [[ "${EVENT_NAME}" == "workflow_dispatch" && -n "${CHECKOUT_SHA}" ]]; then
if [[ ! "${CHECKOUT_SHA}" =~ ^[0-9a-f]{40}$ ]]; then
fail "checkout SHA must be lowercase 40-hex"
fi
if [[ ! "${EXPECTED_WORKFLOW_SHA}" =~ ^[0-9a-f]{40}$ || "${WORKFLOW_SHA}" != "${EXPECTED_WORKFLOW_SHA}" || "${WORKFLOW_SHA}" != "${DISPATCH_SHA}" ]]; then
fail "workflow source must match the trusted dispatch revision"
fi
else
# Direct-main mode: schedule or manual trigger on main.
if [[ -n "${CHECKOUT_SHA}" || -n "${EXPECTED_WORKFLOW_SHA}" ]]; then
fail "direct main runs must not request an alternate checkout or workflow revision"
fi
if [[ ! "${WORKFLOW_SHA}" =~ ^[0-9a-f]{40}$ || "${WORKFLOW_SHA}" != "${DISPATCH_SHA}" ]]; then
fail "direct main workflow source must match the run revision"
fi
fi
if [[ "${RUNNER_ENVIRONMENT_KIND}" != "github-hosted" || "${RUNNER_OS_KIND}" != "Linux" || "${RUNNER_ARCH_KIND}" != "X64" ]]; then
fail "swap fallback requires an ephemeral GitHub-hosted Linux x64 runner"
Expand Down Expand Up @@ -1332,7 +1346,7 @@ jobs:
steps:
- id: trusted_hermes_swap
name: Provision trusted Hermes E2E swap
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' }}
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }}
shell: /bin/bash --noprofile --norc -e -o pipefail {0}
env:
BASH_ENV: /dev/null
Expand Down Expand Up @@ -1474,7 +1488,7 @@ jobs:
steps:
- id: trusted_hermes_swap
name: Provision trusted Hermes E2E swap
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' }}
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }}
shell: /bin/bash --noprofile --norc -e -o pipefail {0}
env:
BASH_ENV: /dev/null
Expand Down Expand Up @@ -1981,7 +1995,7 @@ jobs:
steps:
- id: trusted_hermes_swap
name: Provision trusted Hermes E2E swap
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' && (contains(format(',{0},', inputs.jobs), ',hermes-e2e,') || contains(format(',{0},', inputs.targets), ',hermes-e2e,')) }}
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && (github.event_name == 'schedule' || inputs.checkout_sha == '' || (github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && (contains(format(',{0},', inputs.jobs), ',hermes-e2e,') || contains(format(',{0},', inputs.targets), ',hermes-e2e,')))) }}
shell: /bin/bash --noprofile --norc -e -o pipefail {0}
env:
BASH_ENV: /dev/null
Expand Down Expand Up @@ -2321,7 +2335,7 @@ jobs:
steps:
- id: trusted_hermes_swap
name: Provision trusted Hermes E2E swap
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' }}
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }}
shell: /bin/bash --noprofile --norc -e -o pipefail {0}
env:
BASH_ENV: /dev/null
Expand Down Expand Up @@ -2394,6 +2408,26 @@ jobs:
NEMOCLAW_POLICY_TIER: open
OPENSHELL_GATEWAY: nemoclaw
steps:
- id: trusted_hermes_swap
name: Provision trusted Hermes E2E swap
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }}
shell: /bin/bash --noprofile --norc -e -o pipefail {0}
env:
BASH_ENV: /dev/null
CHECKOUT_SHA: ${{ inputs.checkout_sha }}
DISPATCH_SHA: ${{ github.sha }}
ENV: /dev/null
EVENT_NAME: ${{ github.event_name }}
EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }}
LC_ALL: C
REF: ${{ github.ref }}
REPOSITORY: ${{ github.repository }}
RUNNER_ARCH_KIND: ${{ runner.arch }}
RUNNER_ENVIRONMENT_KIND: ${{ runner.environment }}
RUNNER_OS_KIND: ${{ runner.os }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
run: *trusted-hermes-e2e-swap

- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
ref: ${{ inputs.checkout_sha || github.sha }}
Expand Down Expand Up @@ -2550,6 +2584,26 @@ jobs:
NEMOCLAW_RECREATE_SANDBOX: "1"
OPENSHELL_GATEWAY: nemoclaw
steps:
- id: trusted_hermes_swap
name: Provision trusted Hermes E2E swap
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && matrix.scenario == 'hermes-open-reference' }}
shell: /bin/bash --noprofile --norc -e -o pipefail {0}
env:
BASH_ENV: /dev/null
CHECKOUT_SHA: ${{ inputs.checkout_sha }}
DISPATCH_SHA: ${{ github.sha }}
ENV: /dev/null
EVENT_NAME: ${{ github.event_name }}
EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }}
LC_ALL: C
REF: ${{ github.ref }}
REPOSITORY: ${{ github.repository }}
RUNNER_ARCH_KIND: ${{ runner.arch }}
RUNNER_ENVIRONMENT_KIND: ${{ runner.environment }}
RUNNER_OS_KIND: ${{ runner.os }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
run: *trusted-hermes-e2e-swap

- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
ref: ${{ inputs.checkout_sha || github.sha }}
Expand Down Expand Up @@ -2687,7 +2741,7 @@ jobs:
steps:
- id: trusted_hermes_swap
name: Provision trusted Hermes E2E swap
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' }}
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }}
shell: /bin/bash --noprofile --norc -e -o pipefail {0}
env:
BASH_ENV: /dev/null
Expand Down Expand Up @@ -2836,6 +2890,16 @@ jobs:
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28

- name: Add swap for Hermes image rebuild
shell: bash
run: |
set -euo pipefail
sudo fallocate -l 32G /mnt/nemoclaw-hermes-rebuild.swap
sudo chmod 0600 /mnt/nemoclaw-hermes-rebuild.swap
sudo mkswap /mnt/nemoclaw-hermes-rebuild.swap
sudo swapon /mnt/nemoclaw-hermes-rebuild.swap
swapon --show

- name: Initialize runner comparison telemetry
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' }}
continue-on-error: true
Expand All @@ -2849,16 +2913,6 @@ jobs:
set -euo pipefail
env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u NVIDIA_INFERENCE_API_KEY -u GITHUB_TOKEN -u GH_TOKEN bash scripts/install-openshell.sh

- name: Add swap for Hermes image rebuild
shell: bash
run: |
set -euo pipefail
sudo fallocate -l 32G /mnt/nemoclaw-hermes-rebuild.swap
sudo chmod 0600 /mnt/nemoclaw-hermes-rebuild.swap
sudo mkswap /mnt/nemoclaw-hermes-rebuild.swap
sudo swapon /mnt/nemoclaw-hermes-rebuild.swap
swapon --show

- name: Run Hermes rebuild live test
# Preserves the checked-out CLI, Docker/OpenShell, Hermes base-image
# rebuild, registry, messaging-placeholder, and backup hygiene boundaries.
Expand Down Expand Up @@ -2950,6 +3004,16 @@ jobs:
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28

- name: Add swap for Hermes image rebuild
shell: bash
run: |
set -euo pipefail
sudo fallocate -l 32G /mnt/nemoclaw-hermes-rebuild.swap
sudo chmod 0600 /mnt/nemoclaw-hermes-rebuild.swap
sudo mkswap /mnt/nemoclaw-hermes-rebuild.swap
sudo swapon /mnt/nemoclaw-hermes-rebuild.swap
swapon --show

- name: Initialize runner comparison telemetry
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' }}
continue-on-error: true
Expand All @@ -2963,16 +3027,6 @@ jobs:
set -euo pipefail
env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u NVIDIA_INFERENCE_API_KEY -u GITHUB_TOKEN -u GH_TOKEN bash scripts/install-openshell.sh

- name: Add swap for Hermes image rebuild
shell: bash
run: |
set -euo pipefail
sudo fallocate -l 32G /mnt/nemoclaw-hermes-rebuild.swap
sudo chmod 0600 /mnt/nemoclaw-hermes-rebuild.swap
sudo mkswap /mnt/nemoclaw-hermes-rebuild.swap
sudo swapon /mnt/nemoclaw-hermes-rebuild.swap
swapon --show

- name: Run Hermes stale-base rebuild live test
# Uses NEMOCLAW_HERMES_STALE_BASE_REBUILD_E2E=1, preserving issue #3025's
# stale cached base-image regression boundary.
Expand Down Expand Up @@ -3655,7 +3709,7 @@ jobs:
steps:
- id: trusted_hermes_swap
name: Provision trusted Hermes E2E swap
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' && matrix.agent == 'hermes' }}
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && matrix.agent == 'hermes' }}
shell: /bin/bash --noprofile --norc -e -o pipefail {0}
env:
BASH_ENV: /dev/null
Expand Down Expand Up @@ -4938,7 +4992,7 @@ jobs:
steps:
- id: trusted_hermes_swap
name: Provision trusted Hermes E2E swap
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' && matrix.agent == 'hermes' }}
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && matrix.agent == 'hermes' }}
shell: /bin/bash --noprofile --norc -e -o pipefail {0}
env:
BASH_ENV: /dev/null
Expand Down Expand Up @@ -5239,6 +5293,26 @@ jobs:
NEMOCLAW_SANDBOX_NAME: e2e-channels-stop-start-${{ matrix.agent }}
OPENSHELL_GATEWAY: "nemoclaw"
steps:
- id: trusted_hermes_swap
name: Provision trusted Hermes E2E swap
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && matrix.agent == 'hermes' }}
shell: /bin/bash --noprofile --norc -e -o pipefail {0}
env:
BASH_ENV: /dev/null
CHECKOUT_SHA: ${{ inputs.checkout_sha }}
DISPATCH_SHA: ${{ github.sha }}
ENV: /dev/null
EVENT_NAME: ${{ github.event_name }}
EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }}
LC_ALL: C
REF: ${{ github.ref }}
REPOSITORY: ${{ github.repository }}
RUNNER_ARCH_KIND: ${{ runner.arch }}
RUNNER_ENVIRONMENT_KIND: ${{ runner.environment }}
RUNNER_OS_KIND: ${{ runner.os }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
run: *trusted-hermes-e2e-swap

- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
ref: ${{ inputs.checkout_sha || github.sha }}
Expand Down
48 changes: 27 additions & 21 deletions test/e2e/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -62,13 +62,16 @@ no alternate checkout SHA is requested. PR-gate dispatches therefore remain on
standard runners even though they use the trusted workflow definition from
`main`.

Exact-head PR-gate dispatches use a bounded swap fallback for the hosted
Hermes image-building lanes that remain on those standard runners. The trusted
workflow provisions the fallback as the first job step, before checking out or
executing the candidate revision. It requires a controller-supplied lowercase
40-hex checkout SHA, matching trusted workflow and dispatch revisions, and an
ephemeral GitHub-hosted Linux x64 runner. Candidate code cannot supply the
program or arguments passed to `sudo`.
Exact-head PR-gate dispatches and direct scheduled or manual `main` runs use a
bounded swap fallback for eligible hosted Hermes image-building lanes. The
fallback does not change runner routing. The trusted workflow provisions the
fallback as the first job step, before checking out or executing the selected
revision. Exact-head mode requires a controller-supplied lowercase 40-hex
checkout SHA plus matching trusted workflow and dispatch revisions. Direct-main
mode rejects alternate checkout and workflow revisions and requires the
workflow source to match the run revision. Both modes require an ephemeral
GitHub-hosted Linux x64 runner. Candidate code cannot supply the program or
arguments passed to `sudo`.

The trusted step requires at least 32 GiB (34,359,738,368 bytes) of usable swap.
It reuses active swap that meets this requirement.
Expand All @@ -87,20 +90,20 @@ Cleanup removes it only after `swapoff` succeeds.
Successful state is discarded with the ephemeral runner.

The fallback covers agent-turn latency, Hermes inference switch and shields,
the Hermes Bedrock and stable MCP shards, and the `hermes-e2e`,
`hermes-dashboard`, and Hermes security-posture tests. Scheduled and ordinary
manual `main` runs, larger-runner executions, rebuild lanes with
workflow-managed swap, dedicated-runner lanes, `mcp-bridge-dev`, and non-Hermes
shards do not use it.
the Hermes Bedrock and stable MCP shards, the Hermes common-egress and channel
stop/start shards, and the `hermes-e2e`, `hermes-dashboard`, `hermes-discord`,
and Hermes security-posture tests. Rebuild lanes with workflow-managed swap,
dedicated-runner lanes, `mcp-bridge-dev`, and non-Hermes shards do not use it.
Candidate-authored workflow definitions and fork-owned runs cannot reach it.

The fallback exists because the alternate-checkout trust boundary deliberately
keeps PR-authored code from selecting the administrator-managed larger-runner
label; changing the PR checkout cannot safely grant itself that capacity.
Remove the fallback only after the trusted controller routes exact-head PR
gates to an ephemeral GitHub-hosted runner with at least 32 GB RAM without
weakening the exact-SHA guard, and five consecutive runs of every protected
lane complete without runner loss while runner-pressure telemetry reports less
than 1 GiB of swap used.
Remove the fallback only after trusted main and exact-head PR runs use
ephemeral GitHub-hosted runners with at least 32 GB RAM without weakening the
source guards, and five consecutive runs of every protected lane complete
without runner loss while runner-pressure telemetry reports less than 1 GiB of
swap used.

The eligible set is limited to the measured or repeatedly interrupted heavy
lanes:
Expand Down Expand Up @@ -183,7 +186,8 @@ The OpenClaw matrix entries for `mcp-bridge`,
Each execution writes one bounded, ordered v2 time series to the canonical
`runner-comparison.jsonl` ledger. It contains:

- an `initialize` endpoint after workspace preparation;
- an `initialize` endpoint after workspace preparation and any fixed-capacity
rebuild swap;
- a distinct `scenario-start` for every test handled by the execution;
- a `periodic` sample on an approximately 60-second fixed cadence;
- a `phase` sample before each semantic phase transition and when the final
Expand All @@ -202,7 +206,9 @@ records exist to reserve the last slot for `finalize`. A missing, historical-v1,
already-finalized, full, or invalid ledger permanently disables comparison
sampling for that test progress instance. In `rebuild-hermes` and
`rebuild-hermes-stale-base`, where legacy phase resource evidence is configured,
the existing five-minute full snapshot then becomes the best-effort fallback.
the workflow establishes its 32 GiB swap before `initialize` so the ledger sees
one stable swap capacity. If canonical sampling becomes unavailable, the
existing five-minute full snapshot becomes the best-effort fallback.
That full profile may run `ps`, `docker stats`, and `docker system df`
sequentially with a 15-second timeout each, or 45 seconds in the worst case;
canonical sampling suppresses this heavier collection while it remains active.
Expand Down Expand Up @@ -801,8 +807,8 @@ memory-heavy image build. The rebuild fixture verifies that floor and
provisions the same swap file on GitHub Actions when a trusted control-plane
run uses the workflow definition from `main`. Those paths build large Hermes
image layers and can otherwise exhaust the runner's default memory and swap
during Docker layer export. Other E2E jobs keep the standard runner memory
configuration except for the exact-head Hermes PR-gate fallback described in
during Docker layer export. Apart from those rebuild and export paths, E2E jobs
add swap only through the trusted Hermes main-workflow fallback described in
[Larger-runner routing](#larger-runner-routing).

These assertions run inside the existing `full-e2e` lifecycle instead of a
Expand Down
Loading
Loading