Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 33 additions & 11 deletions .github/workflows/e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,7 @@ jobs:
test_matrix: ${{ steps.matrix.outputs.test_matrix }}
hermes_selected: ${{ steps.matrix.outputs.hermes_selected }}
explicit_only_jobs: ${{ steps.matrix.outputs.explicit_only_jobs }}
runner_routing: ${{ steps.runner_routing.outputs.runner_routing }}
steps:
- id: controller_matrix
name: Build trusted controller target matrix
Expand All @@ -167,6 +168,27 @@ jobs:
esac
printf 'matrix=%s\n' "${matrix}" >> "${GITHUB_OUTPUT}"

- id: runner_routing
name: Build trusted larger-runner routing
env:
CHECKOUT_SHA: ${{ inputs.checkout_sha }}
LARGER_RUNNER_LABEL: ${{ vars.E2E_LARGER_RUNNER_LABEL }}
REF: ${{ github.ref }}
REPOSITORY: ${{ github.repository }}
shell: bash
run: |
set -euo pipefail
larger_runner="ubuntu-latest"
if [[ "${REPOSITORY}" == "NVIDIA/NemoClaw" && "${REF}" == "refs/heads/main" && -z "${CHECKOUT_SHA}" && -n "${LARGER_RUNNER_LABEL}" ]]; then
if [[ ! "${LARGER_RUNNER_LABEL}" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$ ]]; then
echo "::error::E2E_LARGER_RUNNER_LABEL must be a 1-64 character workflow label using letters, digits, dots, underscores, or hyphens" >&2
exit 1
fi
larger_runner="${LARGER_RUNNER_LABEL}"
fi
runner_routing="$(jq -cn --arg standard "ubuntu-latest" --arg larger "${larger_runner}" '{"channels-stop-start-hermes":$larger,"channels-stop-start-openclaw":$standard,"common-egress-agent":$larger,"hermes-dashboard":$larger,"hermes-discord":$larger,"hermes-e2e":$larger,"hermes-inference-switch":$larger,"hermes-shields-config":$larger,"mcp-bridge-deepagents":$larger,"mcp-bridge-hermes":$larger,"mcp-bridge-openclaw":$standard,"rebuild-hermes":$larger,"rebuild-hermes-stale-base":$larger,"security-posture-hermes":$larger,"security-posture-openclaw":$standard}')"
printf 'runner_routing=%s\n' "${runner_routing}" >> "${GITHUB_OUTPUT}"

- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
ref: ${{ inputs.checkout_sha || github.sha }}
Expand Down Expand Up @@ -533,7 +555,7 @@ jobs:
mcp-bridge:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',mcp-bridge,') || contains(format(',{0},', inputs.targets), ',mcp-bridge,') }}
runs-on: ubuntu-latest
runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)[format('mcp-bridge-{0}', matrix.agent)] }}
permissions:
contents: read
# Keep each destructive agent lifecycle on a fresh runner. This bounds the
Expand Down Expand Up @@ -1130,7 +1152,7 @@ jobs:
hermes-inference-switch:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',hermes-inference-switch,') || contains(format(',{0},', inputs.targets), ',hermes-inference-switch,') }}
runs-on: ubuntu-latest
runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['hermes-inference-switch'] }}
permissions:
contents: read
timeout-minutes: 55
Expand Down Expand Up @@ -1629,7 +1651,7 @@ jobs:
hermes-e2e:
needs: generate-matrix
if: ${{ needs.generate-matrix.outputs.hermes_selected == 'true' }}
runs-on: ubuntu-latest
runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['hermes-e2e'] }}
timeout-minutes: 75
env:
E2E_JOB: "1"
Expand Down Expand Up @@ -1936,7 +1958,7 @@ jobs:
hermes-dashboard:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',hermes-dashboard,') || contains(format(',{0},', inputs.targets), ',hermes-dashboard,') }}
runs-on: ubuntu-latest
runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['hermes-dashboard'] }}
timeout-minutes: 75
env:
E2E_JOB: "1"
Expand Down Expand Up @@ -1982,7 +2004,7 @@ jobs:
hermes-discord:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',hermes-discord,') || contains(format(',{0},', inputs.targets), ',hermes-discord,') }}
runs-on: ubuntu-latest
runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['hermes-discord'] }}
timeout-minutes: 75
env:
E2E_JOB: "1"
Expand Down Expand Up @@ -2096,7 +2118,7 @@ jobs:
common-egress-agent:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',common-egress-agent,') || contains(format(',{0},', inputs.targets), ',common-egress-agent,') }}
runs-on: ubuntu-latest
runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['common-egress-agent'] }}
timeout-minutes: 120
env:
E2E_JOB: "1"
Expand Down Expand Up @@ -2216,7 +2238,7 @@ jobs:
hermes-shields-config:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',hermes-shields-config,') || contains(format(',{0},', inputs.targets), ',hermes-shields-config,') }}
runs-on: ubuntu-latest
runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['hermes-shields-config'] }}
timeout-minutes: 45
env:
E2E_JOB: "1"
Expand Down Expand Up @@ -2318,7 +2340,7 @@ jobs:
rebuild-hermes:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',rebuild-hermes,') || contains(format(',{0},', inputs.targets), ',rebuild-hermes,') }}
runs-on: ubuntu-latest
runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['rebuild-hermes'] }}
timeout-minutes: 90
env:
E2E_JOB: "1"
Expand Down Expand Up @@ -2419,7 +2441,7 @@ jobs:
rebuild-hermes-stale-base:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',rebuild-hermes-stale-base,') || contains(format(',{0},', inputs.targets), ',rebuild-hermes-stale-base,') }}
runs-on: ubuntu-latest
runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['rebuild-hermes-stale-base'] }}
timeout-minutes: 90
env:
E2E_JOB: "1"
Expand Down Expand Up @@ -3108,7 +3130,7 @@ jobs:
contents: read
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',security-posture,') || contains(format(',{0},', inputs.targets), ',security-posture,') }}
runs-on: ubuntu-latest
runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)[format('security-posture-{0}', matrix.agent)] }}
timeout-minutes: 75
strategy:
fail-fast: false
Expand Down Expand Up @@ -4585,7 +4607,7 @@ jobs:
channels-stop-start:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',channels-stop-start,') || contains(format(',{0},', inputs.targets), ',channels-stop-start,') }}
runs-on: ubuntu-latest
runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)[format('channels-stop-start-{0}', matrix.agent)] }}
timeout-minutes: 90
strategy:
fail-fast: false
Expand Down
15 changes: 15 additions & 0 deletions ci/source-shape-test-budget.json
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,21 @@
"test": "rejects free-standing E2E artifact uploads from raw temp paths",
"category": "security"
},
{
"file": "test/e2e/support/larger-runner-routing-workflow-boundary.test.ts",
"test": "keeps every candidate on standard runners when $name (#7145)",
"category": "security"
},
{
"file": "test/e2e/support/larger-runner-routing-workflow-boundary.test.ts",
"test": "rejects malformed administrator workflow labels (#7145)",
"category": "security"
},
{
"file": "test/e2e/support/larger-runner-routing-workflow-boundary.test.ts",
"test": "routes only the measured heavy lanes on trusted main (#7145)",
"category": "security"
},
{
"file": "test/fetch-guard-patch-regression.test.ts",
"test": "requires classifier review and integrity evidence when the OpenClaw build pin changes",
Expand Down
66 changes: 66 additions & 0 deletions test/e2e/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,72 @@ discovery command locally to inspect the generated test matrix:
npx tsx tools/e2e/credential-free-tests.mts
```

## Larger-runner routing

The larger-runner experiment is inactive while the configuration variable
`E2E_LARGER_RUNNER_LABEL` is unset. In that state, every eligible lane continues
to use `ubuntu-latest`. The trusted `generate-matrix` job builds one runner map
before checking out test code, and it consumes the variable only when the
workflow repository is `NVIDIA/NemoClaw`, the ref is `refs/heads/main`, and
no alternate checkout SHA is requested. PR-gate dispatches therefore remain on
standard runners even though they use the trusted workflow definition from
`main`.

Exact-head PR-gate dispatches use a bounded swap fallback for the hosted
Hermes image-building lanes that remain on those standard runners. The live
Vitest helper activates the fallback only when GitHub Actions supplies a
validated lowercase 40-hex checkout SHA. It reuses at least 32 GiB of active
swap when available; otherwise, it creates one fixed 32 GiB swap file under
`/mnt` before agent-turn latency, Hermes inference switch and shields, the
Hermes Bedrock and stable MCP shards, or the `hermes-e2e`, `hermes-dashboard`,
and Hermes security-posture tests. Setup failure stops before Vitest. Scheduled
and ordinary manual `main` runs, larger-runner executions, rebuild lanes with
workflow-managed swap, dedicated-runner lanes, `mcp-bridge-dev`, and non-Hermes
shards do not use this fallback.

The fallback exists because the alternate-checkout trust boundary deliberately
keeps PR-authored code from selecting the administrator-managed larger-runner
label; changing the PR checkout cannot safely grant itself that capacity.
Remove the fallback only after the trusted controller routes exact-head PR
gates to an ephemeral GitHub-hosted runner with at least 32 GB RAM without
weakening the exact-SHA guard, and five consecutive runs of every protected
lane complete without runner loss while runner-pressure telemetry reports less
than 1 GiB of swap used.

The eligible set is limited to the measured or repeatedly interrupted heavy
lanes:

- `common-egress-agent`;
- `hermes-e2e`, `hermes-dashboard`, and `hermes-discord`;
- both `hermes-inference-switch` modes;
- `hermes-shields-config`;
- the Hermes shards of `security-posture` and `channels-stop-start`;
- `rebuild-hermes`;
- `rebuild-hermes-stale-base`;
- the `hermes` and `deepagents` shards of `mcp-bridge`.

The OpenClaw shards of the matrix jobs, the `openclaw` MCP shard, and
`mcp-bridge-dev` remain on `ubuntu-latest`; unrelated jobs retain their
existing runner assignments. Before setting the variable, an organization
owner must:

1. Create a GitHub-hosted Ubuntu x64 larger runner with 8 vCPU, 32 GB RAM, and
300 GB SSD in a dedicated runner group.
2. Set the group maximum concurrency to 4 and restrict repository access to
`NVIDIA/NemoClaw` and workflow access to
`NVIDIA/NemoClaw/.github/workflows/e2e.yaml@refs/heads/main`.
3. Record at least five standard-runner samples for each eligible lane,
including queue time, execution time, peak CPU, memory and disk use,
infrastructure failures, and estimated cost.
4. Copy the larger runner's workflow label into the repository variable, then
repeat the same measurements for at least five representative executions
per migrated lane.

Clearing `E2E_LARGER_RUNNER_LABEL` is the rollback. It sends the eligible lanes
back to `ubuntu-latest` without changing selectors, test setup, or test
semantics. Do not replace this experiment with a persistent self-hosted runner;
that requires a separate decision.

## Scheduled operations

The consolidated workflow keeps its operational reporting in the same job
Expand Down
2 changes: 0 additions & 2 deletions test/e2e/support/hermes-dashboard-workflow-boundary.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ describe("Hermes dashboard workflow boundary", () => {
expect(validateHermesDashboardWorkflowBoundary()).toEqual([]);
const dashboardMode = readHermesDashboardWorkflow();
const dashboardJob = dashboardMode.jobs["hermes-dashboard"];
dashboardJob["runs-on"] = "self-hosted";
dashboardJob["timeout-minutes"] = 30;
dashboardJob.env!.E2E_ARTIFACT_DIR = "/tmp/hermes-dashboard";
dashboardJob.env!.NEMOCLAW_E2E_HERMES_DASHBOARD = "0";
Expand All @@ -25,7 +24,6 @@ describe("Hermes dashboard workflow boundary", () => {
checkout.with!["persist-credentials"] = true;
expect(validateHermesDashboardWorkflow(dashboardMode)).toEqual(
expect.arrayContaining([
"hermes-dashboard must run on ubuntu-latest",
"hermes-dashboard timeout must be 75 minutes",
"hermes-dashboard must use its isolated artifact directory",
"hermes-dashboard must enable Hermes dashboard coverage",
Expand Down
Loading
Loading