Skip to content
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 33 additions & 11 deletions .github/workflows/e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ jobs:
test_matrix: ${{ steps.matrix.outputs.test_matrix }}
hermes_selected: ${{ steps.matrix.outputs.hermes_selected }}
explicit_only_jobs: ${{ steps.matrix.outputs.explicit_only_jobs }}
runner_routing: ${{ steps.runner_routing.outputs.runner_routing }}
steps:
- id: controller_matrix
name: Build trusted controller target matrix
Expand All @@ -114,6 +115,27 @@ jobs:
esac
printf 'matrix=%s\n' "${matrix}" >> "${GITHUB_OUTPUT}"

- id: runner_routing
name: Build trusted larger-runner routing
env:
CHECKOUT_SHA: ${{ inputs.checkout_sha }}
LARGER_RUNNER_LABEL: ${{ vars.E2E_LARGER_RUNNER_LABEL }}
REF: ${{ github.ref }}
REPOSITORY: ${{ github.repository }}
shell: bash
run: |
set -euo pipefail
larger_runner="ubuntu-latest"
if [[ "${REPOSITORY}" == "NVIDIA/NemoClaw" && "${REF}" == "refs/heads/main" && -z "${CHECKOUT_SHA}" && -n "${LARGER_RUNNER_LABEL}" ]]; then
if [[ ! "${LARGER_RUNNER_LABEL}" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$ ]]; then
echo "::error::E2E_LARGER_RUNNER_LABEL must be a 1-64 character workflow label using letters, digits, dots, underscores, or hyphens" >&2
exit 1
fi
larger_runner="${LARGER_RUNNER_LABEL}"
fi
runner_routing="$(jq -cn --arg standard "ubuntu-latest" --arg larger "${larger_runner}" '{"channels-stop-start-hermes":$larger,"channels-stop-start-openclaw":$standard,"common-egress-agent":$larger,"hermes-dashboard":$larger,"hermes-discord":$larger,"hermes-e2e":$larger,"hermes-inference-switch":$larger,"hermes-shields-config":$larger,"mcp-bridge-deepagents":$larger,"mcp-bridge-hermes":$larger,"mcp-bridge-openclaw":$standard,"rebuild-hermes":$larger,"rebuild-hermes-stale-base":$larger,"security-posture-hermes":$larger,"security-posture-openclaw":$standard}')"
printf 'runner_routing=%s\n' "${runner_routing}" >> "${GITHUB_OUTPUT}"

- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
ref: ${{ inputs.checkout_sha || github.sha }}
Expand Down Expand Up @@ -480,7 +502,7 @@ jobs:
mcp-bridge:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',mcp-bridge,') || contains(format(',{0},', inputs.targets), ',mcp-bridge,') }}
runs-on: ubuntu-latest
runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)[format('mcp-bridge-{0}', matrix.agent)] }}
permissions:
contents: read
# Keep each destructive agent lifecycle on a fresh runner. This bounds the
Expand Down Expand Up @@ -1077,7 +1099,7 @@ jobs:
hermes-inference-switch:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',hermes-inference-switch,') || contains(format(',{0},', inputs.targets), ',hermes-inference-switch,') }}
runs-on: ubuntu-latest
runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['hermes-inference-switch'] }}
permissions:
contents: read
timeout-minutes: 55
Expand Down Expand Up @@ -1576,7 +1598,7 @@ jobs:
hermes-e2e:
needs: generate-matrix
if: ${{ needs.generate-matrix.outputs.hermes_selected == 'true' }}
runs-on: ubuntu-latest
runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['hermes-e2e'] }}
timeout-minutes: 75
env:
E2E_JOB: "1"
Expand Down Expand Up @@ -1883,7 +1905,7 @@ jobs:
hermes-dashboard:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',hermes-dashboard,') || contains(format(',{0},', inputs.targets), ',hermes-dashboard,') }}
runs-on: ubuntu-latest
runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['hermes-dashboard'] }}
timeout-minutes: 75
env:
E2E_JOB: "1"
Expand Down Expand Up @@ -1929,7 +1951,7 @@ jobs:
hermes-discord:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',hermes-discord,') || contains(format(',{0},', inputs.targets), ',hermes-discord,') }}
runs-on: ubuntu-latest
runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['hermes-discord'] }}
timeout-minutes: 75
env:
E2E_JOB: "1"
Expand Down Expand Up @@ -2043,7 +2065,7 @@ jobs:
common-egress-agent:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',common-egress-agent,') || contains(format(',{0},', inputs.targets), ',common-egress-agent,') }}
runs-on: ubuntu-latest
runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['common-egress-agent'] }}
timeout-minutes: 120
env:
E2E_JOB: "1"
Expand Down Expand Up @@ -2163,7 +2185,7 @@ jobs:
hermes-shields-config:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',hermes-shields-config,') || contains(format(',{0},', inputs.targets), ',hermes-shields-config,') }}
runs-on: ubuntu-latest
runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['hermes-shields-config'] }}
timeout-minutes: 45
env:
E2E_JOB: "1"
Expand Down Expand Up @@ -2265,7 +2287,7 @@ jobs:
rebuild-hermes:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',rebuild-hermes,') || contains(format(',{0},', inputs.targets), ',rebuild-hermes,') }}
runs-on: ubuntu-latest
runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['rebuild-hermes'] }}
timeout-minutes: 90
env:
E2E_JOB: "1"
Expand Down Expand Up @@ -2366,7 +2388,7 @@ jobs:
rebuild-hermes-stale-base:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',rebuild-hermes-stale-base,') || contains(format(',{0},', inputs.targets), ',rebuild-hermes-stale-base,') }}
runs-on: ubuntu-latest
runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['rebuild-hermes-stale-base'] }}
timeout-minutes: 90
env:
E2E_JOB: "1"
Expand Down Expand Up @@ -3055,7 +3077,7 @@ jobs:
contents: read
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',security-posture,') || contains(format(',{0},', inputs.targets), ',security-posture,') }}
runs-on: ubuntu-latest
runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)[format('security-posture-{0}', matrix.agent)] }}
timeout-minutes: 75
strategy:
fail-fast: false
Expand Down Expand Up @@ -4532,7 +4554,7 @@ jobs:
channels-stop-start:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',channels-stop-start,') || contains(format(',{0},', inputs.targets), ',channels-stop-start,') }}
runs-on: ubuntu-latest
runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)[format('channels-stop-start-{0}', matrix.agent)] }}
timeout-minutes: 90
strategy:
fail-fast: false
Expand Down
15 changes: 15 additions & 0 deletions ci/source-shape-test-budget.json
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,21 @@
"test": "rejects free-standing E2E artifact uploads from raw temp paths",
"category": "security"
},
{
"file": "test/e2e/support/larger-runner-routing-workflow-boundary.test.ts",
"test": "keeps every candidate on standard runners when $name (#7145)",
"category": "security"
},
{
"file": "test/e2e/support/larger-runner-routing-workflow-boundary.test.ts",
"test": "rejects malformed administrator workflow labels (#7145)",
"category": "security"
},
{
"file": "test/e2e/support/larger-runner-routing-workflow-boundary.test.ts",
"test": "routes only the measured heavy lanes on trusted main (#7145)",
"category": "security"
},
{
"file": "test/fetch-guard-patch-regression.test.ts",
"test": "requires classifier review and integrity evidence when the OpenClaw build pin changes",
Expand Down
45 changes: 45 additions & 0 deletions test/e2e/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,51 @@ discovery command locally to inspect the generated test matrix:
npx tsx tools/e2e/credential-free-tests.mts
```

## Larger-runner routing

The larger-runner experiment is inactive while the configuration variable
`E2E_LARGER_RUNNER_LABEL` is unset. In that state, every eligible lane continues
to use `ubuntu-latest`. The trusted `generate-matrix` job builds one runner map
before checking out test code, and it consumes the variable only when the
workflow repository is `NVIDIA/NemoClaw`, the ref is `refs/heads/main`, and
no alternate checkout SHA is requested. PR-gate dispatches therefore remain on
standard runners even though they use the trusted workflow definition from
`main`.

The eligible set is limited to the measured or repeatedly interrupted heavy
lanes:

- `common-egress-agent`;
- `hermes-e2e`, `hermes-dashboard`, and `hermes-discord`;
- both `hermes-inference-switch` modes;
- `hermes-shields-config`;
- the Hermes shards of `security-posture` and `channels-stop-start`;
- `rebuild-hermes`;
- `rebuild-hermes-stale-base`;
- the `hermes` and `deepagents` shards of `mcp-bridge`.

The OpenClaw shards of the matrix jobs, the `openclaw` MCP shard, and
`mcp-bridge-dev` remain on `ubuntu-latest`; unrelated jobs retain their
existing runner assignments. Before setting the variable, an organization
owner must:

1. Create a GitHub-hosted Ubuntu x64 larger runner with 8 vCPU, 32 GB RAM, and
300 GB SSD in a dedicated runner group.
2. Set the group maximum concurrency to 4 and restrict repository access to
`NVIDIA/NemoClaw` and workflow access to
`NVIDIA/NemoClaw/.github/workflows/e2e.yaml@refs/heads/main`.
3. Record at least five standard-runner samples for each eligible lane,
including queue time, execution time, peak CPU, memory and disk use,
infrastructure failures, and estimated cost.
4. Copy the larger runner's workflow label into the repository variable, then
repeat the same measurements for at least five representative executions
per migrated lane.

Clearing `E2E_LARGER_RUNNER_LABEL` is the rollback. It sends the eligible lanes
back to `ubuntu-latest` without changing selectors, test setup, or test
semantics. Do not replace this experiment with a persistent self-hosted runner;
that requires a separate decision.

## Scheduled operations

The consolidated workflow keeps its operational reporting in the same job
Expand Down
2 changes: 0 additions & 2 deletions test/e2e/support/hermes-dashboard-workflow-boundary.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ describe("Hermes dashboard workflow boundary", () => {
expect(validateHermesDashboardWorkflowBoundary()).toEqual([]);
const dashboardMode = readHermesDashboardWorkflow();
const dashboardJob = dashboardMode.jobs["hermes-dashboard"];
dashboardJob["runs-on"] = "self-hosted";
dashboardJob["timeout-minutes"] = 30;
dashboardJob.env!.E2E_ARTIFACT_DIR = "/tmp/hermes-dashboard";
dashboardJob.env!.NEMOCLAW_E2E_HERMES_DASHBOARD = "0";
Expand All @@ -25,7 +24,6 @@ describe("Hermes dashboard workflow boundary", () => {
checkout.with!["persist-credentials"] = true;
expect(validateHermesDashboardWorkflow(dashboardMode)).toEqual(
expect.arrayContaining([
"hermes-dashboard must run on ubuntu-latest",
"hermes-dashboard timeout must be 75 minutes",
"hermes-dashboard must use its isolated artifact directory",
"hermes-dashboard must enable Hermes dashboard coverage",
Expand Down
Loading
Loading