diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 48a40a99502..b0bdcd439b0 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -91,6 +91,7 @@ jobs: test_matrix: ${{ steps.matrix.outputs.test_matrix }} hermes_selected: ${{ steps.matrix.outputs.hermes_selected }} explicit_only_jobs: ${{ steps.matrix.outputs.explicit_only_jobs }} + runner_routing: ${{ steps.runner_routing.outputs.runner_routing }} steps: - id: controller_matrix name: Build trusted controller target matrix @@ -114,6 +115,26 @@ jobs: esac printf 'matrix=%s\n' "${matrix}" >> "${GITHUB_OUTPUT}" + - id: runner_routing + name: Build trusted larger-runner routing + env: + LARGER_RUNNER_LABEL: ${{ vars.E2E_LARGER_RUNNER_LABEL }} + REF: ${{ github.ref }} + REPOSITORY: ${{ github.repository }} + shell: bash + run: | + set -euo pipefail + larger_runner="ubuntu-latest" + if [[ "${REPOSITORY}" == "NVIDIA/NemoClaw" && "${REF}" == "refs/heads/main" && -n "${LARGER_RUNNER_LABEL}" ]]; then + if [[ ! "${LARGER_RUNNER_LABEL}" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$ ]]; then + echo "::error::E2E_LARGER_RUNNER_LABEL must be a 1-64 character workflow label using letters, digits, dots, underscores, or hyphens" >&2 + exit 1 + fi + larger_runner="${LARGER_RUNNER_LABEL}" + fi + runner_routing="$(jq -cn --arg standard "ubuntu-latest" --arg larger "${larger_runner}" '{"common-egress-agent":$larger,"mcp-bridge-deepagents":$larger,"mcp-bridge-hermes":$larger,"mcp-bridge-openclaw":$standard,"rebuild-hermes":$larger,"rebuild-hermes-stale-base":$larger}')" + printf 'runner_routing=%s\n' "${runner_routing}" >> "${GITHUB_OUTPUT}" + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: ref: ${{ inputs.checkout_sha || github.sha }} @@ -480,7 +501,7 @@ jobs: mcp-bridge: needs: generate-matrix if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',mcp-bridge,') || contains(format(',{0},', inputs.targets), ',mcp-bridge,') }} - runs-on: ubuntu-latest + runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)[format('mcp-bridge-{0}', matrix.agent)] }} permissions: contents: read # Keep each destructive agent lifecycle on a fresh runner. This bounds the @@ -2038,7 +2059,7 @@ jobs: common-egress-agent: needs: generate-matrix if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',common-egress-agent,') || contains(format(',{0},', inputs.targets), ',common-egress-agent,') }} - runs-on: ubuntu-latest + runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['common-egress-agent'] }} timeout-minutes: 120 env: E2E_JOB: "1" @@ -2260,7 +2281,7 @@ jobs: rebuild-hermes: needs: generate-matrix if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',rebuild-hermes,') || contains(format(',{0},', inputs.targets), ',rebuild-hermes,') }} - runs-on: ubuntu-latest + runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['rebuild-hermes'] }} timeout-minutes: 90 env: E2E_JOB: "1" @@ -2335,7 +2356,7 @@ jobs: rebuild-hermes-stale-base: needs: generate-matrix if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',rebuild-hermes-stale-base,') || contains(format(',{0},', inputs.targets), ',rebuild-hermes-stale-base,') }} - runs-on: ubuntu-latest + runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['rebuild-hermes-stale-base'] }} timeout-minutes: 90 env: E2E_JOB: "1" diff --git a/ci/source-shape-test-budget.json b/ci/source-shape-test-budget.json index 57d1ac7d22c..a46282185be 100644 --- a/ci/source-shape-test-budget.json +++ b/ci/source-shape-test-budget.json @@ -256,6 +256,21 @@ "test": "rejects free-standing E2E artifact uploads from raw temp paths", "category": "security" }, + { + "file": "test/e2e/support/larger-runner-routing-workflow-boundary.test.ts", + "test": "keeps every candidate on standard runners when $name (#7145)", + "category": "security" + }, + { + "file": "test/e2e/support/larger-runner-routing-workflow-boundary.test.ts", + "test": "rejects malformed administrator workflow labels (#7145)", + "category": "security" + }, + { + "file": "test/e2e/support/larger-runner-routing-workflow-boundary.test.ts", + "test": "routes only the measured heavy lanes on trusted main (#7145)", + "category": "security" + }, { "file": "test/fetch-guard-patch-regression.test.ts", "test": "requires classifier review and integrity evidence when the OpenClaw build pin changes", diff --git a/test/e2e/README.md b/test/e2e/README.md index 01fc9007c0c..a4df5cea099 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -51,6 +51,42 @@ discovery command locally to inspect the generated test matrix: npx tsx tools/e2e/credential-free-tests.mts ``` +## Larger-runner routing + +The larger-runner experiment is inactive while the configuration variable +`E2E_LARGER_RUNNER_LABEL` is unset. In that state, every eligible lane continues +to use `ubuntu-latest`. The trusted `generate-matrix` job builds one runner map +before checking out test code, and it consumes the variable only when the +workflow repository is `NVIDIA/NemoClaw` and the ref is `refs/heads/main`. + +The initial eligible set is limited to the measured heavy lanes: + +- `common-egress-agent`; +- `rebuild-hermes`; +- `rebuild-hermes-stale-base`; +- the `hermes` and `deepagents` shards of `mcp-bridge`. + +The `openclaw` MCP shard and `mcp-bridge-dev` remain on `ubuntu-latest`; +unrelated jobs retain their existing runner assignments. Before setting the +variable, an organization owner must: + +1. Create a GitHub-hosted Ubuntu x64 larger runner with 8 vCPU, 32 GB RAM, and + 300 GB SSD in a dedicated runner group. +2. Set the group maximum concurrency to 4 and restrict repository access to + `NVIDIA/NemoClaw` and workflow access to + `NVIDIA/NemoClaw/.github/workflows/e2e.yaml@refs/heads/main`. +3. Record at least five standard-runner samples for each eligible lane, + including queue time, execution time, peak CPU, memory and disk use, + infrastructure failures, and estimated cost. +4. Copy the larger runner's workflow label into the repository variable, then + repeat the same measurements for at least five representative executions + per migrated lane. + +Clearing `E2E_LARGER_RUNNER_LABEL` is the rollback. It sends the eligible lanes +back to `ubuntu-latest` without changing selectors, test setup, or test +semantics. Do not replace this experiment with a persistent self-hosted runner; +that requires a separate decision. + ## Scheduled operations The consolidated workflow keeps its operational reporting in the same job diff --git a/test/e2e/support/larger-runner-routing-workflow-boundary.test.ts b/test/e2e/support/larger-runner-routing-workflow-boundary.test.ts new file mode 100644 index 00000000000..77f55ad6bac --- /dev/null +++ b/test/e2e/support/larger-runner-routing-workflow-boundary.test.ts @@ -0,0 +1,177 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import { validateE2eWorkflow } from "../../../tools/e2e/workflow-boundary.mts"; +import { readWorkflow } from "../../helpers/e2e-workflow-contract"; +import { requireFixture } from "./require-fixture"; + +type WorkflowStep = { + env?: Record; + id?: string; + name?: string; + run?: string; + uses?: string; +}; + +type RoutingWorkflow = { + jobs: Record< + string, + { + outputs?: Record; + steps?: WorkflowStep[]; + "runs-on"?: string; + } + >; +}; + +function routingStep(workflow: RoutingWorkflow): WorkflowStep { + const step = workflow.jobs["generate-matrix"]?.steps?.find( + (candidate) => candidate.id === "runner_routing", + ); + requireFixture(step?.run, "trusted larger-runner routing step is missing"); + return step; +} + +function evaluateRouting( + workflow: RoutingWorkflow, + env: { label: string; ref: string; repository: string }, +): Record { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-runner-routing-")); + const outputPath = path.join(directory, "github-output"); + try { + execFileSync("bash", ["-c", routingStep(workflow).run!], { + env: { + ...process.env, + GITHUB_OUTPUT: outputPath, + LARGER_RUNNER_LABEL: env.label, + REF: env.ref, + REPOSITORY: env.repository, + }, + }); + const output = fs.readFileSync(outputPath, "utf8").trim(); + requireFixture(output.startsWith("runner_routing="), "runner routing output is missing"); + return JSON.parse(output.slice("runner_routing=".length)) as Record; + } finally { + fs.rmSync(directory, { force: true, recursive: true }); + } +} + +const standardRouting = { + "common-egress-agent": "ubuntu-latest", + "mcp-bridge-deepagents": "ubuntu-latest", + "mcp-bridge-hermes": "ubuntu-latest", + "mcp-bridge-openclaw": "ubuntu-latest", + "rebuild-hermes": "ubuntu-latest", + "rebuild-hermes-stale-base": "ubuntu-latest", +}; + +describe("larger-runner workflow routing boundary", () => { + // source-shape-contract: security -- Executes the shipped pre-checkout router to prove untrusted and unset configurations stay on standard runners + it.each([ + { + label: "", + name: "the administrator label is unset", + ref: "refs/heads/main", + repository: "NVIDIA/NemoClaw", + }, + { + label: "ubuntu-24.04-8core", + name: "the workflow is not running from main", + ref: "refs/heads/feature", + repository: "NVIDIA/NemoClaw", + }, + { + label: "ubuntu-24.04-8core", + name: "the workflow belongs to another repository", + ref: "refs/heads/main", + repository: "someone/NemoClaw", + }, + ])("keeps every candidate on standard runners when $name (#7145)", ({ + label, + ref, + repository, + }) => { + expect(evaluateRouting(readWorkflow() as RoutingWorkflow, { label, ref, repository })).toEqual( + standardRouting, + ); + }); + + // source-shape-contract: security -- Executes the shipped pre-checkout router to prove trusted main can reach only the reviewed heavy lanes + it("routes only the measured heavy lanes on trusted main (#7145)", () => { + const largerRunner = "ubuntu-24.04-8core"; + expect( + evaluateRouting(readWorkflow() as RoutingWorkflow, { + label: largerRunner, + ref: "refs/heads/main", + repository: "NVIDIA/NemoClaw", + }), + ).toEqual({ + ...standardRouting, + "common-egress-agent": largerRunner, + "mcp-bridge-deepagents": largerRunner, + "mcp-bridge-hermes": largerRunner, + "rebuild-hermes": largerRunner, + "rebuild-hermes-stale-base": largerRunner, + }); + }); + + // source-shape-contract: security -- Executes the shipped pre-checkout router to prove malformed administrator labels fail before job routing + it("rejects malformed administrator workflow labels (#7145)", () => { + expect(() => + evaluateRouting(readWorkflow() as RoutingWorkflow, { + label: "invalid runner\nlabel", + ref: "refs/heads/main", + repository: "NVIDIA/NemoClaw", + }), + ).toThrow(); + }); + + it("rejects moving or weakening the trusted pre-checkout map (#7145)", () => { + const workflow = readWorkflow() as RoutingWorkflow; + const generate = workflow.jobs["generate-matrix"]; + const steps = generate.steps!; + const routing = routingStep(workflow); + generate.outputs!.runner_routing = "${{ steps.matrix.outputs.runner_routing }}"; + routing.env!.REF = "${{ inputs.base_sha }}"; + routing.run = routing.run!.replace('"${REF}" == "refs/heads/main" && ', ""); + steps.splice(steps.indexOf(routing), 1); + steps.push(routing); + + expect(validateE2eWorkflow(workflow)).toEqual( + expect.arrayContaining([ + "generate-matrix job must expose the trusted larger-runner routing output", + "trusted larger-runner routing step must bind only the administrator label and trusted repository identity", + "trusted larger-runner routing step must preserve the exact main-only map and ubuntu-latest fallback", + "trusted larger-runner routing step must run before PR checkout", + ]), + ); + }); + + it("rejects routing any lane outside the centralized eligible set (#7145)", () => { + const workflow = readWorkflow() as RoutingWorkflow; + workflow.jobs["common-egress-agent"]["runs-on"] = "ubuntu-latest"; + workflow.jobs["mcp-bridge"]["runs-on"] = "ubuntu-latest"; + workflow.jobs["mcp-bridge-dev"]["runs-on"] = + "${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['mcp-bridge-deepagents'] }}"; + workflow.jobs["network-policy"]["runs-on"] = "${{ vars.E2E_LARGER_RUNNER_LABEL }}"; + workflow.jobs["shields-config"]["runs-on"] = + "${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['common-egress-agent'] }}"; + + expect(validateE2eWorkflow(workflow)).toEqual( + expect.arrayContaining([ + "common-egress-agent job must use the trusted larger-runner routing map", + "mcp-bridge job must route each agent through the trusted runner map", + "mcp-bridge-dev job must remain on ubuntu-latest", + "network-policy job must not consume E2E_LARGER_RUNNER_LABEL directly", + "shields-config job must not use the larger-runner routing map", + ]), + ); + }); +}); diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index b44387d20b6..16ed622ab20 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -144,6 +144,32 @@ const GUARDED_DOCKER_HUB_AUTH_REQUIRED = `\${{ ${TRUSTED_DOCKER_HUB_PREDICATE} & const GUARDED_DOCKER_HUB_USERNAME = `\${{ ${TRUSTED_DOCKER_HUB_PREDICATE} && secrets.DOCKERHUB_USERNAME || '' }}`; const GUARDED_DOCKER_HUB_TOKEN = `\${{ ${TRUSTED_DOCKER_HUB_PREDICATE} && secrets.DOCKERHUB_TOKEN || '' }}`; const GUARDED_HERMES_E2E_INFERENCE_KEY = `\${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '' && (inputs.inference_mode || 'mock') != 'mock' && secrets.NVIDIA_INFERENCE_API_KEY || '' }}`; +const RUNNER_ROUTING_OUTPUT = "${{ steps.runner_routing.outputs.runner_routing }}"; +const RUNNER_ROUTING_STEP_NAME = "Build trusted larger-runner routing"; +const RUNNER_ROUTING_SCRIPT = [ + "set -euo pipefail", + 'larger_runner="ubuntu-latest"', + 'if [[ "${REPOSITORY}" == "NVIDIA/NemoClaw" && "${REF}" == "refs/heads/main" && -n "${LARGER_RUNNER_LABEL}" ]]; then', + ' if [[ ! "${LARGER_RUNNER_LABEL}" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$ ]]; then', + ' echo "::error::E2E_LARGER_RUNNER_LABEL must be a 1-64 character workflow label using letters, digits, dots, underscores, or hyphens" >&2', + " exit 1", + " fi", + ' larger_runner="${LARGER_RUNNER_LABEL}"', + "fi", + 'runner_routing="$(jq -cn --arg standard "ubuntu-latest" --arg larger "${larger_runner}" \'{"common-egress-agent":$larger,"mcp-bridge-deepagents":$larger,"mcp-bridge-hermes":$larger,"mcp-bridge-openclaw":$standard,"rebuild-hermes":$larger,"rebuild-hermes-stale-base":$larger}\')"', + 'printf \'runner_routing=%s\\n\' "${runner_routing}" >> "${GITHUB_OUTPUT}"', +].join("\n"); +const ROUTED_JOB_RUNNER_EXPRESSIONS = { + "common-egress-agent": + "${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['common-egress-agent'] }}", + "rebuild-hermes": + "${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['rebuild-hermes'] }}", + "rebuild-hermes-stale-base": + "${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['rebuild-hermes-stale-base'] }}", +} as const; +const MCP_BRIDGE_RUNNER_EXPRESSION = + "${{ fromJSON(needs.generate-matrix.outputs.runner_routing)[format('mcp-bridge-{0}', matrix.agent)] }}"; +const ROUTED_JOB_NAMES = new Set([...Object.keys(ROUTED_JOB_RUNNER_EXPRESSIONS), "mcp-bridge"]); function asRecord(value: unknown): WorkflowRecord { return value && typeof value === "object" && !Array.isArray(value) @@ -679,6 +705,92 @@ function requireRunDoesNotContain( } } +function validateLargerRunnerRouting( + errors: string[], + jobs: WorkflowRecord, + generateMatrix: WorkflowRecord, + generateSteps: readonly WorkflowStep[], + generateCheckout: WorkflowStep | undefined, +): void { + const generateOutputs = asRecord(generateMatrix.outputs); + if (generateOutputs.runner_routing !== RUNNER_ROUTING_OUTPUT) { + errors.push("generate-matrix job must expose the trusted larger-runner routing output"); + } + + const routing = requireJobStep( + errors, + "generate-matrix", + generateSteps, + RUNNER_ROUTING_STEP_NAME, + ); + if (!routing) return; + if (routing.id !== "runner_routing") { + errors.push("trusted larger-runner routing step must use id runner_routing"); + } + if (routing.if !== undefined) { + errors.push("trusted larger-runner routing step must always publish the fallback map"); + } + if (routing.shell !== "bash") { + errors.push("trusted larger-runner routing step must use bash"); + } + const expectedEnv = { + LARGER_RUNNER_LABEL: "${{ vars.E2E_LARGER_RUNNER_LABEL }}", + REF: "${{ github.ref }}", + REPOSITORY: "${{ github.repository }}", + }; + if (!isDeepStrictEqual(asRecord(routing.env), expectedEnv)) { + errors.push( + "trusted larger-runner routing step must bind only the administrator label and trusted repository identity", + ); + } + if (stringValue(routing.run).trimEnd() !== RUNNER_ROUTING_SCRIPT) { + errors.push( + "trusted larger-runner routing step must preserve the exact main-only map and ubuntu-latest fallback", + ); + } + if ( + generateCheckout && + generateSteps.indexOf(routing) >= generateSteps.indexOf(generateCheckout) + ) { + errors.push("trusted larger-runner routing step must run before PR checkout"); + } + + for (const [jobName, expected] of Object.entries(ROUTED_JOB_RUNNER_EXPRESSIONS)) { + if (asRecord(jobs[jobName])["runs-on"] !== expected) { + errors.push(`${jobName} job must use the trusted larger-runner routing map`); + } + } + if (asRecord(jobs["mcp-bridge"])["runs-on"] !== MCP_BRIDGE_RUNNER_EXPRESSION) { + errors.push("mcp-bridge job must route each agent through the trusted runner map"); + } + if (asRecord(jobs["mcp-bridge-dev"])["runs-on"] !== "ubuntu-latest") { + errors.push("mcp-bridge-dev job must remain on ubuntu-latest"); + } + + for (const [jobName, jobValue] of Object.entries(jobs)) { + const runsOn = stringValue(asRecord(jobValue)["runs-on"]); + if ( + runsOn.includes("needs.generate-matrix.outputs.runner_routing") && + !ROUTED_JOB_NAMES.has(jobName) + ) { + errors.push(`${jobName} job must not use the larger-runner routing map`); + } + if ( + jobName !== "generate-matrix" && + JSON.stringify(jobValue).includes("vars.E2E_LARGER_RUNNER_LABEL") + ) { + errors.push(`${jobName} job must not consume E2E_LARGER_RUNNER_LABEL directly`); + } + } + for (const step of generateSteps) { + if (step !== routing && JSON.stringify(step).includes("vars.E2E_LARGER_RUNNER_LABEL")) { + errors.push( + "only the trusted larger-runner routing step may consume E2E_LARGER_RUNNER_LABEL", + ); + } + } +} + function requireUploadPathContains(errors: string[], uploadPath: string, expected: string): void { if (!uploadPath.includes(expected)) { errors.push(`artifact upload path must include ${expected}`); @@ -1257,9 +1369,6 @@ function validateCommonEgressAgentJob(errors: string[], jobs: WorkflowRecord): v return; } - if (job["runs-on"] !== "ubuntu-latest") { - errors.push("common-egress-agent job must run on ubuntu-latest"); - } validateFreeStandingJobSelector(errors, jobs, jobName, "common-egress-agent"); if (job["timeout-minutes"] !== 120) { errors.push("common-egress-agent job must keep the legacy 120 minute timeout"); @@ -1522,9 +1631,6 @@ function validateRebuildHermesJob( return; } - if (job["runs-on"] !== "ubuntu-latest") { - errors.push(`${jobName} job must run on ubuntu-latest`); - } validateFreeStandingJobSelector(errors, jobs, jobName, targetName); if (job["timeout-minutes"] !== 90) { errors.push(`${jobName} job must keep the legacy 90 minute timeout`); @@ -3944,6 +4050,7 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { if (asRecord(generateCheckout?.with)["persist-credentials"] !== false) { errors.push("generate-matrix checkout step must set persist-credentials=false"); } + validateLargerRunnerRouting(errors, jobs, generateMatrix, generateSteps, generateCheckout); const generate = requireStep(errors, generateSteps, "Generate E2E target matrix"); const generateEnv = asRecord(generate?.env); if (generateEnv.CHECKOUT_SHA !== "${{ inputs.checkout_sha }}") {