Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 25 additions & 4 deletions .github/workflows/e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ jobs:
test_matrix: ${{ steps.matrix.outputs.test_matrix }}
hermes_selected: ${{ steps.matrix.outputs.hermes_selected }}
explicit_only_jobs: ${{ steps.matrix.outputs.explicit_only_jobs }}
runner_routing: ${{ steps.runner_routing.outputs.runner_routing }}
steps:
- id: controller_matrix
name: Build trusted controller target matrix
Expand All @@ -114,6 +115,26 @@ jobs:
esac
printf 'matrix=%s\n' "${matrix}" >> "${GITHUB_OUTPUT}"

- id: runner_routing
name: Build trusted larger-runner routing
env:
LARGER_RUNNER_LABEL: ${{ vars.E2E_LARGER_RUNNER_LABEL }}
REF: ${{ github.ref }}
REPOSITORY: ${{ github.repository }}
shell: bash
run: |
set -euo pipefail
larger_runner="ubuntu-latest"
if [[ "${REPOSITORY}" == "NVIDIA/NemoClaw" && "${REF}" == "refs/heads/main" && -n "${LARGER_RUNNER_LABEL}" ]]; then
if [[ ! "${LARGER_RUNNER_LABEL}" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$ ]]; then
echo "::error::E2E_LARGER_RUNNER_LABEL must be a 1-64 character workflow label using letters, digits, dots, underscores, or hyphens" >&2
exit 1
fi
larger_runner="${LARGER_RUNNER_LABEL}"
fi
runner_routing="$(jq -cn --arg standard "ubuntu-latest" --arg larger "${larger_runner}" '{"common-egress-agent":$larger,"mcp-bridge-deepagents":$larger,"mcp-bridge-hermes":$larger,"mcp-bridge-openclaw":$standard,"rebuild-hermes":$larger,"rebuild-hermes-stale-base":$larger}')"
printf 'runner_routing=%s\n' "${runner_routing}" >> "${GITHUB_OUTPUT}"

- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
ref: ${{ inputs.checkout_sha || github.sha }}
Expand Down Expand Up @@ -480,7 +501,7 @@ jobs:
mcp-bridge:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',mcp-bridge,') || contains(format(',{0},', inputs.targets), ',mcp-bridge,') }}
runs-on: ubuntu-latest
runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)[format('mcp-bridge-{0}', matrix.agent)] }}
permissions:
contents: read
# Keep each destructive agent lifecycle on a fresh runner. This bounds the
Expand Down Expand Up @@ -2038,7 +2059,7 @@ jobs:
common-egress-agent:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',common-egress-agent,') || contains(format(',{0},', inputs.targets), ',common-egress-agent,') }}
runs-on: ubuntu-latest
runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['common-egress-agent'] }}
timeout-minutes: 120
env:
E2E_JOB: "1"
Expand Down Expand Up @@ -2260,7 +2281,7 @@ jobs:
rebuild-hermes:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',rebuild-hermes,') || contains(format(',{0},', inputs.targets), ',rebuild-hermes,') }}
runs-on: ubuntu-latest
runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['rebuild-hermes'] }}
timeout-minutes: 90
env:
E2E_JOB: "1"
Expand Down Expand Up @@ -2335,7 +2356,7 @@ jobs:
rebuild-hermes-stale-base:
needs: generate-matrix
if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',rebuild-hermes-stale-base,') || contains(format(',{0},', inputs.targets), ',rebuild-hermes-stale-base,') }}
runs-on: ubuntu-latest
runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['rebuild-hermes-stale-base'] }}
timeout-minutes: 90
env:
E2E_JOB: "1"
Expand Down
15 changes: 15 additions & 0 deletions ci/source-shape-test-budget.json
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,21 @@
"test": "rejects free-standing E2E artifact uploads from raw temp paths",
"category": "security"
},
{
"file": "test/e2e/support/larger-runner-routing-workflow-boundary.test.ts",
"test": "keeps every candidate on standard runners when $name (#7145)",
"category": "security"
},
{
"file": "test/e2e/support/larger-runner-routing-workflow-boundary.test.ts",
"test": "rejects malformed administrator workflow labels (#7145)",
"category": "security"
},
{
"file": "test/e2e/support/larger-runner-routing-workflow-boundary.test.ts",
"test": "routes only the measured heavy lanes on trusted main (#7145)",
"category": "security"
},
{
"file": "test/fetch-guard-patch-regression.test.ts",
"test": "requires classifier review and integrity evidence when the OpenClaw build pin changes",
Expand Down
36 changes: 36 additions & 0 deletions test/e2e/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,42 @@ discovery command locally to inspect the generated test matrix:
npx tsx tools/e2e/credential-free-tests.mts
```

## Larger-runner routing

The larger-runner experiment is inactive while the configuration variable
`E2E_LARGER_RUNNER_LABEL` is unset. In that state, every eligible lane continues
to use `ubuntu-latest`. The trusted `generate-matrix` job builds one runner map
before checking out test code, and it consumes the variable only when the
workflow repository is `NVIDIA/NemoClaw` and the ref is `refs/heads/main`.

The initial eligible set is limited to the measured heavy lanes:

- `common-egress-agent`;
- `rebuild-hermes`;
- `rebuild-hermes-stale-base`;
- the `hermes` and `deepagents` shards of `mcp-bridge`.

The `openclaw` MCP shard and `mcp-bridge-dev` remain on `ubuntu-latest`;
unrelated jobs retain their existing runner assignments. Before setting the
variable, an organization owner must:

1. Create a GitHub-hosted Ubuntu x64 larger runner with 8 vCPU, 32 GB RAM, and
300 GB SSD in a dedicated runner group.
2. Set the group maximum concurrency to 4 and restrict repository access to
`NVIDIA/NemoClaw` and workflow access to
`NVIDIA/NemoClaw/.github/workflows/e2e.yaml@refs/heads/main`.
3. Record at least five standard-runner samples for each eligible lane,
including queue time, execution time, peak CPU, memory and disk use,
infrastructure failures, and estimated cost.
4. Copy the larger runner's workflow label into the repository variable, then
repeat the same measurements for at least five representative executions
per migrated lane.

Clearing `E2E_LARGER_RUNNER_LABEL` is the rollback. It sends the eligible lanes
back to `ubuntu-latest` without changing selectors, test setup, or test
semantics. Do not replace this experiment with a persistent self-hosted runner;
that requires a separate decision.

## Scheduled operations

The consolidated workflow keeps its operational reporting in the same job
Expand Down
177 changes: 177 additions & 0 deletions test/e2e/support/larger-runner-routing-workflow-boundary.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,177 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { execFileSync } from "node:child_process";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";

import { describe, expect, it } from "vitest";

import { validateE2eWorkflow } from "../../../tools/e2e/workflow-boundary.mts";
import { readWorkflow } from "../../helpers/e2e-workflow-contract";
import { requireFixture } from "./require-fixture";

type WorkflowStep = {
env?: Record<string, string>;
id?: string;
name?: string;
run?: string;
uses?: string;
};

type RoutingWorkflow = {
jobs: Record<
string,
{
outputs?: Record<string, string>;
steps?: WorkflowStep[];
"runs-on"?: string;
}
>;
};

function routingStep(workflow: RoutingWorkflow): WorkflowStep {
const step = workflow.jobs["generate-matrix"]?.steps?.find(
(candidate) => candidate.id === "runner_routing",
);
requireFixture(step?.run, "trusted larger-runner routing step is missing");
return step;
}

function evaluateRouting(
workflow: RoutingWorkflow,
env: { label: string; ref: string; repository: string },
): Record<string, string> {
const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-runner-routing-"));
const outputPath = path.join(directory, "github-output");
try {
execFileSync("bash", ["-c", routingStep(workflow).run!], {
env: {
...process.env,
GITHUB_OUTPUT: outputPath,
LARGER_RUNNER_LABEL: env.label,
REF: env.ref,
REPOSITORY: env.repository,
},
});
const output = fs.readFileSync(outputPath, "utf8").trim();
requireFixture(output.startsWith("runner_routing="), "runner routing output is missing");
return JSON.parse(output.slice("runner_routing=".length)) as Record<string, string>;
} finally {
fs.rmSync(directory, { force: true, recursive: true });
}
}

const standardRouting = {
"common-egress-agent": "ubuntu-latest",
"mcp-bridge-deepagents": "ubuntu-latest",
"mcp-bridge-hermes": "ubuntu-latest",
"mcp-bridge-openclaw": "ubuntu-latest",
"rebuild-hermes": "ubuntu-latest",
"rebuild-hermes-stale-base": "ubuntu-latest",
};

describe("larger-runner workflow routing boundary", () => {
// source-shape-contract: security -- Executes the shipped pre-checkout router to prove untrusted and unset configurations stay on standard runners
it.each([
{
label: "",
name: "the administrator label is unset",
ref: "refs/heads/main",
repository: "NVIDIA/NemoClaw",
},
{
label: "ubuntu-24.04-8core",
name: "the workflow is not running from main",
ref: "refs/heads/feature",
repository: "NVIDIA/NemoClaw",
},
{
label: "ubuntu-24.04-8core",
name: "the workflow belongs to another repository",
ref: "refs/heads/main",
repository: "someone/NemoClaw",
},
])("keeps every candidate on standard runners when $name (#7145)", ({
label,
ref,
repository,
}) => {
expect(evaluateRouting(readWorkflow() as RoutingWorkflow, { label, ref, repository })).toEqual(
standardRouting,
);
});

// source-shape-contract: security -- Executes the shipped pre-checkout router to prove trusted main can reach only the reviewed heavy lanes
it("routes only the measured heavy lanes on trusted main (#7145)", () => {
const largerRunner = "ubuntu-24.04-8core";
expect(
evaluateRouting(readWorkflow() as RoutingWorkflow, {
label: largerRunner,
ref: "refs/heads/main",
repository: "NVIDIA/NemoClaw",
}),
).toEqual({
...standardRouting,
"common-egress-agent": largerRunner,
"mcp-bridge-deepagents": largerRunner,
"mcp-bridge-hermes": largerRunner,
"rebuild-hermes": largerRunner,
"rebuild-hermes-stale-base": largerRunner,
});
});

// source-shape-contract: security -- Executes the shipped pre-checkout router to prove malformed administrator labels fail before job routing
it("rejects malformed administrator workflow labels (#7145)", () => {
expect(() =>
evaluateRouting(readWorkflow() as RoutingWorkflow, {
label: "invalid runner\nlabel",
ref: "refs/heads/main",
repository: "NVIDIA/NemoClaw",
}),
).toThrow();
});

it("rejects moving or weakening the trusted pre-checkout map (#7145)", () => {
const workflow = readWorkflow() as RoutingWorkflow;
const generate = workflow.jobs["generate-matrix"];
const steps = generate.steps!;
const routing = routingStep(workflow);
generate.outputs!.runner_routing = "${{ steps.matrix.outputs.runner_routing }}";
routing.env!.REF = "${{ inputs.base_sha }}";
routing.run = routing.run!.replace('"${REF}" == "refs/heads/main" && ', "");
steps.splice(steps.indexOf(routing), 1);
steps.push(routing);

expect(validateE2eWorkflow(workflow)).toEqual(
expect.arrayContaining([
"generate-matrix job must expose the trusted larger-runner routing output",
"trusted larger-runner routing step must bind only the administrator label and trusted repository identity",
"trusted larger-runner routing step must preserve the exact main-only map and ubuntu-latest fallback",
"trusted larger-runner routing step must run before PR checkout",
]),
);
});

it("rejects routing any lane outside the centralized eligible set (#7145)", () => {
const workflow = readWorkflow() as RoutingWorkflow;
workflow.jobs["common-egress-agent"]["runs-on"] = "ubuntu-latest";
workflow.jobs["mcp-bridge"]["runs-on"] = "ubuntu-latest";
workflow.jobs["mcp-bridge-dev"]["runs-on"] =
"${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['mcp-bridge-deepagents'] }}";
workflow.jobs["network-policy"]["runs-on"] = "${{ vars.E2E_LARGER_RUNNER_LABEL }}";
workflow.jobs["shields-config"]["runs-on"] =
"${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['common-egress-agent'] }}";

expect(validateE2eWorkflow(workflow)).toEqual(
expect.arrayContaining([
"common-egress-agent job must use the trusted larger-runner routing map",
"mcp-bridge job must route each agent through the trusted runner map",
"mcp-bridge-dev job must remain on ubuntu-latest",
"network-policy job must not consume E2E_LARGER_RUNNER_LABEL directly",
"shields-config job must not use the larger-runner routing map",
]),
);
});
});
Loading
Loading