Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions .github/workflows/pr-e2e-gate.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,43 @@ jobs:
--head "$HEAD_SHA"
--base "$BASE_SHA"

required:
name: E2E / PR Gate
if: ${{ github.event_name == 'pull_request_target' && github.repository == 'NVIDIA/NemoClaw' && github.event.action != 'closed' }}
runs-on: ubuntu-latest
timeout-minutes: 170
permissions:
checks: read
contents: read
pull-requests: read
concurrency:
group: pr-e2e-required-${{ github.event.pull_request.number }}
cancel-in-progress: true
steps:
- name: Checkout observer
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
ref: ${{ github.workflow_sha }}
persist-credentials: false

- name: Setup Node
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0
with:
node-version: "22"

- name: Wait for trusted exact-diff verdict
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
GITHUB_TOKEN: ${{ github.token }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
PR_NUMBER: ${{ github.event.pull_request.number }}
run: >-
node --experimental-strip-types tools/e2e/pr-e2e-required.mts
--pr "$PR_NUMBER"
--head "$HEAD_SHA"
--base "$BASE_SHA"
--timeout-seconds 9900

cancel-superseded:
if: ${{ github.event_name == 'pull_request_target' && github.repository == 'NVIDIA/NemoClaw' && github.event.pull_request.head.repo.full_name == github.repository && (github.event.action != 'edited' || github.event.changes.base != null) }}
runs-on: ubuntu-latest
Expand Down
165 changes: 102 additions & 63 deletions test/e2e/README.md

Large diffs are not rendered by default.

24 changes: 17 additions & 7 deletions test/e2e/docs/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -115,16 +115,26 @@ test/e2e/
ledger. The advisor uses it as recommendation context, while the controller
applies it independently without model output.

- `.github/workflows/pr-e2e-gate.yaml` reserves `E2E / PR Gate` on every exact
PR head, including forks, before `CI / Pull Request` completes. The trusted
- `.github/workflows/pr-e2e-gate.yaml` reserves the internal
`E2E / PR Gate Coordination` check on every exact PR head, including forks,
before `CI / Pull Request` completes. Its default-branch
`pull_request_target` path also publishes the native GitHub Actions job named
`E2E / PR Gate`. The read-only observer runs from `github.workflow_sha`,
validates the live PR head and base, waits for the matching trusted
coordination identity, and mirrors the terminal verdict into the required
job. Its summary is static, while the job log includes the validated trusted
controller-run link. Authorization states remain pending while the maintainer
decision is recorded. During rollout, the observer also accepts the former
`E2E / PR Gate` custom-check name for the same exact-diff identity. The
controller builds the risk plan from GitHub's complete file list. Internal
revisions normally dispatch every selected job and verify each expected
`risk-signal.json`; this remains automatic when their `e2e-control-plane`
matches are drawn only from the trusted controller workflow and script.
Other or mixed internal control-plane revisions require a maintainer-authorized
exact-SHA run; only its verified evidence can pass the gate. Risky forks
retain the audited credentialed-E2E skip approval. See
[NemoClaw E2E CI](../README.md) for the full lifecycle.
matches are drawn only from the trusted controller workflow and scripts.
Other or mixed internal
control-plane revisions require a maintainer-authorized exact-SHA run; only
its verified evidence can pass coordination. Risky forks retain the audited
credentialed-E2E skip approval. See [NemoClaw E2E CI](../README.md) for the
full lifecycle.

- `.github/workflows/e2e.yaml` runs selected or all supported
live E2E targets and uploads an explicit artifact allowlist with
Expand Down
4 changes: 4 additions & 0 deletions test/helpers/vitest-watch-triggers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,10 @@ export const vitestWatchTriggerPatterns: VitestWatchTriggerPattern[] = [
pattern: /(?:^|\/)\.github\/workflows\/e2e\.yaml$/,
testsToRun: runTests(...E2E_WORKFLOW_CONTRACTS),
},
{
pattern: /(?:^|\/)\.github\/workflows\/pr-e2e-gate\.yaml$/,
testsToRun: runTests("test/pr-e2e-gate-workflow.test.ts", "test/pr-e2e-required.test.ts"),
},
{
pattern:
/(?:^|\/)(?:\.github\/workflows\/platform-vitest-main\.yaml|ci\/platform-vitest-macos-requirements\.lock)$/,
Expand Down
2 changes: 1 addition & 1 deletion test/pr-e2e-gate-fork-skip.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ function emptyPrGateCheckRunsRoute() {
function exactPrGateCheck(overrides: Record<string, unknown> = {}) {
return {
id: 17,
name: "E2E / PR Gate",
name: "E2E / PR Gate Coordination",
head_sha: HEAD_SHA,
external_id: prGateExternalId(42, HEAD_SHA, BASE_SHA),
status: "in_progress",
Expand Down
2 changes: 1 addition & 1 deletion test/pr-e2e-gate-lifecycle.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ function emptyPrGateCheckRunsRoute() {
function exactPrGateCheck(overrides: Record<string, unknown> = {}) {
return {
id: 17,
name: "E2E / PR Gate",
name: "E2E / PR Gate Coordination",
head_sha: HEAD_SHA,
external_id: prGateExternalId(42, HEAD_SHA, BASE_SHA),
status: "in_progress",
Expand Down
33 changes: 31 additions & 2 deletions test/pr-e2e-gate-workflow.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -358,6 +358,7 @@ describe("PR E2E gate workflow", () => {
it("limits triggers and job permissions", () => {
const workflow = readYaml<TriggeredWorkflow>(PR_GATE_PATH);
const initialize = workflow.jobs.initialize;
const required = workflow.jobs.required;
const cancel = workflow.jobs["cancel-superseded"];
const coordinate = workflow.jobs.coordinate;
const approveForkSkip = workflow.jobs["approve-fork-e2e-skip"];
Expand Down Expand Up @@ -425,6 +426,34 @@ describe("PR E2E gate workflow", () => {
expect(initialize.concurrency?.group).toBe(
"pr-e2e-gate-${{ github.event.pull_request.head.repo.full_name }}-${{ github.event.pull_request.head.ref }}",
);
expect(required.name).toBe("E2E / PR Gate");
expect(required.if).toContain("github.event_name == 'pull_request_target'");
expect(required.if).toContain("github.event.action != 'closed'");
expect(required.permissions).toEqual({
checks: "read",
contents: "read",
"pull-requests": "read",
});
expect(required.concurrency).toEqual({
group: "pr-e2e-required-${{ github.event.pull_request.number }}",
"cancel-in-progress": true,
});
expect(required["timeout-minutes"]).toBe(170);
expect(required.secrets).toBeUndefined();
expect(step(required, "Checkout observer").with).toEqual({
ref: "${{ github.workflow_sha }}",
"persist-credentials": false,
});
const observer = step(required, "Wait for trusted exact-diff verdict");
expect(observer.env).toEqual({
BASE_SHA: "${{ github.event.pull_request.base.sha }}",
GITHUB_TOKEN: "${{ github.token }}",
HEAD_SHA: "${{ github.event.pull_request.head.sha }}",
PR_NUMBER: "${{ github.event.pull_request.number }}",
});
expect(observer.run).toContain("tools/e2e/pr-e2e-required.mts");
expect(observer.run).toContain('--head "$HEAD_SHA"');
expect(observer.run).toContain('--base "$BASE_SHA"');
expect(cancel.if).toContain("github.event_name == 'pull_request_target'");
expect(cancel.if).toContain(
"github.event.pull_request.head.repo.full_name == github.repository",
Expand Down Expand Up @@ -555,15 +584,15 @@ describe("PR E2E gate workflow", () => {
(candidate) => candidate.name === "Install controller dependencies",
);

expect(checkouts).toHaveLength(5);
expect(checkouts).toHaveLength(6);
expect(
checkouts.every(
(checkout) =>
checkout.with?.ref === "${{ github.workflow_sha }}" &&
checkout.with?.["persist-credentials"] === false,
),
).toBe(true);
expect(nodeSetups).toHaveLength(5);
expect(nodeSetups).toHaveLength(6);
expect(nodeSetups.every((setup) => setup.with?.["node-version"] === "22")).toBe(true);
expect(nodeSetups.every((setup) => !("cache" in (setup.with ?? {})))).toBe(true);
expect(installs).toHaveLength(5);
Expand Down
4 changes: 2 additions & 2 deletions test/pr-e2e-gate.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ function emptyPrGateCheckRunsRoute() {
function exactPrGateCheck(overrides: Record<string, unknown> = {}) {
return {
id: 17,
name: "E2E / PR Gate",
name: "E2E / PR Gate Coordination",
head_sha: HEAD_SHA,
external_id: prGateExternalId(42, HEAD_SHA, BASE_SHA),
status: "in_progress",
Expand Down Expand Up @@ -1349,7 +1349,7 @@ describe("PR E2E controller", () => {
(request) => request.url.endsWith("/check-runs") && request.method === "POST",
);
expect(checkCreation?.body).toMatchObject({
name: "E2E / PR Gate",
name: "E2E / PR Gate Coordination",
head_sha: HEAD_SHA,
external_id: prGateExternalId(42, HEAD_SHA, BASE_SHA),
status: "in_progress",
Expand Down
209 changes: 209 additions & 0 deletions test/pr-e2e-required.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,209 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { spawnSync } from "node:child_process";
import { fileURLToPath } from "node:url";

import { afterEach, describe, expect, it, vi } from "vitest";
import {
type CoordinationCheckRun,
classifyCoordinationCheck,
coordinationExternalId,
findCoordinationCheck,
type RequiredGateIdentity,
waitForRequiredGate,
} from "../tools/e2e/pr-e2e-required.mts";
import { createGitHubFetchRouter, githubFetchRoute } from "./support/github-fetch-router.ts";

const HEAD_SHA = "a".repeat(40);
const BASE_SHA = "b".repeat(40);
const SCRIPT = fileURLToPath(new URL("../tools/e2e/pr-e2e-required.mts", import.meta.url));

const identity: RequiredGateIdentity = {
repository: "NVIDIA/NemoClaw",
token: "token",
prNumber: 42,
headSha: HEAD_SHA,
baseSha: BASE_SHA,
};

afterEach(() => {
vi.restoreAllMocks();
});

function githubResponse(value: unknown, status = 200): Response {
return {
ok: status >= 200 && status < 300,
status,
text: async () => JSON.stringify(value),
} as Response;
}

function pullRequest(overrides: Record<string, unknown> = {}) {
return {
number: 42,
state: "open",
head: { sha: HEAD_SHA },
base: { sha: BASE_SHA },
...overrides,
};
}

function check(
name = "E2E / PR Gate Coordination",
overrides: Partial<CoordinationCheckRun> = {},
): CoordinationCheckRun {
return {
id: 17,
name,
head_sha: HEAD_SHA,
external_id: coordinationExternalId(42, HEAD_SHA, BASE_SHA),
status: "completed",
conclusion: "success",
details_url: "https://github.com/NVIDIA/NemoClaw/actions/runs/99",
output: { title: "All selected E2E jobs passed" },
app: { id: 15368 },
...overrides,
};
}

function listing(checks: CoordinationCheckRun[]) {
return { total_count: checks.length, check_runs: checks };
}

describe("native PR E2E required job", () => {
it("loads under the workflow Node runtime", () => {
const result = spawnSync(
process.execPath,
["--experimental-strip-types", SCRIPT, "--pr", "42"],
{ encoding: "utf8" },
);

expect(result.status).toBe(1);
expect(result.stderr).toContain("--head is required");
expect(result.stderr).not.toContain("ERR_UNSUPPORTED_TYPESCRIPT_SYNTAX");
});

it("classifies authorization-required failures as pending", () => {
expect(
classifyCoordinationCheck(
check("E2E / PR Gate", {
conclusion: "failure",
output: { title: "Maintainer authorization required to run E2E" },
}),
),
).toEqual({
state: "waiting",
description: "Maintainer authorization required to run E2E",
});
});

it("returns terminal trusted conclusions and drops untrusted detail links", () => {
expect(
classifyCoordinationCheck(
check(undefined, {
conclusion: "failure",
details_url: "https://example.com/untrusted",
output: { title: "Selected E2E jobs failed" },
}),
),
).toEqual({
state: "complete",
result: { conclusion: "failure", title: "Selected E2E jobs failed" },
});
});

it("prefers the renamed coordination check without querying the legacy name", async () => {
const urls: string[] = [];
vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => {
const url = String(input);
urls.push(url);
return githubResponse(listing([check()]));
});

await expect(findCoordinationCheck(identity)).resolves.toMatchObject({ id: 17 });
expect(urls).toHaveLength(1);
expect(urls[0]).toContain("E2E%20%2F%20PR%20Gate%20Coordination");
});

it("uses the old required-name check during the native-job migration", async () => {
vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => {
const url = String(input);
return githubResponse(
url.includes("Coordination") ? listing([]) : listing([check("E2E / PR Gate")]),
);
});

await expect(findCoordinationCheck(identity)).resolves.toMatchObject({
name: "E2E / PR Gate",
});
});

it("rejects an exact-diff identity claimed by another app", async () => {
vi.spyOn(globalThis, "fetch").mockResolvedValue(
githubResponse(listing([check(undefined, { app: { id: 1 } })])),
);

await expect(findCoordinationCheck(identity)).rejects.toThrow("unexpected GitHub App");
});

it("waits through authorization and revalidates the exact PR before passing", async () => {
let legacyQueries = 0;
let clock = 0;
vi.spyOn(globalThis, "fetch").mockImplementation(
createGitHubFetchRouter([
githubFetchRoute(
({ url }) => url.includes("/pulls/42"),
() => githubResponse(pullRequest()),
),
githubFetchRoute(
({ url }) => url.includes("Coordination"),
() => githubResponse(listing([])),
),
githubFetchRoute(
({ url }) => url.includes("/check-runs") && !url.includes("Coordination"),
() => {
legacyQueries += 1;
return githubResponse(
listing([
legacyQueries === 1
? check("E2E / PR Gate", {
conclusion: "failure",
output: { title: "Maintainer authorization required to run E2E" },
})
: check("E2E / PR Gate"),
]),
);
},
),
]),
);

await expect(
waitForRequiredGate(identity, {
timeoutMs: 100,
pollIntervalMs: 10,
now: () => clock,
sleep: async (milliseconds) => {
clock += milliseconds;
},
}),
).resolves.toMatchObject({ conclusion: "success" });
expect(legacyQueries).toBe(2);
});

it("fails closed when the PR head changes before a terminal verdict is accepted", async () => {
vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => {
const url = String(input);
return githubResponse(
url.includes("/pulls/42")
? pullRequest({ head: { sha: "c".repeat(40) } })
: listing([check()]),
);
});

await expect(
waitForRequiredGate(identity, { timeoutMs: 100, pollIntervalMs: 10 }),
).rejects.toThrow("no longer matches the exact head and base revision");
});
});
Loading