Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 34 additions & 6 deletions .github/workflows/e2e.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# SPDX-License-Identifier: Apache-2.0

name: E2E
run-name: ${{ inputs.risk_correlation != '' && format('E2E risk {0}', inputs.risk_correlation) || format('E2E {0}', github.ref_name) }}
run-name: "${{ inputs.risk_pr && format('E2E PR #{0} risk {1}', inputs.pr_number, inputs.risk_correlation) || (inputs.risk_correlation != '' && format('E2E risk {0}', inputs.risk_correlation) || format('E2E {0}', github.ref_name)) }}"

on:
schedule:
Expand Down Expand Up @@ -54,13 +54,18 @@ on:
required: false
default: false
type: boolean
risk_pr:
description: Mark an exact-commit risk run as trusted PR validation.
required: false
default: false
type: boolean

permissions:
contents: read

concurrency:
group: e2e-${{ github.ref }}-${{ inputs.risk_shadow && github.run_id || inputs.targets || 'supported' }}-${{ inputs.risk_shadow && 'risk-shadow' || inputs.jobs || 'all-jobs' }}
cancel-in-progress: false
group: ${{ inputs.risk_pr && format('e2e-required-live-pr-{0}', inputs.pr_number) || format('e2e-{0}-{1}-{2}', github.ref, inputs.risk_shadow && github.run_id || inputs.targets || 'supported', inputs.risk_shadow && 'risk-shadow' || inputs.jobs || 'all-jobs') }}
cancel-in-progress: ${{ inputs.risk_pr }}

env:
NEMOCLAW_E2E_EXPECTED_SHA: ${{ inputs.checkout_sha }}
Expand All @@ -72,6 +77,9 @@ env:
jobs:
generate-matrix:
runs-on: ubuntu-latest
permissions:
contents: read
pull-requests: read
outputs:
matrix: ${{ steps.matrix.outputs.matrix }}
hermes_selected: ${{ steps.matrix.outputs.hermes_selected }}
Expand All @@ -84,12 +92,15 @@ jobs:
persist-credentials: false

- name: Validate exact-commit dispatch
if: ${{ inputs.checkout_sha != '' || inputs.risk_plan_hash != '' || inputs.risk_correlation != '' || inputs.risk_shadow }}
if: ${{ inputs.checkout_sha != '' || inputs.risk_plan_hash != '' || inputs.risk_correlation != '' || inputs.risk_shadow || inputs.risk_pr }}
env:
CHECKOUT_SHA: ${{ inputs.checkout_sha }}
GITHUB_READ_TOKEN: ${{ github.token }}
JOBS: ${{ inputs.jobs }}
PLAN_HASH: ${{ inputs.risk_plan_hash }}
PR_NUMBER: ${{ inputs.pr_number }}
RISK_CORRELATION: ${{ inputs.risk_correlation }}
RISK_PR: ${{ inputs.risk_pr }}
RISK_SHADOW: ${{ inputs.risk_shadow }}
TARGETS: ${{ inputs.targets }}
WORKFLOW_EVENT: ${{ github.event_name }}
Expand All @@ -100,9 +111,17 @@ jobs:
[[ "$WORKFLOW_EVENT" == "workflow_dispatch" && "$WORKFLOW_REF" == "refs/heads/main" ]] || { echo "::error::exact-commit runs require a workflow_dispatch from main"; exit 1; }
[[ "$RISK_SHADOW" == "true" ]] || { echo "::error::exact-commit inputs require risk_shadow=true"; exit 1; }
[[ "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ ]] || { echo "::error::checkout_sha must be a lowercase 40-character SHA"; exit 1; }
[[ "$CHECKOUT_SHA" == "$WORKFLOW_SHA" ]] || { echo "::error::checkout_sha must equal the current main workflow commit"; exit 1; }
[[ "$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]] || { echo "::error::checked-out HEAD does not match checkout_sha"; exit 1; }
git merge-base --is-ancestor "$CHECKOUT_SHA" origin/main || { echo "::error::checkout_sha must already be reachable from main"; exit 1; }
if [[ "$RISK_PR" == "true" ]]; then
[[ "$PR_NUMBER" =~ ^[1-9][0-9]*$ ]] || { echo "::error::risk PR requires a positive PR number"; exit 1; }
pr_json="$(curl -fsSL -H 'Accept: application/vnd.github+json' -H "Authorization: Bearer $GITHUB_READ_TOKEN" -H 'X-GitHub-Api-Version: 2022-11-28' "https://api.github.com/repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER")"
[[ "$(jq -r .state <<<"$pr_json")" == "open" ]] || { echo "::error::risk PR must still be open"; exit 1; }
[[ "$(jq -r .head.repo.full_name <<<"$pr_json")" == "$GITHUB_REPOSITORY" ]] || { echo "::error::automatic risk PR must be first-party"; exit 1; }
[[ "$(jq -r .head.sha <<<"$pr_json")" == "$CHECKOUT_SHA" ]] || { echo "::error::checkout_sha is stale for the PR"; exit 1; }
else
[[ "$CHECKOUT_SHA" == "$WORKFLOW_SHA" ]] || { echo "::error::checkout_sha must equal the current main workflow commit"; exit 1; }
git merge-base --is-ancestor "$CHECKOUT_SHA" origin/main || { echo "::error::checkout_sha must already be reachable from main"; exit 1; }
fi
[[ "$PLAN_HASH" =~ ^[a-f0-9]{64}$ ]] || { echo "::error::risk_plan_hash must be a lowercase SHA-256"; exit 1; }
[[ "$RISK_CORRELATION" =~ ^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$ ]] || { echo "::error::risk_correlation must be a lowercase UUIDv4"; exit 1; }
[[ -n "$JOBS" && -z "$TARGETS" ]] || { echo "::error::shadow risk runs require selective jobs and forbid targets/fan-out"; exit 1; }
Expand Down Expand Up @@ -3519,6 +3538,15 @@ jobs:

- *dockerhub-auth

- name: Reclaim runner disk for dual-version upgrade images
shell: bash
run: |
set -euo pipefail
df -h /
sudo rm -rf -- /usr/local/lib/android /usr/share/dotnet /opt/ghc
docker system prune --all --force
df -h /

- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28

Expand Down
183 changes: 183 additions & 0 deletions .github/workflows/pr-e2e-risk-gate.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,183 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

name: E2E / Required Live Coordinator

on:
pull_request_target:
types: [synchronize, reopened, closed]
workflow_run:
workflows: ["CI / Pull Request"]
types: [completed]

permissions: {}

concurrency:
group: required-live-${{ github.event.pull_request.head.repo.full_name || github.event.workflow_run.head_repository.full_name }}-${{ github.event.pull_request.head.ref || github.event.workflow_run.head_branch }}
cancel-in-progress: true
Comment thread
coderabbitai[bot] marked this conversation as resolved.

jobs:
cancel-superseded:
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'pull_request_target' }}
runs-on: ubuntu-latest
permissions:
actions: write
contents: read
timeout-minutes: 5
steps:
- name: Checkout trusted coordinator
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
repository: NVIDIA/NemoClaw
ref: ${{ github.sha }}
persist-credentials: false

- name: Cancel superseded required-live runs
env:
GITHUB_TOKEN: ${{ github.token }}
PR_NUMBER: ${{ github.event.pull_request.number }}
run: >-
node --experimental-strip-types tools/e2e-advisor/pr-risk-gate.mts
--mode cancel --pr "$PR_NUMBER"

coordinate:
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event_name == 'workflow_run' && github.event.workflow_run.event == 'pull_request' }}
runs-on: ubuntu-latest
permissions:
actions: write
checks: write
contents: read
pull-requests: read
timeout-minutes: 150
steps:
- name: Checkout trusted coordinator
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
repository: NVIDIA/NemoClaw
ref: ${{ github.sha }}
fetch-depth: 0
persist-credentials: false

- name: Setup Node
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0
with:
node-version: "22"
cache: npm

- name: Install trusted coordinator dependencies
run: npm ci --ignore-scripts

- id: initialize
name: Create exact-head required-live check
env:
GITHUB_TOKEN: ${{ github.token }}
HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
run: >-
node --experimental-strip-types tools/e2e-advisor/pr-risk-gate.mts
--mode initialize --head "$HEAD_SHA"

- id: resolve
name: Resolve exact PR and CI result
env:
CI_CONCLUSION: ${{ github.event.workflow_run.conclusion }}
GITHUB_TOKEN: ${{ github.token }}
HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }}
HEAD_REPO: ${{ github.event.workflow_run.head_repository.full_name }}
HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
run: >-
node --experimental-strip-types tools/e2e-advisor/pr-risk-gate.mts
--mode resolve

- name: Fetch exact first-party PR objects
if: ${{ steps.resolve.outputs.first_party == 'true' && steps.resolve.outputs.ci_green == 'true' }}
env:
BASE_SHA: ${{ steps.resolve.outputs.base_sha }}
HEAD_SHA: ${{ steps.resolve.outputs.head_sha }}
run: |
git fetch --no-tags origin "$BASE_SHA" "$HEAD_SHA"

- id: advisor
name: Wait for exact-head Advisor artifacts
if: ${{ steps.resolve.outputs.first_party == 'true' && steps.resolve.outputs.ci_green == 'true' }}
env:
GH_TOKEN: ${{ github.token }}
HEAD_SHA: ${{ steps.resolve.outputs.head_sha }}
run: |
set -euo pipefail
for attempt in $(seq 1 60); do
run_id="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow e2e-advisor.yaml --commit "$HEAD_SHA" --event pull_request --status completed --limit 1 --json databaseId --jq '.[0].databaseId // empty')"
if [ -n "$run_id" ]; then
rm -rf "$RUNNER_TEMP/e2e-advisor"
if gh run download "$run_id" --repo "$GITHUB_REPOSITORY" --name e2e-advisor --dir "$RUNNER_TEMP/e2e-advisor"; then
echo "artifact_dir=$RUNNER_TEMP/e2e-advisor" >> "$GITHUB_OUTPUT"
exit 0
fi
fi
sleep 15
done
echo "::error::Exact-head E2E Advisor artifact did not become available"
exit 1

- id: start
name: Build exact-head plan and dispatch required live E2E
if: ${{ always() }}
env:
ADVISOR_DIR: ${{ steps.advisor.outputs.artifact_dir }}
BASE_SHA: ${{ steps.resolve.outputs.base_sha }}
CHECK_ID: ${{ steps.initialize.outputs.check_id }}
CI_GREEN: ${{ steps.resolve.outputs.ci_green }}
GITHUB_TOKEN: ${{ github.token }}
HEAD_REPO: ${{ steps.resolve.outputs.head_repo }}
HEAD_SHA: ${{ steps.resolve.outputs.head_sha }}
PR_NUMBER: ${{ steps.resolve.outputs.pr_number }}
run: >-
node --experimental-strip-types tools/e2e-advisor/pr-risk-gate.mts
--mode start
--check-id "$CHECK_ID"
--pr "$PR_NUMBER"
--base "$BASE_SHA"
--head "$HEAD_SHA"
--head-repo "$HEAD_REPO"
--ci-green "$CI_GREEN"
--advisor-dir "$ADVISOR_DIR"

- name: Wait for correlated live E2E
if: ${{ steps.start.outputs.dispatched == 'true' }}
env:
GH_TOKEN: ${{ github.token }}
RUN_ID: ${{ steps.start.outputs.run_id }}
run: timeout --signal=TERM --kill-after=30s 105m gh run watch "$RUN_ID" --repo "$GITHUB_REPOSITORY" --exit-status

- name: Download correlated evidence
if: ${{ always() && steps.start.outputs.dispatched == 'true' }}
env:
GH_TOKEN: ${{ github.token }}
RUN_ID: ${{ steps.start.outputs.run_id }}
run: gh run download "$RUN_ID" --repo "$GITHUB_REPOSITORY" --dir "$RUNNER_TEMP/required-live-evidence"

- id: finish
name: Complete exact-head required-live check
if: ${{ always() && steps.start.outputs.dispatched == 'true' }}
env:
CHECK_ID: ${{ steps.initialize.outputs.check_id }}
GITHUB_TOKEN: ${{ github.token }}
RUN_ID: ${{ steps.start.outputs.run_id }}
STATE_HASH: ${{ steps.start.outputs.state_hash }}
run: >-
node --experimental-strip-types tools/e2e-advisor/pr-risk-gate.mts
--mode finish
--check-id "$CHECK_ID"
--run-id "$RUN_ID"
--state "$RUNNER_TEMP/required-live-state.json"
--state-hash "$STATE_HASH"
--evidence "$RUNNER_TEMP/required-live-evidence"

- name: Close required-live check after coordinator failure
if: ${{ always() && steps.initialize.outputs.check_id != '' && steps.finish.outcome != 'success' && (failure() || cancelled()) }}
env:
CHECK_ID: ${{ steps.initialize.outputs.check_id }}
GITHUB_TOKEN: ${{ github.token }}
RUN_ID: ${{ steps.start.outputs.run_id }}
run: >-
node --experimental-strip-types tools/e2e-advisor/pr-risk-gate.mts
--mode abandon --check-id "$CHECK_ID" --run-id "$RUN_ID"
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ describe("MCP status wire-level credential-resolution probe", () => {
),
).toBe(true);
expect(payload.exitCode).toBe(0);
});
}, 15_000);

it("skips status probe traffic until exact policy and provider readiness are verified (#6379)", () => {
const home = createTempHome("nemoclaw-mcp-resolution-readiness-");
Expand Down
11 changes: 11 additions & 0 deletions test/e2e/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,17 @@ scheduled/manual scorecard, including scorecard Slack reporting.
Controller or evidence-verification errors close an already-created check as
neutral so incomplete evidence cannot appear successful.

For pull requests, `.github/workflows/pr-e2e-risk-gate.yaml` waits for the
exact-head `CI / Pull Request` run to finish while the E2E Advisor runs in
parallel. A green CI result and matching Advisor artifact produce the union of
the deterministic risk floor and Advisor-required jobs. The coordinator
dispatches at most three jobs concurrently through `e2e.yaml` and reports one
`E2E / Required Live` check on the PR head. A new commit immediately cancels
older PR-correlated live runs; exact-head checks in both the coordinator and
child workflow reject stale results. Secret-bearing execution is automatic
only for branches in `NVIDIA/NemoClaw`. Fork PRs receive a non-green approval
boundary instead of exposing repository credentials to fork code.

## Onboard performance budget

The scheduled/manual scorecard evaluates the trusted `cloud-onboard` timing
Expand Down
19 changes: 18 additions & 1 deletion test/e2e/support/e2e-workflow-contract.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

import { describe, expect, it } from "vitest";

import { removeJobNeed } from "../../helpers/e2e-workflow-contract";
import { readYaml, removeJobNeed, type Workflow } from "../../helpers/e2e-workflow-contract";

describe("E2E workflow test helpers", () => {
it("refuses to remove a dependency from a later job", () => {
Expand All @@ -26,4 +26,21 @@ describe("E2E workflow test helpers", () => {
"owner does not need misplaced",
);
});

it("binds every checkout to the validated exact revision", () => {
const workflow = readYaml<Workflow>(".github/workflows/e2e.yaml");
const checkouts = Object.entries(workflow.jobs).flatMap(([jobId, job]) =>
(job.steps ?? [])
.filter((step) => step.uses?.startsWith("actions/checkout@"))
.map((step) => ({ jobId, step })),
);

expect(checkouts.length).toBeGreaterThan(0);
for (const { jobId, step } of checkouts) {
expect(step.with?.ref, `${jobId} checkout revision`).toBe(
"${{ inputs.checkout_sha || github.sha }}",
);
expect(step.with?.["persist-credentials"], `${jobId} checkout credentials`).toBe(false);
}
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { describe, expect, it } from "vitest";
import { readYaml, type Workflow } from "../../helpers/e2e-workflow-contract.ts";
import {
evaluateE2eWorkflowDispatchSelectors,
readFreeStandingJobsInventory,
Expand Down Expand Up @@ -52,6 +53,24 @@ describe("OpenShell gateway upgrade workflow boundary", () => {
);
});

it("reclaims deterministic hosted-runner space before building both upgrade images", () => {
const workflow = readYaml<Workflow>(".github/workflows/e2e.yaml");
const steps = workflow.jobs["openshell-gateway-upgrade"]?.steps ?? [];
const reclaimIndex = steps.findIndex(
(step) => step.name === "Reclaim runner disk for dual-version upgrade images",
);
const testIndex = steps.findIndex(
(step) => step.name === "Run OpenShell gateway upgrade live Vitest test",
);

expect(reclaimIndex).toBeGreaterThan(0);
expect(reclaimIndex).toBeLessThan(testIndex);
expect(steps[reclaimIndex]?.run).toContain(
"sudo rm -rf -- /usr/local/lib/android /usr/share/dotnet /opt/ghc",
);
expect(steps[reclaimIndex]?.run).toContain("docker system prune --all --force");
});

it("freshens only the retryable old fixture install", () => {
expect(oldGatewayUpgradeInstallerArgs("old-install.sh")).toEqual([
"old-install.sh",
Expand Down
Loading
Loading