From 90a425a3148c5716aee407c26c80c3498ab33480 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Fri, 10 Jul 2026 15:20:26 -0700 Subject: [PATCH 1/7] ci(e2e): require deterministic live validation --- .github/workflows/e2e.yaml | 72 +- .../post-merge-e2e-risk-gate-shadow.yaml | 249 ---- .github/workflows/required-live-e2e.yaml | 220 +++ docs/about/release-notes.mdx | 2 +- test/e2e-advisor-targets.test.ts | 2 +- test/e2e-private-file.test.ts | 7 +- test/e2e-risk-signal-reporter.test.ts | 13 +- test/e2e/README.md | 98 +- test/e2e/docs/README.md | 39 +- test/e2e/risk-signal-reporter.ts | 17 +- .../e2e-operations-workflow-boundary.test.ts | 49 +- .../post-merge-e2e-risk-gate-workflow.test.ts | 353 ----- test/post-merge-e2e-risk-gate.test.ts | 593 -------- test/pr-risk-plan.test.ts | 29 +- test/required-live-workflow.test.ts | 396 ++++++ test/required-live.test.ts | 769 ++++++++++ tools/advisors/risk-plan.mts | 12 +- tools/e2e-advisor/README.md | 103 +- tools/e2e-advisor/post-merge-risk-gate.mts | 965 ------------- tools/e2e/operations-workflow-boundary.mts | 137 +- tools/{e2e-advisor => e2e}/private-file.ts | 0 tools/e2e/required-live.mts | 1261 +++++++++++++++++ tools/{e2e-advisor => e2e}/risk-signal.ts | 0 tools/pr-review-advisor/README.md | 11 +- tools/pr-review-advisor/analyze.mts | 4 - 25 files changed, 3055 insertions(+), 2346 deletions(-) delete mode 100644 .github/workflows/post-merge-e2e-risk-gate-shadow.yaml create mode 100644 .github/workflows/required-live-e2e.yaml delete mode 100644 test/post-merge-e2e-risk-gate-workflow.test.ts delete mode 100644 test/post-merge-e2e-risk-gate.test.ts create mode 100644 test/required-live-workflow.test.ts create mode 100644 test/required-live.test.ts delete mode 100755 tools/e2e-advisor/post-merge-risk-gate.mts rename tools/{e2e-advisor => e2e}/private-file.ts (100%) create mode 100755 tools/e2e/required-live.mts rename tools/{e2e-advisor => e2e}/risk-signal.ts (100%) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 8d0a63ab293..1d1853c3541 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 name: E2E -run-name: ${{ inputs.risk_correlation != '' && format('E2E risk {0}', inputs.risk_correlation) || format('E2E {0}', github.ref_name) }} +run-name: "${{ inputs.checkout_sha != '' && format('E2E PR #{0} required live {1}', inputs.pr_number, inputs.correlation_id) || format('E2E {0}', github.ref_name) }}" on: schedule: @@ -35,39 +35,34 @@ on: default: false type: boolean checkout_sha: - description: Current main commit selected by the trusted post-merge shadow controller. + description: Immutable pull request commit selected by the trusted required-live controller. required: false default: "" type: string - risk_plan_hash: - description: Deterministic post-merge risk-plan hash for shadow evidence correlation. + plan_hash: + description: Deterministic required-live plan hash for evidence correlation. required: false default: "" type: string - risk_correlation: - description: UUIDv4 correlation id for an exact-commit shadow risk run. + correlation_id: + description: UUIDv4 correlation id for a required-live run. required: false default: "" type: string - risk_shadow: - description: Mark this selective run as post-merge risk-gate shadow evidence. - required: false - default: false - type: boolean permissions: contents: read + pull-requests: read concurrency: - group: e2e-${{ github.ref }}-${{ inputs.risk_shadow && github.run_id || inputs.targets || 'supported' }}-${{ inputs.risk_shadow && 'risk-shadow' || inputs.jobs || 'all-jobs' }} - cancel-in-progress: false + group: e2e-${{ github.ref }}-${{ inputs.checkout_sha != '' && format('pr-{0}', inputs.pr_number) || inputs.targets || 'supported' }}-${{ inputs.checkout_sha != '' && 'required-live' || inputs.jobs || 'all-jobs' }} + cancel-in-progress: ${{ inputs.checkout_sha != '' }} env: NEMOCLAW_E2E_EXPECTED_SHA: ${{ inputs.checkout_sha }} - NEMOCLAW_E2E_RISK_PLAN_HASH: ${{ inputs.risk_plan_hash }} - NEMOCLAW_E2E_RISK_CORRELATION: ${{ inputs.risk_correlation }} - NEMOCLAW_E2E_RISK_SHARD: default - NEMOCLAW_E2E_RISK_SHADOW: ${{ inputs.risk_shadow && '1' || '0' }} + NEMOCLAW_E2E_PLAN_HASH: ${{ inputs.plan_hash }} + NEMOCLAW_E2E_CORRELATION_ID: ${{ inputs.correlation_id }} + NEMOCLAW_E2E_SHARD: default jobs: generate-matrix: @@ -83,29 +78,36 @@ jobs: fetch-depth: 0 persist-credentials: false - - name: Validate exact-commit dispatch - if: ${{ inputs.checkout_sha != '' || inputs.risk_plan_hash != '' || inputs.risk_correlation != '' || inputs.risk_shadow }} + - name: Validate required-live dispatch + if: ${{ inputs.checkout_sha != '' }} env: CHECKOUT_SHA: ${{ inputs.checkout_sha }} + GITHUB_TOKEN: ${{ github.token }} JOBS: ${{ inputs.jobs }} - PLAN_HASH: ${{ inputs.risk_plan_hash }} - RISK_CORRELATION: ${{ inputs.risk_correlation }} - RISK_SHADOW: ${{ inputs.risk_shadow }} + PLAN_HASH: ${{ inputs.plan_hash }} + PR_NUMBER: ${{ inputs.pr_number }} + CORRELATION_ID: ${{ inputs.correlation_id }} TARGETS: ${{ inputs.targets }} WORKFLOW_EVENT: ${{ github.event_name }} WORKFLOW_REF: ${{ github.ref }} - WORKFLOW_SHA: ${{ github.sha }} run: | set -euo pipefail - [[ "$WORKFLOW_EVENT" == "workflow_dispatch" && "$WORKFLOW_REF" == "refs/heads/main" ]] || { echo "::error::exact-commit runs require a workflow_dispatch from main"; exit 1; } - [[ "$RISK_SHADOW" == "true" ]] || { echo "::error::exact-commit inputs require risk_shadow=true"; exit 1; } + [[ "$WORKFLOW_EVENT" == "workflow_dispatch" && "$WORKFLOW_REF" == "refs/heads/main" ]] || { echo "::error::required-live runs require a workflow_dispatch from main"; exit 1; } [[ "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ ]] || { echo "::error::checkout_sha must be a lowercase 40-character SHA"; exit 1; } - [[ "$CHECKOUT_SHA" == "$WORKFLOW_SHA" ]] || { echo "::error::checkout_sha must equal the current main workflow commit"; exit 1; } - [[ "$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]] || { echo "::error::checked-out HEAD does not match checkout_sha"; exit 1; } - git merge-base --is-ancestor "$CHECKOUT_SHA" origin/main || { echo "::error::checkout_sha must already be reachable from main"; exit 1; } - [[ "$PLAN_HASH" =~ ^[a-f0-9]{64}$ ]] || { echo "::error::risk_plan_hash must be a lowercase SHA-256"; exit 1; } - [[ "$RISK_CORRELATION" =~ ^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$ ]] || { echo "::error::risk_correlation must be a lowercase UUIDv4"; exit 1; } - [[ -n "$JOBS" && -z "$TARGETS" ]] || { echo "::error::shadow risk runs require selective jobs and forbid targets/fan-out"; exit 1; } + [[ "$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]] || { echo "::error::checked-out commit does not match checkout_sha"; exit 1; } + [[ "$PLAN_HASH" =~ ^[a-f0-9]{64}$ ]] || { echo "::error::plan_hash must be a lowercase SHA-256"; exit 1; } + [[ "$CORRELATION_ID" =~ ^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$ ]] || { echo "::error::correlation_id must be a lowercase UUIDv4"; exit 1; } + [[ "$PR_NUMBER" =~ ^[1-9][0-9]*$ ]] || { echo "::error::pr_number must be a positive integer"; exit 1; } + [[ -n "$JOBS" && -z "$TARGETS" ]] || { echo "::error::required-live runs require selective jobs and forbid targets/fan-out"; exit 1; } + + pull_json="$(curl --fail --silent --show-error --proto '=https' \ + --header "Authorization: Bearer ${GITHUB_TOKEN}" \ + --header "Accept: application/vnd.github+json" \ + --header "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}")" + [[ "$(jq -r '.state' <<< "$pull_json")" == "open" ]] || { echo "::error::pull request must still be open"; exit 1; } + [[ "$(jq -r '.head.repo.full_name // ""' <<< "$pull_json")" == "$GITHUB_REPOSITORY" ]] || { echo "::error::pull request must originate from this repository"; exit 1; } + [[ "$(jq -r '.head.sha' <<< "$pull_json")" == "$CHECKOUT_SHA" ]] || { echo "::error::checkout_sha must match the pull request's current commit"; exit 1; } - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 @@ -3036,7 +3038,7 @@ jobs: # set, and NoNewPrivs remain evidence unless an opt-in expectation is set. NEMOCLAW_E2E_EXPECT_NON_ROOT_HOST: "1" NEMOCLAW_E2E_SECURITY_POSTURE: "1" - NEMOCLAW_E2E_RISK_SHARD: ${{ matrix.agent }} + NEMOCLAW_E2E_SHARD: ${{ matrix.agent }} NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS: "60" NEMOCLAW_RECREATE_SANDBOX: "1" @@ -4559,7 +4561,7 @@ jobs: NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" - NEMOCLAW_E2E_RISK_SHARD: ${{ matrix.agent }} + NEMOCLAW_E2E_SHARD: ${{ matrix.agent }} NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" NEMOCLAW_AGENT: ${{ matrix.agent }} @@ -5035,7 +5037,7 @@ jobs: channels-stop-start, spark-install, ] - if: ${{ always() && github.event_name == 'workflow_dispatch' && !inputs.risk_shadow }} + if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '' }} permissions: # The issue-comment endpoint accepts pull request write permission for PR comments. # Keep issues: write absent so this job cannot restore general issue routing. @@ -5238,7 +5240,7 @@ jobs: scorecard: runs-on: ubuntu-latest needs: *e2e-result-jobs - if: ${{ always() && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && !inputs.risk_shadow)) }} + if: ${{ always() && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '')) }} permissions: actions: read contents: read diff --git a/.github/workflows/post-merge-e2e-risk-gate-shadow.yaml b/.github/workflows/post-merge-e2e-risk-gate-shadow.yaml deleted file mode 100644 index d05a6598723..00000000000 --- a/.github/workflows/post-merge-e2e-risk-gate-shadow.yaml +++ /dev/null @@ -1,249 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -name: E2E / Post-merge Risk Gate Shadow - -on: - push: - branches: [main] - -permissions: - # Required to dispatch the separate E2E workflow; no other Actions mutation is performed. - actions: write - checks: write - contents: read - -concurrency: - group: e2e-post-merge-risk-gate-${{ github.sha }} - cancel-in-progress: false - -jobs: - shadow: - if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' }} - runs-on: ubuntu-latest - timeout-minutes: 120 - steps: - - name: Checkout trusted controller - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - ref: ${{ github.event.after }} - fetch-depth: 0 - persist-credentials: false - - - name: Setup Node - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 - with: - node-version: "22" - cache: npm - - - name: Install trusted controller dependencies - run: npm ci --ignore-scripts - - - id: workspace - name: Create private controller workspace - shell: bash - run: | - set -euo pipefail - work_dir="$(mktemp -d "${RUNNER_TEMP}/nemoclaw-e2e-risk-gate.XXXXXX")" - chmod 700 "$work_dir" - printf 'work_dir=%s\n' "$work_dir" >> "$GITHUB_OUTPUT" - - - id: start - name: Build plan and dispatch exact-commit E2E - env: - GITHUB_TOKEN: ${{ github.token }} - run: >- - node --experimental-strip-types tools/e2e-advisor/post-merge-risk-gate.mts - --mode start - --base "${{ github.event.before }}" - --commit "${{ github.event.after }}" - --work-dir "${{ steps.workspace.outputs.work_dir }}" - - - name: Upload post-merge risk plan - if: ${{ always() }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: post-merge-risk-plan-${{ github.event.after }} - path: ${{ steps.workspace.outputs.work_dir }}/post-merge-risk-plan.json - if-no-files-found: ignore - retention-days: 14 - - - name: Close shadow check after controller startup failure - if: ${{ always() && steps.start.outputs.check_id != '' && steps.start.outputs.dispatched != 'true' && steps.start.outputs.finalized != 'true' }} - env: - GITHUB_TOKEN: ${{ github.token }} - run: >- - node --experimental-strip-types tools/e2e-advisor/post-merge-risk-gate.mts - --mode abandon - --check-id "${{ steps.start.outputs.check_id }}" - - - id: wait - name: Wait for correlated E2E run - if: ${{ steps.start.outputs.dispatched == 'true' }} - continue-on-error: true - env: - GH_TOKEN: ${{ github.token }} - RUN_ID: ${{ steps.start.outputs.run_id }} - run: | - wait_status=0 - timeout --signal=TERM --kill-after=30s 105m bash -s <<'WAIT' || wait_status=$? - set -euo pipefail - - if [[ ! "$RUN_ID" =~ ^[1-9][0-9]*$ ]]; then - printf '::error title=Invalid correlated E2E run ID::The controller did not provide a positive numeric run ID.\n' >&2 - exit 1 - fi - - run_url="https://github.com/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}" - - last_state="" - while true; do - if ! state="$( - gh run view "$RUN_ID" --repo "$GITHUB_REPOSITORY" \ - --json status,conclusion \ - --jq '.status + ":" + (if (.conclusion == null or .conclusion == "") then "none" else .conclusion end)' - )"; then - printf '::error title=Correlated E2E status query failed::Unable to query child run %s. %s\n' \ - "$RUN_ID" "$run_url" >&2 - exit 1 - fi - - if [[ "$state" != "$last_state" ]]; then - case "$state" in - queued:none | in_progress:none | requested:none | waiting:none | pending:none) - printf 'Correlated E2E run %s status=%s url=%s\n' \ - "$RUN_ID" "${state%%:*}" "$run_url" - ;; - completed:success) - printf 'Correlated E2E run %s status=completed conclusion=success url=%s\n' \ - "$RUN_ID" "$run_url" - ;; - completed:failure | completed:cancelled | completed:timed_out | completed:action_required | completed:neutral | completed:skipped | completed:stale | completed:startup_failure) - printf '::error title=Correlated E2E run did not succeed::Run %s completed with conclusion %s. %s\n' \ - "$RUN_ID" "${state#*:}" "$run_url" >&2 - ;; - *) - printf '::error title=Unexpected correlated E2E state::Run %s returned an unsupported status/conclusion pair. %s\n' \ - "$RUN_ID" "$run_url" >&2 - ;; - esac - last_state="$state" - fi - - case "$state" in - queued:none | in_progress:none | requested:none | waiting:none | pending:none) - sleep 10 - ;; - completed:success) - exit 0 - ;; - *) - exit 1 - ;; - esac - done - WAIT - - if [ "$wait_status" -eq 124 ]; then - printf '::error title=Correlated E2E wait timed out::The child run did not complete within 105 minutes.\n' >&2 - fi - exit "$wait_status" - - - id: evidence - name: Download correlated E2E evidence - if: ${{ always() && steps.start.outputs.dispatched == 'true' }} - continue-on-error: true - env: - GH_TOKEN: ${{ github.token }} - RUN_ID: ${{ steps.start.outputs.run_id }} - run: >- - gh run download "$RUN_ID" --repo "$GITHUB_REPOSITORY" - --dir "${{ steps.workspace.outputs.work_dir }}/evidence" - - - id: finish - name: Complete exact-commit shadow check - if: ${{ always() && steps.start.outputs.dispatched == 'true' }} - env: - GITHUB_TOKEN: ${{ github.token }} - run: >- - node --experimental-strip-types tools/e2e-advisor/post-merge-risk-gate.mts - --mode finish - --work-dir "${{ steps.workspace.outputs.work_dir }}" - --state-hash "${{ steps.start.outputs.state_hash }}" - --check-id "${{ steps.start.outputs.check_id }}" - --run-id "${{ steps.start.outputs.run_id }}" - - - name: Close shadow check after completion failure - if: ${{ always() && steps.start.outputs.check_id != '' && steps.start.outputs.dispatched == 'true' && steps.finish.outcome == 'failure' && steps.finish.outputs.finalized != 'true' }} - env: - GITHUB_TOKEN: ${{ github.token }} - run: >- - node --experimental-strip-types tools/e2e-advisor/post-merge-risk-gate.mts - --mode abandon - --check-id "${{ steps.start.outputs.check_id }}" - - - name: Summarize shadow controller - if: ${{ always() }} - env: - GH_TOKEN: ${{ github.token }} - COMMIT_SHA: ${{ github.event.after }} - CHILD_RUN_ID: ${{ steps.start.outputs.run_id }} - WORK_DIR: ${{ steps.workspace.outputs.work_dir }} - START_OUTCOME: ${{ steps.start.outcome }} - WAIT_OUTCOME: ${{ steps.wait.outcome }} - EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }} - FINISH_OUTCOME: ${{ steps.finish.outcome }} - run: | - set -euo pipefail - - selected_jobs="unavailable" - plan_path="${WORK_DIR}/post-merge-risk-plan.json" - if [ -n "$WORK_DIR" ] && [ -f "$plan_path" ]; then - selected_jobs="$(node -e ' - const fs = require("node:fs"); - const plan = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); - process.stdout.write(plan.automaticJobs.join(",") || "none"); - ' "$plan_path" 2>/dev/null || printf unavailable)" - fi - - child_conclusion="not-dispatched" - if [ -n "$CHILD_RUN_ID" ]; then - child_conclusion="$( - gh run view "$CHILD_RUN_ID" --repo "$GITHUB_REPOSITORY" \ - --json conclusion --jq '.conclusion // "none"' 2>/dev/null || printf unavailable - )" - fi - - failure_phase="none" - if [ "$START_OUTCOME" = "failure" ]; then - failure_phase="plan-and-dispatch" - elif [ "$FINISH_OUTCOME" = "failure" ]; then - failure_phase="evidence-verification" - elif [ "$WAIT_OUTCOME" = "failure" ]; then - failure_phase="child-run-or-wait" - elif [ "$EVIDENCE_OUTCOME" = "failure" ]; then - failure_phase="evidence-download" - fi - - { - echo "## Post-merge E2E risk gate shadow" - echo - printf -- '- Controller run: [run `%s`](https://github.com/%s/actions/runs/%s)\n' \ - "$GITHUB_RUN_ID" "$GITHUB_REPOSITORY" "$GITHUB_RUN_ID" - printf -- '- Controller commit: `%s`\n' "$COMMIT_SHA" - printf -- '- Selected jobs: `%s`\n' "$selected_jobs" - printf -- '- Plan and dispatch: `%s`\n' "$START_OUTCOME" - if [ -n "$CHILD_RUN_ID" ]; then - printf -- '- Correlated E2E: [run `%s`](https://github.com/%s/actions/runs/%s)\n' \ - "$CHILD_RUN_ID" "$GITHUB_REPOSITORY" "$CHILD_RUN_ID" - fi - printf -- '- Child conclusion: `%s`\n' "$child_conclusion" - printf -- '- Child wait: `%s`\n' "$WAIT_OUTCOME" - printf -- '- Evidence download: `%s`\n' "$EVIDENCE_OUTCOME" - printf -- '- Evidence verification: `%s`\n' "$FINISH_OUTCOME" - printf -- '- Failure phase: `%s`\n' "$failure_phase" - } >> "$GITHUB_STEP_SUMMARY" - - - name: Remove private controller workspace - if: ${{ always() && steps.workspace.outputs.work_dir != '' }} - run: rm -rf -- "${{ steps.workspace.outputs.work_dir }}" diff --git a/.github/workflows/required-live-e2e.yaml b/.github/workflows/required-live-e2e.yaml new file mode 100644 index 00000000000..c2c75c6d935 --- /dev/null +++ b/.github/workflows/required-live-e2e.yaml @@ -0,0 +1,220 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: E2E / Required Live + +on: + workflow_run: + workflows: ["CI / Pull Request"] + types: [completed] + pull_request_target: + types: [synchronize, reopened, closed] + +permissions: {} + +jobs: + cancel-superseded: + if: ${{ github.event_name == 'pull_request_target' && github.repository == 'NVIDIA/NemoClaw' }} + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + actions: write + contents: read + steps: + - name: Checkout trusted controller + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: ${{ github.workflow_sha }} + persist-credentials: false + + - name: Setup Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: "22" + + - name: Install trusted controller dependencies + run: npm ci --ignore-scripts + + - name: Cancel superseded required live runs + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + node --experimental-strip-types tools/e2e/required-live.mts + --mode cancel + --pr "${{ github.event.pull_request.number }}" + + coordinate: + if: ${{ github.event_name == 'workflow_run' && github.repository == 'NVIDIA/NemoClaw' && github.event.workflow_run.event == 'pull_request' }} + runs-on: ubuntu-latest + timeout-minutes: 180 + permissions: + actions: write + checks: write + contents: read + pull-requests: read + concurrency: + group: required-live-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }} + # Let an older coordinator observe child cancellation and close its check. + cancel-in-progress: false + steps: + - name: Checkout trusted controller + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: ${{ github.workflow_sha }} + persist-credentials: false + + - name: Setup Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: "22" + + - name: Install trusted controller dependencies + run: npm ci --ignore-scripts + + - id: workspace + name: Create private controller workspace + shell: bash + run: | + set -euo pipefail + work_dir="$(mktemp -d "${RUNNER_TEMP}/nemoclaw-required-live.XXXXXX")" + chmod 700 "$work_dir" + printf 'work_dir=%s\n' "$work_dir" >> "$GITHUB_OUTPUT" + + - id: start + name: Start required live evaluation + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + node --experimental-strip-types tools/e2e/required-live.mts + --mode start + --head "${{ github.event.workflow_run.head_sha }}" + --head-repo "${{ github.event.workflow_run.head_repository.full_name }}" + --head-branch "${{ github.event.workflow_run.head_branch }}" + --workflow-sha "${{ github.workflow_sha }}" + --ci-conclusion "${{ github.event.workflow_run.conclusion }}" + --work-dir "${{ steps.workspace.outputs.work_dir }}" + + - name: Upload required live plan + if: ${{ always() && steps.workspace.outputs.work_dir != '' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: required-live-plan-${{ github.event.workflow_run.head_sha }} + path: ${{ steps.workspace.outputs.work_dir }}/required-live-plan.json + if-no-files-found: ignore + retention-days: 14 + + - id: wait + name: Wait for required live run + if: ${{ steps.start.outputs.dispatched == 'true' }} + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + RUN_ID: ${{ steps.start.outputs.run_id }} + run: | + wait_status=0 + timeout --signal=TERM --kill-after=30s 105m bash -s <<'WAIT' || wait_status=$? + set -euo pipefail + + if [[ ! "$RUN_ID" =~ ^[1-9][0-9]*$ ]]; then + printf '::error title=Invalid required live run ID::The controller did not provide a positive numeric run ID.\n' >&2 + exit 1 + fi + + run_url="https://github.com/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}" + last_state="" + while true; do + if ! state="$( + gh run view "$RUN_ID" --repo "$GITHUB_REPOSITORY" \ + --json status,conclusion \ + --jq '.status + ":" + (if (.conclusion == null or .conclusion == "") then "none" else .conclusion end)' + )"; then + printf '::error title=Required live status query failed::Unable to query run %s. %s\n' \ + "$RUN_ID" "$run_url" >&2 + exit 1 + fi + + if [[ "$state" != "$last_state" ]]; then + case "$state" in + queued:none | in_progress:none | requested:none | waiting:none | pending:none) + printf 'Required live run %s status=%s url=%s\n' \ + "$RUN_ID" "${state%%:*}" "$run_url" + ;; + completed:success) + printf 'Required live run %s status=completed conclusion=success url=%s\n' \ + "$RUN_ID" "$run_url" + ;; + completed:failure | completed:cancelled | completed:timed_out | completed:action_required | completed:neutral | completed:skipped | completed:stale | completed:startup_failure) + printf '::error title=Required live run did not succeed::Run %s completed with conclusion %s. %s\n' \ + "$RUN_ID" "${state#*:}" "$run_url" >&2 + ;; + *) + printf '::error title=Unexpected required live state::Run %s returned an unsupported status/conclusion pair. %s\n' \ + "$RUN_ID" "$run_url" >&2 + ;; + esac + last_state="$state" + fi + + case "$state" in + queued:none | in_progress:none | requested:none | waiting:none | pending:none) + sleep 10 + ;; + completed:success) + exit 0 + ;; + *) + exit 1 + ;; + esac + done + WAIT + + if [ "$wait_status" -eq 124 ]; then + printf '::error title=Required live wait timed out::The run did not complete within 105 minutes.\n' >&2 + fi + exit "$wait_status" + + - id: evidence + name: Download required live evidence + if: ${{ always() && steps.start.outputs.dispatched == 'true' }} + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + RUN_ID: ${{ steps.start.outputs.run_id }} + run: | + set -euo pipefail + download_status=0 + timeout --signal=TERM --kill-after=30s 10m \ + gh run download "$RUN_ID" --repo "$GITHUB_REPOSITORY" \ + --dir "${{ steps.workspace.outputs.work_dir }}/evidence" || download_status=$? + if [ "$download_status" -eq 124 ]; then + printf '::error title=Required live evidence download timed out::Artifact download exceeded 10 minutes.\n' >&2 + fi + exit "$download_status" + + - id: finish + name: Finish required live evaluation + if: ${{ always() && steps.start.outputs.dispatched == 'true' }} + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + node --experimental-strip-types tools/e2e/required-live.mts + --mode finish + --work-dir "${{ steps.workspace.outputs.work_dir }}" + --state-hash "${{ steps.start.outputs.state_hash }}" + --check-id "${{ steps.start.outputs.check_id }}" + --run-id "${{ steps.start.outputs.run_id }}" + + - name: Close incomplete required live check + if: ${{ always() && steps.start.outputs.check_id != '' && steps.start.outputs.finalized != 'true' && steps.finish.outputs.finalized != 'true' }} + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + node --experimental-strip-types tools/e2e/required-live.mts + --mode abandon + --check-id "${{ steps.start.outputs.check_id }}" + --run-id "${{ steps.start.outputs.run_id }}" + + - name: Remove private controller workspace + if: ${{ always() && steps.workspace.outputs.work_dir != '' }} + run: rm -rf -- "${{ steps.workspace.outputs.work_dir }}" diff --git a/docs/about/release-notes.mdx b/docs/about/release-notes.mdx index e20e2dbfba0..e4ecc91df92 100644 --- a/docs/about/release-notes.mdx +++ b/docs/about/release-notes.mdx @@ -42,7 +42,7 @@ The release also refreshes quickstarts and variant rendering so OpenClaw, Hermes Resume recovery now follows one explicit finite-state path, pending route reservations survive resume, sandbox create-failure reporting is separated from create-step handling, BuildKit progress no longer forces plain output, and null-name resume sessions are covered so canceled or malformed session state does not send users down the wrong recovery path. For more information, refer to [NemoClaw Quickstart with OpenClaw](../get-started/quickstart), [NemoClaw CLI Commands Reference](../reference/commands), [Manage Sandbox Lifecycle](../manage-sandboxes/lifecycle), and [Troubleshooting](../reference/troubleshooting). - Documentation and release validation are more deterministic. - The docs define extension taxonomy and SDK readiness gates, streamline the agent quickstarts, clarify legacy k3s sandbox resources, preserve list spacing in generated agent-variant pages, and add release-train risk planning with post-merge E2E shadow signals, queued Jetson dispatch guards, TUI idle regression coverage, and reusable live-readiness polling primitives. + The docs define extension taxonomy and SDK readiness gates, streamline the agent quickstarts, clarify legacy k3s sandbox resources, preserve list spacing in generated agent-variant pages, and add release-train risk planning with a deterministic required live E2E check for pull requests, queued Jetson dispatch guards, TUI idle regression coverage, and reusable live-readiness polling primitives. For more information, refer to [Extension Taxonomy and SDK Readiness](../reference/extension-taxonomy-sdk-readiness), [NemoClaw Quickstart with OpenClaw](../get-started/quickstart), [Architecture Details](../reference/architecture), and the [NemoClaw E2E README](https://github.com/NVIDIA/NemoClaw/blob/main/test/e2e/README.md). ## v0.0.78 diff --git a/test/e2e-advisor-targets.test.ts b/test/e2e-advisor-targets.test.ts index 1d1f6c1cff3..8ad25d5a313 100644 --- a/test/e2e-advisor-targets.test.ts +++ b/test/e2e-advisor-targets.test.ts @@ -68,7 +68,7 @@ describe("E2E target advisor — prompt construction", () => { expect(turn.contextToolResults?.[1]?.content).toContain( "test/e2e/fixtures/phases/onboarding.ts", ); - expect(turn.contextToolResults?.[2]?.content).toContain('"version":1'); + expect(turn.contextToolResults?.[2]?.content).toContain('"version":2'); expect(turn.contextToolResults?.[3]?.content).toContain("+ echo ok"); expect(turn.contextToolResults?.[4]?.content).toContain("test-schema"); for (const result of turn.contextToolResults ?? []) { diff --git a/test/e2e-private-file.test.ts b/test/e2e-private-file.test.ts index fe4c54c8645..dc8d779df9b 100644 --- a/test/e2e-private-file.test.ts +++ b/test/e2e-private-file.test.ts @@ -8,10 +8,7 @@ import path from "node:path"; import { pathToFileURL } from "node:url"; import { describe, expect, it } from "vitest"; -import { - readPrivateRegularFile, - writePrivateRegularFile, -} from "../tools/e2e-advisor/private-file.ts"; +import { readPrivateRegularFile, writePrivateRegularFile } from "../tools/e2e/private-file.ts"; describe("private E2E controller files", () => { it("writes private regular files without following links or truncating hardlink targets", () => { @@ -44,7 +41,7 @@ describe("private E2E controller files", () => { const fifo = path.join(directory, "state.json"); try { execFileSync("mkfifo", [fifo]); - const moduleUrl = pathToFileURL(path.resolve("tools/e2e-advisor/private-file.ts")).href; + const moduleUrl = pathToFileURL(path.resolve("tools/e2e/private-file.ts")).href; const read = spawnSync( process.execPath, [ diff --git a/test/e2e-risk-signal-reporter.test.ts b/test/e2e-risk-signal-reporter.test.ts index b45f4f47105..2be2eaf2abd 100644 --- a/test/e2e-risk-signal-reporter.test.ts +++ b/test/e2e-risk-signal-reporter.test.ts @@ -41,12 +41,12 @@ function environment(artifactDir: string): RiskSignalEnvironment { } describe("E2E risk signal reporter", () => { - it("stays disabled outside shadow runs", () => { + it("stays disabled when no expected commit is configured", () => { expect(configuredEnvironment({})).toBeNull(); }); - it("fails closed when shadow metadata is incomplete", () => { - expect(() => configuredEnvironment({ NEMOCLAW_E2E_RISK_SHADOW: "1" })).toThrow( + it("fails closed when required-live metadata is incomplete", () => { + expect(() => configuredEnvironment({ NEMOCLAW_E2E_EXPECTED_SHA: EXPECTED_SHA })).toThrow( /E2E_ARTIFACT_DIR/u, ); }); @@ -57,10 +57,9 @@ describe("E2E risk signal reporter", () => { E2E_TARGET_ID: "onboard-resume", GITHUB_WORKSPACE: "/workspace", NEMOCLAW_E2E_EXPECTED_SHA: EXPECTED_SHA, - NEMOCLAW_E2E_RISK_PLAN_HASH: PLAN_HASH, - NEMOCLAW_E2E_RISK_CORRELATION: CORRELATION_ID, - NEMOCLAW_E2E_RISK_SHARD: "default", - NEMOCLAW_E2E_RISK_SHADOW: "1", + NEMOCLAW_E2E_PLAN_HASH: PLAN_HASH, + NEMOCLAW_E2E_CORRELATION_ID: CORRELATION_ID, + NEMOCLAW_E2E_SHARD: "default", }; expect(configuredEnvironment(env, () => EXPECTED_SHA)?.testedSha).toBe(EXPECTED_SHA); diff --git a/test/e2e/README.md b/test/e2e/README.md index 8709ec27630..7bd4b156d37 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -8,11 +8,10 @@ Direct E2E coverage runs through Vitest. Interactive TUI targets require `expect`. The unified workflow installs it before those targets run; local runners must provide it themselves. -- `.github/workflows/e2e.yaml` is the scheduled and manually - dispatchable live target workflow. -- `.github/workflows/post-merge-e2e-risk-gate-shadow.yaml` is the trusted post-merge - controller that selects and dispatches a bounded exact-commit subset after - pushes to `main`. +- `.github/workflows/e2e.yaml` is the scheduled, manually dispatchable, and + selectively dispatched live target workflow. +- `.github/workflows/required-live-e2e.yaml` is the trusted pull request + controller that owns the required `E2E / Required Live` check. - `.github/workflows/e2e-branch-validation.yaml` provisions Brev instances and runs focused E2E targets from source on a clean machine. - Platform workflows such as macOS, WSL, Ollama proxy, sandbox image, and @@ -56,41 +55,64 @@ artifact so baseline aggregation stays stable. Older issue references to Vitest target artifacts under `e2e-artifacts/vitest/` map to this consolidated `e2e-artifacts/live/` registry-target artifact layout. -## Post-merge risk shadow - -Every push to the `main` branch of `NVIDIA/NemoClaw` starts a model-independent -shadow controller. It builds the deterministic risk plan from the exact -`github.event.before` and `github.event.after` range after confirming that its checkout -matches the pushed commit. If the plan matches runtime regression families, -the controller dispatches at most three `automaticJobs` through `e2e.yaml`. -The workflow definition stays on `main`, while every E2E checkout uses the -merged commit supplied through `checkout_sha`. GitHub returns the -dispatched workflow's run ID directly, and the controller uses that ID as the -sole child-run selector for waiting, evidence download, and completion. - -Before E2E preparation or selected jobs can use repository secrets, -`e2e.yaml` verifies that the requested SHA equals the workflow's own current -`main` commit, confirms that checked-out `HEAD` matches it, proves reachability, -and accepts only selective job dispatch with valid plan and correlation -metadata. If `main` advances before an older controller dispatches, that child -fails closed and the controller records failure instead of running historical -code with current secrets. The shadow-only Vitest -reporter then writes a `risk-signal.json` for each selected job and matrix -shard. Each signal binds the observed checkout SHA, expected SHA, plan hash, +## Required live PR check + +When `CI / Pull Request` completes for a same-repository pull request, the +trusted `.github/workflows/required-live-e2e.yaml` workflow creates the +`E2E / Required Live` check for that revision. +The model-independent controller resolves the open pull request, reads its +complete changed-file list from GitHub, and builds the deterministic risk plan. +If runtime regression families match, it dispatches every selected +`requiredJobs` entry through `e2e.yaml`. +If no family matches, the check succeeds without dispatching live E2E. + +The controller verifies that the pull request did not change while the plan +was prepared. +It records the trusted workflow revision, requires that revision to remain the +current `main` revision immediately before dispatch, and accepts only a child +workflow run created from that same revision. +The `e2e.yaml` workflow definition stays on `main`, while each selected job +checks out the pull request revision supplied through `checkout_sha`. +Before E2E preparation or selected jobs can use repository secrets, the child +workflow verifies that the pull request is still open, comes from +`NVIDIA/NemoClaw`, and still points to that revision. +It also accepts only selective job dispatches with an empty target fan-out and +valid plan and correlation metadata. +GitHub returns the dispatched workflow's run ID directly, and the controller +uses that ID as the sole child-run selector for waiting, evidence download, +and completion. + +The Vitest reporter writes one `risk-signal.json` for each selected job and +matrix shard. +The checked workflow boundary requires every policy-selected job to expose its +matching job identity, attach the reporter to every Vitest invocation, and +always upload its evidence artifact. +Each signal binds the observed checkout SHA, expected SHA, plan hash, correlation ID, and pass, failure, skip, pending, and unhandled-error counts. -The controller retains `post-merge-risk-plan-` for 14 days, while each +The controller retains `required-live-plan-` for 14 days, while each signal travels in the selected job's existing E2E artifact. - -The controller reports `E2E / Post-merge Risk Gate (shadow)` on the merged -commit. It reports success only when every expected shard produces a complete, -unskipped pass and the three-job cap did not omit required jobs. Selected E2E -workflow or test failures for the merged commit report failure. Missing, partial, skipped, -ambiguous, or manual-expansion evidence reports neutral. A plan with no matched -runtime risk reports success without dispatching live E2E. This shadow check runs -after merge and is not a required PR check. It disables PR comments and the -scheduled/manual scorecard, including scorecard Slack reporting. -Controller or evidence-verification errors close an already-created check as -neutral so incomplete evidence cannot appear successful. +Its private dispatch state is protected by a SHA-256 digest that is verified +before downloaded evidence is classified. + +The required check has a binary result. +It succeeds only when the correlated E2E workflow succeeds and every expected +job shard produces one complete, unskipped pass. +Workflow or test failures, missing or duplicate signals, skipped or pending +tests, interrupted runs, and controller or evidence-validation errors fail the +check. +The coordinator has a 180-minute job budget and gives evidence download its +own 10-minute limit, so a stalled download fails instead of consuming the +remaining coordination time. +Required-live dispatches suppress PR comments and the scheduled or manual +scorecard, including scorecard Slack reporting. + +Pull request synchronization, reopening, or closure cancels active child runs +for that pull request. +The E2E workflow also cancels a superseded child run when a new revision is +dispatched, while the earlier controller remains available to close its check +as failed. +The controller does not read PR Review Advisor or E2E Advisor output, so model +availability and recommendations are not part of merge authority. ## Onboard performance budget diff --git a/test/e2e/docs/README.md b/test/e2e/docs/README.md index 27eaceda958..c7e68d815ab 100644 --- a/test/e2e/docs/README.md +++ b/test/e2e/docs/README.md @@ -106,16 +106,21 @@ test/e2e/ ## CI Entry Points - `tools/advisors/risk-plan.mts` is the small deterministic selection policy - shared by PR Review Advisor, E2E Advisor, and the model-independent post-merge - shadow controller. It maps changed runtime surfaces to invariant families and + shared by PR Review Advisor, E2E Advisor, and the model-independent required + live controller. It maps changed runtime surfaces to invariant families and canonical `e2e.yaml` jobs; it is not a second test runner or migration-status - ledger. - -- `.github/workflows/post-merge-e2e-risk-gate-shadow.yaml` runs only for trusted pushes to - `main`. `tools/e2e-advisor/post-merge-risk-gate.mts` builds a plan from the exact - before/after SHAs, dispatches at most three automatic jobs, and validates - `risk-signal.json` evidence for every expected job and matrix shard. The - resulting check is post-merge shadow evidence, not a required PR gate. + ledger. The advisors use it as recommendation context, while the required + controller applies it independently without model output. + +- `.github/workflows/required-live-e2e.yaml` owns the required + `E2E / Required Live` check for same-repository pull request revisions after + `CI / Pull Request` completes. `tools/e2e/required-live.mts` builds a plan + from GitHub's complete pull request file list, dispatches every selected job, + and validates `risk-signal.json` evidence for every expected job and matrix + shard. It also requires its trusted workflow revision to remain current on + `main` immediately before dispatch and verifies that the child run uses that + revision. Pull request synchronization, reopening, or closure cancels active + child runs for that pull request. - `.github/workflows/e2e.yaml` runs selected or all supported live E2E targets and uploads an explicit artifact allowlist with @@ -128,12 +133,16 @@ test/e2e/ These per-target timing summaries are artifact evidence only. The Slack and GitHub scorecard timing comparison remains scoped to the dedicated `cloud-onboard` artifact. - Exact-commit shadow dispatches require the requested checkout to equal the - workflow's current `main` commit and verify its reachability before - preparation. The controller uses GitHub's returned workflow-dispatch run ID - as the sole child-run selector for waiting, evidence download, and - completion, attaches `test/e2e/risk-signal-reporter.ts` to live Vitest - invocations, and suppresses PR reporting and scorecards. + Required-live dispatches require an open pull request from the base + repository whose current revision matches `checkout_sha` before preparation. + The controller uses GitHub's returned workflow-dispatch run ID as the sole + child-run selector for waiting, evidence download, and completion, attaches + `test/e2e/risk-signal-reporter.ts` to live Vitest invocations, and suppresses + PR reporting and scorecards. The workflow boundary requires every job named + by the deterministic policy to expose matching job identity, attach that + reporter to every Vitest invocation, and always upload one evidence artifact. + The check succeeds only for one complete, unskipped passing signal from every + expected job shard; every other outcome is a failure. - `.github/workflows/e2e-branch-validation.yaml`, `macos-e2e.yaml`, `wsl-e2e.yaml`, `ollama-proxy-e2e.yaml`, and `regression-e2e.yaml` call focused E2E targets directly for their E2E coverage. diff --git a/test/e2e/risk-signal-reporter.ts b/test/e2e/risk-signal-reporter.ts index 138ffeec14b..d500d250c84 100644 --- a/test/e2e/risk-signal-reporter.ts +++ b/test/e2e/risk-signal-reporter.ts @@ -7,11 +7,8 @@ import path from "node:path"; import type { TestModule } from "vitest/node"; import type { Reporter, TestRunEndReason } from "vitest/reporters"; -import { - readPrivateRegularFile, - writePrivateRegularFile, -} from "../../tools/e2e-advisor/private-file.ts"; -import type { E2eRiskSignal } from "../../tools/e2e-advisor/risk-signal.ts"; +import { readPrivateRegularFile, writePrivateRegularFile } from "../../tools/e2e/private-file.ts"; +import type { E2eRiskSignal } from "../../tools/e2e/risk-signal.ts"; export const RISK_SIGNAL_FILE = "risk-signal.json"; @@ -44,14 +41,14 @@ export function configuredEnvironment( env: NodeJS.ProcessEnv, resolveHead: (workspace: string) => string = checkedOutSha, ): RiskSignalEnvironment | null { - if (env.NEMOCLAW_E2E_RISK_SHADOW !== "1") return null; + if (!env.NEMOCLAW_E2E_EXPECTED_SHA) return null; const values = { artifactDir: env.E2E_ARTIFACT_DIR ?? "", jobId: env.E2E_TARGET_ID ?? "", - shardId: env.NEMOCLAW_E2E_RISK_SHARD ?? "", - expectedSha: env.NEMOCLAW_E2E_EXPECTED_SHA ?? "", - planHash: env.NEMOCLAW_E2E_RISK_PLAN_HASH ?? "", - correlationId: env.NEMOCLAW_E2E_RISK_CORRELATION ?? "", + shardId: env.NEMOCLAW_E2E_SHARD ?? "", + expectedSha: env.NEMOCLAW_E2E_EXPECTED_SHA, + planHash: env.NEMOCLAW_E2E_PLAN_HASH ?? "", + correlationId: env.NEMOCLAW_E2E_CORRELATION_ID ?? "", }; if (!values.artifactDir) throw new Error("risk signal requires E2E_ARTIFACT_DIR"); if (!JOB_PATTERN.test(values.jobId)) throw new Error("risk signal requires a safe E2E_TARGET_ID"); diff --git a/test/e2e/support/e2e-operations-workflow-boundary.test.ts b/test/e2e/support/e2e-operations-workflow-boundary.test.ts index ec616eb535b..6b8ebe26285 100644 --- a/test/e2e/support/e2e-operations-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-operations-workflow-boundary.test.ts @@ -57,7 +57,7 @@ describe("E2E operations workflow boundary", () => { ); }); - it("keeps PR reporting and scorecards disabled for E2E risk shadow runs", () => { + it("keeps PR reporting and scorecards disabled for required-live runs", () => { const workflow = readE2eOperationsWorkflow(); workflow.jobs["report-to-pr"].if = "${{ always() && github.event_name == 'workflow_dispatch' }}"; @@ -72,6 +72,53 @@ describe("E2E operations workflow boundary", () => { ); }); + it("rejects required-live protocol and pull request validation drift", () => { + const workflow = readE2eOperationsWorkflow(); + delete workflow.on?.workflow_dispatch?.inputs?.plan_hash; + workflow.env!.NEMOCLAW_E2E_PLAN_HASH = "${{ inputs.checkout_sha }}"; + workflow.concurrency!["cancel-in-progress"] = false; + const validation = workflow.jobs["generate-matrix"].steps!.find( + (step) => step.name === "Validate required-live dispatch", + )!; + validation.if = "${{ inputs.plan_hash != '' }}"; + validation.run = "echo unchecked"; + const checkout = workflow.jobs["generate-matrix"].steps!.find((step) => + step.uses?.startsWith("actions/checkout@"), + )!; + checkout.with!.ref = "${{ github.sha }}"; + + expect(validateE2eOperationsWorkflow(workflow)).toEqual( + expect.arrayContaining([ + "workflow_dispatch plan_hash must be an optional string with an empty default", + "E2E workflow must bind NEMOCLAW_E2E_PLAN_HASH to required-live metadata", + "required-live concurrency must cancel obsolete pull request runs", + "required-live validation must be activated only by checkout_sha", + 'required-live validation must retain "$PR_NUMBER" =~ ^[1-9][0-9]*$', + "generate-matrix checkout must use the selected immutable commit", + ]), + ); + }); + + it("keeps every planned job wired to bound evidence", () => { + const workflow = readE2eOperationsWorkflow(); + const job = workflow.jobs["cloud-onboard"]; + job.env!.E2E_TARGET_ID = "different-job"; + const run = job.steps!.find((step) => String(step.run ?? "").includes("npx vitest"))!; + run.run = run.run!.replace("test/e2e/risk-signal-reporter.ts", "default"); + const upload = job.steps!.find((step) => + step.uses?.startsWith("NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@"), + )!; + upload.if = "success()"; + + expect(validateE2eOperationsWorkflow(workflow)).toEqual( + expect.arrayContaining([ + "cloud-onboard must expose matching required-live job identity", + "cloud-onboard must attach the required-live reporter to every Vitest invocation", + "cloud-onboard must always upload one required-live evidence artifact", + ]), + ); + }); + it("rejects restoration of scheduled issue routing or broad issue-write access", () => { const workflow = readE2eOperationsWorkflow(); workflow.permissions = "write-all"; diff --git a/test/post-merge-e2e-risk-gate-workflow.test.ts b/test/post-merge-e2e-risk-gate-workflow.test.ts deleted file mode 100644 index 28f8bbd80c4..00000000000 --- a/test/post-merge-e2e-risk-gate-workflow.test.ts +++ /dev/null @@ -1,353 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { spawnSync } from "node:child_process"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { describe, expect, it } from "vitest"; -import { RISK_RULES } from "../tools/advisors/risk-plan.mts"; -import { - readYaml, - type Workflow, - type WorkflowJob, - type WorkflowStep, -} from "./helpers/e2e-workflow-contract.ts"; - -const SHADOW_PATH = ".github/workflows/post-merge-e2e-risk-gate-shadow.yaml"; -const E2E_PATH = ".github/workflows/e2e.yaml"; - -type TriggeredWorkflow = Workflow & { - on?: Record; - permissions?: Record; - concurrency?: { group: string; "cancel-in-progress": boolean }; -}; - -function step(job: WorkflowJob, name: string): WorkflowStep { - const match = job.steps?.find((candidate) => candidate.name === name); - expect(match, `missing workflow step ${name}`).toBeDefined(); - return match!; -} - -function collectStrings(value: unknown): string[] { - return typeof value === "string" - ? [value] - : Array.isArray(value) - ? value.flatMap(collectStrings) - : value && typeof value === "object" - ? Object.values(value).flatMap(collectStrings) - : []; -} - -function runWaitStep( - scenario: "success" | "failure" | "query-failure" | "timeout" | "unsupported", - options: { runId?: string } = {}, -) { - const workflow = readYaml(SHADOW_PATH); - const wait = step(workflow.jobs.shadow, "Wait for correlated E2E run"); - const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-shadow-wait-")); - const binDir = path.join(tempDir, "bin"); - const callCountPath = path.join(tempDir, "gh-call-count"); - fs.mkdirSync(binDir); - fs.writeFileSync(callCountPath, "0\n"); - fs.writeFileSync( - path.join(binDir, "gh"), - `#!/usr/bin/env bash -set -euo pipefail -count="$(cat "$FAKE_GH_CALL_COUNT")" -count=$((count + 1)) -printf '%s\n' "$count" > "$FAKE_GH_CALL_COUNT" -case "$FAKE_GH_SCENARIO:$count" in - success:1 | success:2 | failure:1) printf 'in_progress:none\n' ;; - success:*) printf 'completed:success\n' ;; - failure:*) printf 'completed:failure\n' ;; - query-failure:*) printf 'simulated GitHub query failure\n' >&2; exit 1 ;; - unsupported:*) printf 'completed:unknown\n' ;; - *) exit 2 ;; -esac -`, - { mode: 0o755 }, - ); - fs.writeFileSync(path.join(binDir, "sleep"), "#!/usr/bin/env bash\nexit 0\n", { mode: 0o755 }); - fs.writeFileSync( - path.join(binDir, "timeout"), - `#!/usr/bin/env bash -set -euo pipefail -if [ "$FAKE_GH_SCENARIO" = "timeout" ]; then - exit 124 -fi -shift 3 -exec "$@" -`, - { mode: 0o755 }, - ); - - try { - const result = spawnSync("bash", ["-e", "-o", "pipefail", "-c", wait.run!], { - encoding: "utf8", - env: { - ...process.env, - FAKE_GH_CALL_COUNT: callCountPath, - FAKE_GH_SCENARIO: scenario, - GITHUB_REPOSITORY: "NVIDIA/NemoClaw", - PATH: `${binDir}:${process.env.PATH ?? ""}`, - RUN_ID: options.runId ?? "29110351531", - }, - timeout: 5_000, - }); - return { - ...result, - ghCallCount: Number(fs.readFileSync(callCountPath, "utf8").trim()), - }; - } finally { - fs.rmSync(tempDir, { recursive: true, force: true }); - } -} - -describe("post-merge E2E risk gate shadow workflow", () => { - it("uses a trusted main-push controller with minimal write permissions", () => { - const workflow = readYaml(SHADOW_PATH); - const job = workflow.jobs.shadow; - - expect(workflow.on).toEqual({ - push: { branches: ["main"] }, - }); - expect(workflow.permissions).toEqual({ - actions: "write", - checks: "write", - contents: "read", - }); - expect(job.if).toContain("github.repository == 'NVIDIA/NemoClaw'"); - expect(job.if).toContain("github.ref == 'refs/heads/main'"); - expect(collectStrings(workflow).some((value) => value.includes("${{ secrets."))).toBe(false); - }); - - it("builds the plan from the exact trusted push and bounds child-run waiting", () => { - const workflow = readYaml(SHADOW_PATH); - const job = workflow.jobs.shadow; - const checkout = step(job, "Checkout trusted controller"); - const workspace = step(job, "Create private controller workspace"); - const start = step(job, "Build plan and dispatch exact-commit E2E"); - const startupFallback = step(job, "Close shadow check after controller startup failure"); - const wait = step(job, "Wait for correlated E2E run"); - const download = step(job, "Download correlated E2E evidence"); - const finish = step(job, "Complete exact-commit shadow check"); - const completionFallback = step(job, "Close shadow check after completion failure"); - const summary = step(job, "Summarize shadow controller"); - const cleanup = step(job, "Remove private controller workspace"); - - expect(checkout.with).toMatchObject({ - ref: "${{ github.event.after }}", - "fetch-depth": 0, - "persist-credentials": false, - }); - expect(workspace.run).toContain('mktemp -d "${RUNNER_TEMP}/nemoclaw-e2e-risk-gate.XXXXXX"'); - expect(workspace.run).toContain('chmod 700 "$work_dir"'); - expect(start.run).toContain("post-merge-risk-gate.mts --mode start"); - expect(start.run).toContain('--base "${{ github.event.before }}"'); - expect(start.run).toContain('--commit "${{ github.event.after }}"'); - expect(start.run).toContain('--work-dir "${{ steps.workspace.outputs.work_dir }}"'); - expect(start["continue-on-error"]).not.toBe(true); - expect(startupFallback.if).toContain("always()"); - expect(startupFallback.if).toContain("steps.start.outputs.check_id != ''"); - expect(startupFallback.if).toContain("steps.start.outputs.dispatched != 'true'"); - expect(startupFallback.if).toContain("steps.start.outputs.finalized != 'true'"); - expect(startupFallback.run).toContain("post-merge-risk-gate.mts --mode abandon"); - expect(wait.run).toContain("timeout --signal=TERM --kill-after=30s 105m"); - expect(wait.run).toContain('gh run view "$RUN_ID" --repo "$GITHUB_REPOSITORY"'); - expect(wait.run).toContain("--json status,conclusion"); - expect(wait.run).toContain('if [[ "$state" != "$last_state" ]]'); - expect(wait.run).toContain('case "$state" in'); - expect(wait.run).toContain("completed:success"); - expect(wait.run).toContain("completed:failure"); - expect(wait.run).toContain("sleep 10"); - expect(wait.run).toContain('if [ "$wait_status" -eq 124 ]'); - expect(wait.run).toContain('exit "$wait_status"'); - expect(wait.run).not.toContain("gh run watch"); - expect(wait.run).not.toContain("--json jobs"); - expect(wait.run).not.toContain("2>/dev/null"); - expect(wait["continue-on-error"]).toBe(true); - expect(wait.env?.RUN_ID).toBe("${{ steps.start.outputs.run_id }}"); - expect(download.run).toContain('--dir "${{ steps.workspace.outputs.work_dir }}/evidence"'); - expect(download["continue-on-error"]).toBe(true); - expect(download.env?.RUN_ID).toBe("${{ steps.start.outputs.run_id }}"); - expect(finish.id).toBe("finish"); - expect(finish.if).toContain("always()"); - expect(finish["continue-on-error"]).not.toBe(true); - expect(finish.run).toContain("post-merge-risk-gate.mts --mode finish"); - expect(finish.run).toContain('--work-dir "${{ steps.workspace.outputs.work_dir }}"'); - expect(finish.run).toContain('--state-hash "${{ steps.start.outputs.state_hash }}"'); - expect(finish.run).toContain('--check-id "${{ steps.start.outputs.check_id }}"'); - expect(finish.run).toContain('--run-id "${{ steps.start.outputs.run_id }}"'); - expect(completionFallback.if).toContain("always()"); - expect(completionFallback.if).toContain("steps.finish.outcome == 'failure'"); - expect(completionFallback.if).toContain("steps.finish.outputs.finalized != 'true'"); - expect(completionFallback.run).toContain("post-merge-risk-gate.mts --mode abandon"); - expect(job.steps?.some((candidate) => candidate.name?.startsWith("Propagate "))).toBe(false); - expect(summary.if).toContain("always()"); - expect(summary.run).toContain("GITHUB_STEP_SUMMARY"); - expect(summary.run).toContain("Controller run"); - expect(summary.run).toContain("Selected jobs"); - expect(summary.run).toContain("Correlated E2E"); - expect(summary.run).toContain("Child conclusion"); - expect(summary.run).toContain("Failure phase"); - expect(cleanup.if).toContain("always() && steps.workspace.outputs.work_dir != ''"); - expect(cleanup.run).toContain('rm -rf -- "${{ steps.workspace.outputs.work_dir }}"'); - expect(collectStrings(workflow).some((value) => value.includes("/tmp/"))).toBe(false); - }); - - it("logs each child-run state once and exits after success", () => { - const result = runWaitStep("success"); - - expect(result.status).toBe(0); - expect(result.stderr).toBe(""); - expect(result.stdout.trim().split(/\r?\n/u)).toEqual([ - expect.stringContaining("status=in_progress"), - expect.stringContaining("status=completed conclusion=success"), - ]); - expect(result.stdout).not.toContain("JOBS"); - }); - - it("surfaces a terminal child-run failure", () => { - const result = runWaitStep("failure"); - - expect(result.status).toBe(1); - expect(result.stdout.match(/status=in_progress/gu)).toHaveLength(1); - expect(result.stderr).toContain("::error title=Correlated E2E run did not succeed::"); - expect(result.stderr).toContain("completed with conclusion failure"); - }); - - it("preserves GitHub CLI errors when status queries fail", () => { - const result = runWaitStep("query-failure"); - - expect(result.status).toBe(1); - expect(result.stderr).toContain("simulated GitHub query failure"); - expect(result.stderr).toContain("::error title=Correlated E2E status query failed::"); - }); - - it("labels only the bounded wait exit as a timeout", () => { - const result = runWaitStep("timeout"); - - expect(result.status).toBe(124); - expect(result.stderr).toContain("::error title=Correlated E2E wait timed out::"); - expect(result.stderr).toContain("did not complete within 105 minutes"); - }); - - it("rejects an invalid child-run ID before querying GitHub", () => { - const result = runWaitStep("success", { runId: "invalid" }); - - expect(result.status).toBe(1); - expect(result.ghCallCount).toBe(0); - expect(result.stderr).toContain("::error title=Invalid correlated E2E run ID::"); - }); - - it("fails closed for an unsupported child-run state", () => { - const result = runWaitStep("unsupported"); - - expect(result.status).toBe(1); - expect(result.ghCallCount).toBe(1); - expect(result.stderr).toContain("::error title=Unexpected correlated E2E state::"); - }); - - it("binds every E2E checkout and test signal to the merged commit", () => { - const workflow = readYaml< - Workflow & { - env?: Record; - "run-name"?: string; - concurrency?: { group: string; "cancel-in-progress": boolean }; - } - >(E2E_PATH); - const allSteps = Object.values(workflow.jobs).flatMap((job) => job.steps ?? []); - const checkouts = allSteps.filter((candidate) => - candidate.uses?.startsWith("actions/checkout@"), - ); - const testCommands = allSteps - .map((candidate) => candidate.run ?? "") - .filter((run) => run.includes("npx vitest run --project e2e-live")); - - expect(workflow["run-name"]).toContain("inputs.risk_correlation"); - expect(workflow.concurrency?.group).not.toContain("inputs.risk_correlation"); - expect(workflow.concurrency?.group).toContain("inputs.risk_shadow && github.run_id"); - expect(workflow.env).toMatchObject({ - NEMOCLAW_E2E_EXPECTED_SHA: "${{ inputs.checkout_sha }}", - NEMOCLAW_E2E_RISK_PLAN_HASH: "${{ inputs.risk_plan_hash }}", - NEMOCLAW_E2E_RISK_CORRELATION: "${{ inputs.risk_correlation }}", - NEMOCLAW_E2E_RISK_SHARD: "default", - }); - expect(checkouts.length).toBeGreaterThan(50); - expect( - checkouts.every( - (checkout) => checkout.with?.ref === "${{ inputs.checkout_sha || github.sha }}", - ), - ).toBe(true); - expect(testCommands.length).toBeGreaterThan(50); - expect( - testCommands.every((run) => run.includes("--reporter=test/e2e/risk-signal-reporter.ts")), - ).toBe(true); - expect(workflow.jobs["cloud-onboard"].env?.NEMOCLAW_PUBLIC_INSTALL_REF).toBe( - "${{ inputs.checkout_sha || github.sha }}", - ); - }); - - it("keeps every deterministic risk job signal-bearing and artifact-backed", () => { - const workflow = readYaml(E2E_PATH); - const requiredJobs = [...new Set(RISK_RULES.flatMap((rule) => rule.requiredJobs))]; - - for (const jobId of requiredJobs) { - const job = workflow.jobs[jobId]; - expect(job, `missing risk-plan job ${jobId}`).toBeDefined(); - expect( - Array.isArray(job.needs) ? job.needs : [job.needs], - `${jobId} must wait for exact-commit validation`, - ).toContain("generate-matrix"); - expect(job.env?.E2E_TARGET_ID, `${jobId} must identify its risk signal`).toBe(jobId); - const liveRuns = (job.steps ?? []) - .map((candidate) => candidate.run ?? "") - .filter((run) => run.includes("--project e2e-live")); - expect(liveRuns.length, `${jobId} must execute a live Vitest target`).toBeGreaterThan(0); - expect( - liveRuns.every((run) => run.includes("--reporter=test/e2e/risk-signal-reporter.ts")), - `${jobId} must write risk evidence for every live Vitest invocation`, - ).toBe(true); - const upload = (job.steps ?? []).find((candidate) => - candidate.uses?.includes("/.github/actions/upload-e2e-artifacts@"), - ); - expect(upload, `${jobId} must upload its risk signal`).toBeDefined(); - expect(upload?.if).toBe("always()"); - } - - expect(workflow.jobs["security-posture"].env?.NEMOCLAW_E2E_RISK_SHARD).toBe( - "${{ matrix.agent }}", - ); - expect(workflow.jobs["channels-stop-start"].env?.NEMOCLAW_E2E_RISK_SHARD).toBe( - "${{ matrix.agent }}", - ); - }); - - it("validates shadow inputs before preparing or executing the selected workspace", () => { - const workflow = readYaml(E2E_PATH); - const steps = workflow.jobs["generate-matrix"].steps ?? []; - const validateIndex = steps.findIndex( - (candidate) => candidate.name === "Validate exact-commit dispatch", - ); - const prepareIndex = steps.findIndex((candidate) => candidate.name === "Prepare E2E workspace"); - const validate = steps[validateIndex]; - - expect(validateIndex).toBeGreaterThan(0); - expect(validateIndex).toBeLessThan(prepareIndex); - expect(validate?.if).toContain("inputs.checkout_sha != ''"); - expect(validate?.env?.WORKFLOW_SHA).toBe("${{ github.sha }}"); - expect(validate?.run).toContain('[[ "$RISK_SHADOW" == "true" ]]'); - expect(validate?.run).toContain("exact-commit inputs require risk_shadow=true"); - expect(validate?.run).toContain("checkout_sha must be a lowercase 40-character SHA"); - expect(validate?.run).toContain('[[ "$CHECKOUT_SHA" == "$WORKFLOW_SHA" ]]'); - expect(validate?.run).toContain("checkout_sha must equal the current main workflow commit"); - expect(validate?.run).toContain('"$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA"'); - expect(validate?.run).toContain('git merge-base --is-ancestor "$CHECKOUT_SHA" origin/main'); - expect(validate?.run).toContain("forbid targets/fan-out"); - expect(steps[0]?.with?.["fetch-depth"]).toBe(0); - expect(workflow.jobs["report-to-pr"].if).toContain("!inputs.risk_shadow"); - expect(workflow.jobs.scorecard.if).toContain("!inputs.risk_shadow"); - }); -}); diff --git a/test/post-merge-e2e-risk-gate.test.ts b/test/post-merge-e2e-risk-gate.test.ts deleted file mode 100644 index 900d42cc429..00000000000 --- a/test/post-merge-e2e-risk-gate.test.ts +++ /dev/null @@ -1,593 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { execFileSync, spawnSync } from "node:child_process"; -import { createHash } from "node:crypto"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { afterEach, describe, expect, it, vi } from "vitest"; -import { buildRiskPlan, RISK_RULES } from "../tools/advisors/risk-plan.mts"; -import { - assertCorrelatedWorkflowRun, - assertTrustedMainPush, - changedFilesBetween, - classifyRiskEvidence, - dispatchRiskWorkflow, - expectedRiskSignalShards, - findSignalFiles, - finishRiskGate, - parseControllerCommand, - type RiskGateState, - validateRiskGateState, - validateRiskPlan, - validateSignal, - validateWorkflowDispatchDetails, -} from "../tools/e2e-advisor/post-merge-risk-gate.mts"; -import type { E2eRiskSignal } from "../tools/e2e-advisor/risk-signal.ts"; - -const HEAD_SHA = "a".repeat(40); -const ALLOWED = new Set(["onboard-repair", "onboard-resume"]); - -afterEach(() => { - vi.restoreAllMocks(); - vi.unstubAllEnvs(); -}); - -function state(): RiskGateState { - return { - version: 1, - commitSha: HEAD_SHA, - planHash: buildRiskPlan({ headSha: HEAD_SHA, changedFiles: ["src/lib/onboard.ts"] }).planHash, - correlationId: "12345678-1234-4123-8123-123456789abc", - expectedJobs: ["onboard-repair", "onboard-resume"], - expectedShards: { - "onboard-repair": ["default"], - "onboard-resume": ["default"], - }, - requiresManualExpansion: false, - }; -} - -function sha256(value: string): string { - return createHash("sha256").update(value).digest("hex"); -} - -function signal(jobId: string, overrides: Partial = {}): E2eRiskSignal { - const gate = state(); - return { - version: 1, - jobId, - shardId: "default", - expectedSha: gate.commitSha, - testedSha: gate.commitSha, - planHash: gate.planHash, - correlationId: gate.correlationId, - passed: 1, - failed: 0, - skipped: 0, - pending: 0, - unhandledErrors: 0, - runReason: "passed", - ...overrides, - }; -} - -describe("post-merge E2E risk gate", () => { - it("accepts only the exact trusted main-push context", () => { - const trusted = { - eventName: "push", - ref: "refs/heads/main", - sha: HEAD_SHA, - commitSha: HEAD_SHA, - }; - - expect(() => assertTrustedMainPush(trusted)).not.toThrow(); - expect(() => assertTrustedMainPush({ ...trusted, eventName: "pull_request" })).toThrow( - /exact trusted main push/u, - ); - expect(() => assertTrustedMainPush({ ...trusted, ref: "refs/heads/feature" })).toThrow( - /exact trusted main push/u, - ); - expect(() => assertTrustedMainPush({ ...trusted, sha: "b".repeat(40) })).toThrow( - /exact trusted main push/u, - ); - }); - - it("requires a private controller workspace and parses the abandon check id", () => { - const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-risk-controller-")); - try { - expect( - parseControllerCommand([ - "--mode", - "start", - "--base", - "b".repeat(40), - "--commit", - HEAD_SHA, - "--work-dir", - workDir, - ]), - ).toMatchObject({ - mode: "start", - planPath: path.join(workDir, "post-merge-risk-plan.json"), - statePath: path.join(workDir, "e2e-risk-gate-state.json"), - evidencePath: path.join(workDir, "evidence"), - }); - expect(parseControllerCommand(["--mode", "abandon", "--check-id", "17"])).toEqual({ - mode: "abandon", - checkRunId: 17, - }); - expect( - parseControllerCommand([ - "--mode", - "finish", - "--work-dir", - workDir, - "--state-hash", - "b".repeat(64), - "--check-id", - "17", - "--run-id", - "23", - ]), - ).toMatchObject({ - mode: "finish", - checkRunId: 17, - childRunId: 23, - stateHash: "b".repeat(64), - }); - expect(() => - parseControllerCommand(["--mode", "abandon", "--check-id", "9007199254740992"]), - ).toThrow(/safe integer range/u); - expect(() => - parseControllerCommand([ - "--mode", - "finish", - "--work-dir", - workDir, - "--state-hash", - "unsafe", - "--check-id", - "17", - "--run-id", - "23", - ]), - ).toThrow(/state-hash must be a lowercase SHA-256 hash/u); - expect(() => parseControllerCommand(["--mode", "finish"])).toThrow(/--work-dir/u); - - fs.chmodSync(workDir, 0o755); - expect(() => parseControllerCommand(["--mode", "finish", "--work-dir", workDir])).toThrow( - /owned private absolute directory/u, - ); - } finally { - fs.chmodSync(workDir, 0o700); - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("emits a single-line escaped Actions annotation for controller errors", () => { - const missingWorkDir = path.join(os.tmpdir(), "nemoclaw-risk-missing%\nworkspace"); - const result = spawnSync( - process.execPath, - [ - "--experimental-strip-types", - "tools/e2e-advisor/post-merge-risk-gate.mts", - "--mode", - "finish", - "--work-dir", - missingWorkDir, - ], - { - cwd: process.cwd(), - encoding: "utf8", - env: { ...process.env, GITHUB_ACTIONS: "true" }, - }, - ); - - expect(result.status).toBe(1); - const annotations = result.stderr - .split(/\r?\n/gu) - .filter((line) => - line.startsWith("::error title=Post-merge E2E risk gate controller failed::"), - ); - expect(annotations).toHaveLength(1); - expect(annotations[0]).toContain("nemoclaw-risk-missing%25 workspace"); - expect(annotations[0]).not.toMatch(/[\r\t]/u); - }); - - it("derives changed files from an exact checked-out commit range", () => { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-risk-git-")); - const git = (...args: string[]) => - execFileSync("git", args, { cwd: directory, encoding: "utf8" }).trim(); - try { - git("init", "--quiet"); - git("config", "user.name", "Risk Gate Test"); - git("config", "user.email", "risk-gate@example.invalid"); - fs.writeFileSync(path.join(directory, "README.md"), "base\n"); - fs.mkdirSync(path.join(directory, "src", "lib", "credentials"), { recursive: true }); - fs.writeFileSync(path.join(directory, "src", "lib", "credentials", "token.ts"), "base\n"); - git("add", "README.md", "src/lib/credentials/token.ts"); - git("-c", "commit.gpgsign=false", "commit", "--quiet", "-m", "base"); - const base = git("rev-parse", "HEAD"); - fs.mkdirSync(path.join(directory, "docs")); - git("mv", "src/lib/credentials/token.ts", "docs/token.ts"); - fs.writeFileSync(path.join(directory, "src", "feature.ts"), "export {};\n"); - git("add", "docs/token.ts", "src/feature.ts"); - git("-c", "commit.gpgsign=false", "commit", "--quiet", "-m", "feature"); - const head = git("rev-parse", "HEAD"); - - expect(changedFilesBetween(base, head, directory)).toEqual([ - "docs/token.ts", - "src/feature.ts", - "src/lib/credentials/token.ts", - ]); - expect(() => changedFilesBetween(head, base, directory)).toThrow(/does not match/u); - } finally { - fs.rmSync(directory, { recursive: true, force: true }); - } - }); - - it("accepts only the deterministic plan for the tested commit", () => { - const plan = buildRiskPlan({ headSha: HEAD_SHA, changedFiles: ["src/lib/onboard.ts"] }); - - expect(validateRiskPlan(plan, ALLOWED)).toEqual(plan); - expect(() => validateRiskPlan({ ...plan, planHash: "b".repeat(64) }, ALLOWED)).toThrow( - /deterministic hash/u, - ); - expect(() => validateRiskPlan(plan, new Set())).toThrow(/unknown E2E job/u); - }); - - it("accepts only bounded gate state for the exact commit and evidence policy", () => { - const gate = state(); - - expect(validateRiskGateState(gate)).toEqual(gate); - expect(() => validateRiskGateState({ ...gate, commitSha: "unsafe" })).toThrow(/commit SHA/u); - expect(() => validateRiskGateState({ ...gate, expectedJobs: ["../unsafe"] })).toThrow( - /expected jobs/u, - ); - expect(() => validateRiskGateState({ ...gate, expectedShards: {} })).toThrow(/shard jobs/u); - }); - - it("uses the workflow run identity returned by the exact dispatch request", async () => { - const gate = state(); - const fetchMock = vi.spyOn(globalThis, "fetch").mockResolvedValue({ - ok: true, - text: async () => - JSON.stringify({ - workflow_run_id: 23, - run_url: "https://api.github.com/repos/NVIDIA/NemoClaw/actions/runs/23", - html_url: "https://github.com/NVIDIA/NemoClaw/actions/runs/23", - }), - } as Response); - - const runId = await dispatchRiskWorkflow({ - repository: "NVIDIA/NemoClaw", - token: "token", - jobs: ["onboard-repair"], - commitSha: gate.commitSha, - planHash: gate.planHash, - correlationId: gate.correlationId, - }); - - expect(runId).toBe(23); - expect(fetchMock).toHaveBeenCalledOnce(); - expect(String(fetchMock.mock.calls[0]?.[0])).toBe( - "https://api.github.com/repos/NVIDIA/NemoClaw/actions/workflows/e2e.yaml/dispatches", - ); - expect(fetchMock.mock.calls[0]?.[1]?.method).toBe("POST"); - expect(JSON.parse(String(fetchMock.mock.calls[0]?.[1]?.body))).toMatchObject({ - ref: "main", - return_run_details: true, - inputs: { - jobs: "onboard-repair", - checkout_sha: gate.commitSha, - risk_plan_hash: gate.planHash, - risk_correlation: gate.correlationId, - risk_shadow: "true", - }, - }); - expect(() => - validateWorkflowDispatchDetails( - { - workflow_run_id: 23, - run_url: "https://api.github.com/repos/NVIDIA/NemoClaw/actions/runs/24", - html_url: "https://github.com/NVIDIA/NemoClaw/actions/runs/23", - }, - "NVIDIA/NemoClaw", - ), - ).toThrow(/mismatched workflow dispatch URLs/u); - }); - - it("reports the exact mismatched correlated workflow identity fields", () => { - const gate = state(); - const childRunId = 23; - const child = { - id: childRunId, - name: `E2E risk ${gate.correlationId}`, - path: ".github/workflows/e2e.yaml", - workflow_id: 304268429, - event: "workflow_dispatch", - head_sha: gate.commitSha, - status: "completed", - conclusion: "success", - created_at: "2026-07-08T00:00:00.000Z", - display_title: `E2E risk ${gate.correlationId}`, - html_url: `https://github.com/NVIDIA/NemoClaw/actions/runs/${childRunId}`, - }; - const identity = { - childRunId, - correlationId: gate.correlationId, - repository: "NVIDIA/NemoClaw", - }; - const cases = [ - { override: { id: 24 }, expected: "id expected=23 actual=24" }, - { - override: { path: ".github/workflows/other.yaml" }, - expected: - 'path expected=".github/workflows/e2e.yaml" actual=".github/workflows/other.yaml"', - }, - { - override: { display_title: "E2E risk wrong" }, - expected: `display_title expected="E2E risk ${gate.correlationId}" actual="E2E risk wrong"`, - }, - { - override: { workflow_id: 0 }, - expected: 'workflow_id expected="positive safe integer" actual=0', - }, - ]; - - for (const { override, expected } of cases) { - expect(() => assertCorrelatedWorkflowRun({ ...child, ...override }, identity)).toThrow( - expected, - ); - } - }); - - it("finish reports the directly dispatched child failure when main advances", async () => { - const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-risk-finish-")); - const statePath = path.join(workDir, "e2e-risk-gate-state.json"); - const gate = state(); - const serializedState = `${JSON.stringify(gate)}\n`; - const childRunId = 23; - vi.stubEnv("GITHUB_TOKEN", "token"); - vi.stubEnv("GITHUB_REPOSITORY", "NVIDIA/NemoClaw"); - fs.writeFileSync(statePath, serializedState, { mode: 0o600 }); - const fetchMock = vi - .spyOn(globalThis, "fetch") - .mockResolvedValueOnce({ - ok: true, - text: async () => - JSON.stringify({ - id: childRunId, - name: `E2E risk ${gate.correlationId}`, - event: "workflow_dispatch", - head_sha: "b".repeat(40), - status: "completed", - conclusion: "failure", - created_at: "2026-07-08T00:00:00.000Z", - display_title: `E2E risk ${gate.correlationId}`, - path: ".github/workflows/e2e.yaml", - workflow_id: 304268429, - html_url: `https://github.com/NVIDIA/NemoClaw/actions/runs/${childRunId}`, - }), - } as Response) - .mockResolvedValueOnce({ ok: true, text: async () => "{}" } as Response); - - try { - await finishRiskGate({ - statePath, - stateHash: sha256(serializedState), - evidencePath: path.join(workDir, "evidence"), - checkRunId: 17, - childRunId, - }); - - expect(fetchMock).toHaveBeenCalledTimes(2); - expect(String(fetchMock.mock.calls[1]?.[0])).toContain("check-runs/17"); - expect(JSON.parse(String(fetchMock.mock.calls[1]?.[1]?.body))).toMatchObject({ - status: "completed", - conclusion: "failure", - output: { title: "Selected E2E workflow failed" }, - }); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("rejects changed controller state before classifying downloaded evidence", async () => { - const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-risk-state-")); - const statePath = path.join(workDir, "e2e-risk-gate-state.json"); - const outputPath = path.join(workDir, "github-output"); - const originalState = `${JSON.stringify(state())}\n`; - const changedState = `${JSON.stringify({ ...state(), requiresManualExpansion: true })}\n`; - vi.stubEnv("GITHUB_TOKEN", "token"); - vi.stubEnv("GITHUB_REPOSITORY", "NVIDIA/NemoClaw"); - vi.stubEnv("GITHUB_OUTPUT", outputPath); - fs.writeFileSync(statePath, changedState, { mode: 0o600 }); - fs.writeFileSync(outputPath, "", { mode: 0o600 }); - const fetchMock = vi - .spyOn(globalThis, "fetch") - .mockResolvedValue({ ok: true, text: async () => "{}" } as Response); - - try { - await expect( - finishRiskGate({ - statePath, - stateHash: sha256(originalState), - evidencePath: path.join(workDir, "evidence"), - checkRunId: 17, - childRunId: 23, - }), - ).rejects.toThrow(/controller state changed after E2E dispatch/u); - - expect(fetchMock).toHaveBeenCalledTimes(1); - expect(String(fetchMock.mock.calls[0]?.[0])).toContain("check-runs/17"); - expect(JSON.parse(String(fetchMock.mock.calls[0]?.[1]?.body))).toMatchObject({ - status: "completed", - conclusion: "neutral", - output: { title: "Risk-selected E2E evidence could not be verified" }, - }); - expect(JSON.parse(String(fetchMock.mock.calls[0]?.[1]?.body)).output.summary).toContain( - "controller state changed after E2E dispatch", - ); - expect(fs.readFileSync(outputPath, "utf8")).toContain("finalized=true\n"); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("bounds downloaded risk-evidence traversal by entries, depth, and signals", () => { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-risk-evidence-")); - const rootLink = `${directory}-link`; - const nested = path.join(directory, "artifact", "live", "job"); - try { - fs.mkdirSync(nested, { recursive: true }); - const signalFile = path.join(nested, "risk-signal.json"); - fs.writeFileSync(signalFile, "{}\n"); - - expect(findSignalFiles(directory)).toEqual([signalFile]); - expect(() => - findSignalFiles(directory, { maxDepth: 2, maxEntries: 10, maxSignalFiles: 2 }), - ).toThrow(/depth limit/u); - expect(() => - findSignalFiles(directory, { maxDepth: 8, maxEntries: 2, maxSignalFiles: 2 }), - ).toThrow(/entry limit/u); - const second = path.join(directory, "artifact-2"); - fs.mkdirSync(second); - fs.writeFileSync(path.join(second, "risk-signal.json"), "{}\n"); - expect(() => - findSignalFiles(directory, { maxDepth: 8, maxEntries: 10, maxSignalFiles: 1 }), - ).toThrow(/signal-file limit/u); - expect(() => - findSignalFiles(directory, { maxDepth: 8, maxEntries: 10, maxSignalFiles: 0 }), - ).toThrow(/limits are invalid/u); - fs.symlinkSync(directory, rootLink, "dir"); - expect(() => findSignalFiles(rootLink)).toThrow(/root must be a directory, not a symlink/u); - } finally { - fs.rmSync(rootLink, { force: true }); - fs.rmSync(directory, { recursive: true, force: true }); - } - }); - - it("accepts only signals bound to the expected job, SHA, plan, and correlation", () => { - const gate = state(); - - expect(validateSignal(signal("onboard-resume"), gate).jobId).toBe("onboard-resume"); - expect(() => - validateSignal(signal("onboard-resume", { expectedSha: "b".repeat(40) }), gate), - ).toThrow(/SHA mismatch/u); - expect(() => validateSignal(signal("other"), gate)).toThrow(/unexpected/u); - expect(() => validateSignal(signal("onboard-resume", { shardId: "unexpected" }), gate)).toThrow( - /shard/u, - ); - }); - - it("derives expected evidence shards from the trusted E2E workflow", () => { - const jobIds = [...new Set(RISK_RULES.flatMap((rule) => rule.requiredJobs))]; - const shards = expectedRiskSignalShards(jobIds); - - expect(Object.keys(shards).sort()).toEqual(jobIds.sort()); - expect(shards["onboard-resume"]).toEqual(["default"]); - expect(shards["security-posture"]).toEqual(["openclaw", "hermes"]); - expect(shards["channels-stop-start"]).toEqual(["openclaw", "hermes"]); - }); - - it("reports success only for complete unskipped evidence", () => { - const verdict = classifyRiskEvidence({ - workflowConclusion: "success", - expectedJobs: ["onboard-repair", "onboard-resume"], - expectedShards: state().expectedShards, - signals: [signal("onboard-repair"), signal("onboard-resume")], - requiresManualExpansion: false, - }); - - expect(verdict.conclusion).toBe("success"); - expect(verdict.summary).not.toContain("onboard-repair"); - }); - - it.each([ - { - label: "missing signal", - signals: [signal("onboard-repair")], - manual: false, - }, - { - label: "skipped test", - signals: [signal("onboard-repair"), signal("onboard-resume", { skipped: 1 })], - manual: false, - }, - { - label: "manual expansion", - signals: [signal("onboard-repair"), signal("onboard-resume")], - manual: true, - }, - { - label: "duplicate signal", - signals: [signal("onboard-repair"), signal("onboard-repair"), signal("onboard-resume")], - manual: false, - }, - ])("reports neutral for $label", ({ signals, manual }) => { - const verdict = classifyRiskEvidence({ - workflowConclusion: "success", - expectedJobs: ["onboard-repair", "onboard-resume"], - expectedShards: state().expectedShards, - signals, - requiresManualExpansion: manual, - }); - - expect(verdict.conclusion).toBe("neutral"); - expect(verdict.summary).not.toContain("onboard-repair"); - }); - - it("reports a product workflow failure as failure", () => { - const verdict = classifyRiskEvidence({ - workflowConclusion: "failure", - expectedJobs: ["onboard-repair"], - expectedShards: { "onboard-repair": ["default"] }, - signals: [], - requiresManualExpansion: false, - }); - - expect(verdict.conclusion).toBe("failure"); - expect(verdict.summary).not.toContain("onboard-repair"); - }); - - it("reports failed test evidence as failure even when the workflow is green", () => { - const verdict = classifyRiskEvidence({ - workflowConclusion: "success", - expectedJobs: ["onboard-repair"], - expectedShards: { "onboard-repair": ["default"] }, - signals: [signal("onboard-repair", { failed: 1, runReason: "failed" })], - requiresManualExpansion: false, - }); - - expect(verdict.conclusion).toBe("failure"); - expect(verdict.summary).not.toContain("onboard-repair"); - }); - - it("requires every expected matrix shard to pass", () => { - const complete = classifyRiskEvidence({ - workflowConclusion: "success", - expectedJobs: ["security-posture"], - expectedShards: { "security-posture": ["openclaw", "hermes"] }, - signals: [ - signal("security-posture", { shardId: "openclaw" }), - signal("security-posture", { shardId: "hermes" }), - ], - requiresManualExpansion: false, - }); - const missingShard = classifyRiskEvidence({ - workflowConclusion: "success", - expectedJobs: ["security-posture"], - expectedShards: { "security-posture": ["openclaw", "hermes"] }, - signals: [signal("security-posture", { shardId: "openclaw" })], - requiresManualExpansion: false, - }); - - expect(complete.conclusion).toBe("success"); - expect(missingShard.conclusion).toBe("neutral"); - expect(missingShard.summary).not.toContain("security-posture"); - }); -}); diff --git a/test/pr-risk-plan.test.ts b/test/pr-risk-plan.test.ts index d238e15560c..cd47c8573d3 100644 --- a/test/pr-risk-plan.test.ts +++ b/test/pr-risk-plan.test.ts @@ -18,6 +18,7 @@ describe("deterministic PR risk plan", () => { const second = plan("src/lib/onboard.ts", "src/lib/state/registry.ts"); expect(first).toEqual(second); + expect(first.version).toBe(2); expect(first.headSha).toBe(HEAD_SHA); expect(first.planHash).toMatch(/^[a-f0-9]{64}$/u); expect(first.changedFiles).toEqual(["src/lib/onboard.ts", "src/lib/state/registry.ts"]); @@ -29,7 +30,6 @@ describe("deterministic PR risk plan", () => { expect(result.tier).toBe(0); expect(result.families).toEqual([]); expect(result.requiredJobs).toEqual([]); - expect(result.requiresManualExpansion).toBe(false); }); it("keeps the canonical cloud-onboard live test in the platform floor (#6446)", () => { @@ -145,19 +145,32 @@ describe("deterministic PR risk plan", () => { expect(riskPlanRequiredJobIds(result)).toEqual(expect.arrayContaining(jobs)); }); - it("caps automatic execution without dropping required evidence", () => { + it("keeps every required job selected for broad runtime changes (#6446)", () => { const result = plan( "src/lib/onboard.ts", + "src/lib/actions/upgrade-sandboxes.ts", + "src/lib/actions/sandbox/agents/apply.ts", "src/lib/messaging/applier/agent-config.ts", "src/lib/inference/health.ts", + "install.sh", + "src/lib/credentials/provider-list.ts", ); - expect(result.requiredJobs.length).toBeGreaterThan(result.maxAutomaticJobs); - expect(result.automaticJobs).toHaveLength(result.maxAutomaticJobs); - expect(result.requiresManualExpansion).toBe(true); - expect(result.requiredJobs.map((job) => job.id)).toEqual( - expect.arrayContaining(result.automaticJobs), - ); + expect(riskPlanRequiredJobIds(result)).toEqual([ + "cloud-onboard", + "credential-sanitization", + "security-posture", + "channels-add-remove", + "channels-stop-start", + "full-e2e", + "hermes-e2e", + "inference-routing", + "network-policy", + "onboard-repair", + "onboard-resume", + "state-backup-restore", + "upgrade-stale-sandbox", + ]); }); it("raises PR review test depth for a matched runtime risk", () => { diff --git a/test/required-live-workflow.test.ts b/test/required-live-workflow.test.ts new file mode 100644 index 00000000000..96feaefd76f --- /dev/null +++ b/test/required-live-workflow.test.ts @@ -0,0 +1,396 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; +import { + readYaml, + type Workflow, + type WorkflowJob, + type WorkflowStep, +} from "./helpers/e2e-workflow-contract.ts"; + +const REQUIRED_LIVE_PATH = ".github/workflows/required-live-e2e.yaml"; +const E2E_PATH = ".github/workflows/e2e.yaml"; + +type CoordinatorJob = WorkflowJob & { + concurrency?: { group: string; "cancel-in-progress": boolean }; +}; + +type TriggeredWorkflow = Omit & { + name: string; + on: { + workflow_run: { workflows: string[]; types: string[] }; + pull_request_target: { types: string[] }; + }; + permissions: Record; + jobs: Record; +}; + +type DispatchWorkflow = Workflow & { + "run-name": string; + on: { + workflow_dispatch: { + inputs: Record; + }; + }; +}; + +function step(job: WorkflowJob, name: string): WorkflowStep { + const match = job.steps?.find((candidate) => candidate.name === name); + expect(match, `missing workflow step ${name}`).toBeDefined(); + return match!; +} + +function collectStrings(value: unknown): string[] { + return typeof value === "string" + ? [value] + : Array.isArray(value) + ? value.flatMap(collectStrings) + : value && typeof value === "object" + ? Object.values(value).flatMap(collectStrings) + : []; +} + +function runWaitStep( + scenario: "success" | "failure" | "query-failure" | "timeout" | "unsupported", + options: { runId?: string } = {}, +) { + const workflow = readYaml(REQUIRED_LIVE_PATH); + const wait = step(workflow.jobs.coordinate, "Wait for required live run"); + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-required-live-wait-")); + const binDir = path.join(tempDir, "bin"); + const callCountPath = path.join(tempDir, "gh-call-count"); + fs.mkdirSync(binDir); + fs.writeFileSync(callCountPath, "0\n"); + fs.writeFileSync( + path.join(binDir, "gh"), + `#!/usr/bin/env bash +set -euo pipefail +count="$(cat "$FAKE_GH_CALL_COUNT")" +count=$((count + 1)) +printf '%s\n' "$count" > "$FAKE_GH_CALL_COUNT" +case "$FAKE_GH_SCENARIO:$count" in + success:1 | success:2 | failure:1) printf 'in_progress:none\n' ;; + success:*) printf 'completed:success\n' ;; + failure:*) printf 'completed:failure\n' ;; + query-failure:*) printf 'simulated GitHub query failure\n' >&2; exit 1 ;; + unsupported:*) printf 'completed:unknown\n' ;; + *) exit 2 ;; +esac +`, + { mode: 0o755 }, + ); + fs.writeFileSync(path.join(binDir, "sleep"), "#!/usr/bin/env bash\nexit 0\n", { mode: 0o755 }); + fs.writeFileSync( + path.join(binDir, "timeout"), + `#!/usr/bin/env bash +set -euo pipefail +if [ "$FAKE_GH_SCENARIO" = "timeout" ]; then + exit 124 +fi +shift 3 +exec "$@" +`, + { mode: 0o755 }, + ); + + try { + const result = spawnSync("bash", ["-e", "-o", "pipefail", "-c", wait.run!], { + encoding: "utf8", + env: { + ...process.env, + FAKE_GH_CALL_COUNT: callCountPath, + FAKE_GH_SCENARIO: scenario, + GITHUB_REPOSITORY: "NVIDIA/NemoClaw", + PATH: `${binDir}:${process.env.PATH ?? ""}`, + RUN_ID: options.runId ?? "29110351531", + }, + timeout: 5_000, + }); + return { + ...result, + ghCallCount: Number(fs.readFileSync(callCountPath, "utf8").trim()), + }; + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +} + +function runChildValidation(currentPullSha: string) { + const workflow = readYaml(E2E_PATH); + const validation = step(workflow.jobs["generate-matrix"], "Validate required-live dispatch"); + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-required-live-child-")); + const binDir = path.join(tempDir, "bin"); + fs.mkdirSync(binDir); + fs.writeFileSync( + path.join(binDir, "git"), + "#!/usr/bin/env bash\nset -euo pipefail\nprintf '%s\\n' \"$FAKE_CHECKOUT_SHA\"\n", + { mode: 0o755 }, + ); + fs.writeFileSync( + path.join(binDir, "curl"), + "#!/usr/bin/env bash\nset -euo pipefail\nprintf '{}\\n'\n", + { mode: 0o755 }, + ); + fs.writeFileSync( + path.join(binDir, "jq"), + `#!/usr/bin/env bash +set -euo pipefail +case "\${2:-}" in + .state) printf 'open\\n' ;; + .head.repo.full_name*) printf 'NVIDIA/NemoClaw\\n' ;; + .head.sha) printf '%s\\n' "$FAKE_PR_SHA" ;; + *) exit 2 ;; +esac +`, + { mode: 0o755 }, + ); + + try { + return spawnSync("bash", ["-e", "-o", "pipefail", "-c", validation.run!], { + encoding: "utf8", + env: { + ...process.env, + CHECKOUT_SHA: "a".repeat(40), + CORRELATION_ID: "12345678-1234-4123-8123-123456789abc", + FAKE_CHECKOUT_SHA: "a".repeat(40), + FAKE_PR_SHA: currentPullSha, + GITHUB_REPOSITORY: "NVIDIA/NemoClaw", + GITHUB_TOKEN: "token", + JOBS: "onboard-repair", + PATH: `${binDir}:${process.env.PATH ?? ""}`, + PLAN_HASH: "b".repeat(64), + PR_NUMBER: "42", + TARGETS: "", + WORKFLOW_EVENT: "workflow_dispatch", + WORKFLOW_REF: "refs/heads/main", + }, + }); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +} + +describe("required live E2E workflow", () => { + it("runs only from trusted lifecycle events with least-privilege jobs", () => { + const workflow = readYaml(REQUIRED_LIVE_PATH); + const cancel = workflow.jobs["cancel-superseded"]; + const coordinate = workflow.jobs.coordinate; + + expect(workflow.name).toBe("E2E / Required Live"); + expect(workflow.on).toEqual({ + workflow_run: { + workflows: ["CI / Pull Request"], + types: ["completed"], + }, + pull_request_target: { + types: ["synchronize", "reopened", "closed"], + }, + }); + expect(workflow.permissions).toEqual({}); + expect(cancel.if).toContain("github.event_name == 'pull_request_target'"); + expect(cancel.permissions).toEqual({ actions: "write", contents: "read" }); + expect(coordinate.if).toContain("github.event_name == 'workflow_run'"); + expect(coordinate.if).toContain("github.event.workflow_run.event == 'pull_request'"); + expect(coordinate.permissions).toEqual({ + actions: "write", + checks: "write", + contents: "read", + "pull-requests": "read", + }); + expect(collectStrings(workflow).some((value) => value.includes("${{ secrets."))).toBe(false); + }); + + it("pins both controller checkouts and installs without lifecycle scripts or caches", () => { + const workflow = readYaml(REQUIRED_LIVE_PATH); + const allSteps = Object.values(workflow.jobs).flatMap((job) => job.steps ?? []); + const checkouts = allSteps.filter((candidate) => + candidate.uses?.startsWith("actions/checkout@"), + ); + const nodeSetups = allSteps.filter((candidate) => + candidate.uses?.startsWith("actions/setup-node@"), + ); + const installs = allSteps.filter( + (candidate) => candidate.name === "Install trusted controller dependencies", + ); + + expect(checkouts).toHaveLength(2); + expect( + checkouts.every( + (checkout) => + checkout.with?.ref === "${{ github.workflow_sha }}" && + checkout.with?.["persist-credentials"] === false, + ), + ).toBe(true); + expect(nodeSetups).toHaveLength(2); + expect(nodeSetups.every((setup) => setup.with?.["node-version"] === "22")).toBe(true); + expect(nodeSetups.every((setup) => !("cache" in (setup.with ?? {})))).toBe(true); + expect(installs).toHaveLength(2); + expect(installs.every((install) => install.run === "npm ci --ignore-scripts")).toBe(true); + expect( + allSteps.some((candidate) => candidate.uses?.startsWith("actions/download-artifact@")), + ).toBe(false); + }); + + it("cancels superseded pull request runs through the trusted controller", () => { + const workflow = readYaml(REQUIRED_LIVE_PATH); + const cancel = workflow.jobs["cancel-superseded"]; + const cancelStep = step(cancel, "Cancel superseded required live runs"); + + expect(cancelStep.run).toContain("tools/e2e/required-live.mts --mode cancel"); + expect(cancelStep.run).toContain('--pr "${{ github.event.pull_request.number }}"'); + expect(cancelStep.env?.GITHUB_TOKEN).toBe("${{ github.token }}"); + }); + + it("coordinates one check lifecycle around a bounded child run", () => { + const workflow = readYaml(REQUIRED_LIVE_PATH); + const job = workflow.jobs.coordinate; + const workspace = step(job, "Create private controller workspace"); + const start = step(job, "Start required live evaluation"); + const upload = step(job, "Upload required live plan"); + const wait = step(job, "Wait for required live run"); + const download = step(job, "Download required live evidence"); + const finish = step(job, "Finish required live evaluation"); + const fallback = step(job, "Close incomplete required live check"); + const cleanup = step(job, "Remove private controller workspace"); + + expect(job.concurrency).toEqual({ + group: + "required-live-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }}", + "cancel-in-progress": false, + }); + expect(job["timeout-minutes"]).toBe(180); + expect(workspace.run).toContain('mktemp -d "${RUNNER_TEMP}/nemoclaw-required-live.XXXXXX"'); + expect(workspace.run).toContain('chmod 700 "$work_dir"'); + expect(start.run).toContain("tools/e2e/required-live.mts --mode start"); + expect(start.run).toContain('--head "${{ github.event.workflow_run.head_sha }}"'); + expect(start.run).toContain( + '--head-repo "${{ github.event.workflow_run.head_repository.full_name }}"', + ); + expect(start.run).toContain('--head-branch "${{ github.event.workflow_run.head_branch }}"'); + expect(start.run).toContain('--workflow-sha "${{ github.workflow_sha }}"'); + expect(start.run).toContain('--ci-conclusion "${{ github.event.workflow_run.conclusion }}"'); + expect(start.run).toContain('--work-dir "${{ steps.workspace.outputs.work_dir }}"'); + expect(start.run).not.toContain("--mode initialize"); + expect(upload.if).toContain("steps.workspace.outputs.work_dir != ''"); + expect(upload.with?.path).toBe( + "${{ steps.workspace.outputs.work_dir }}/required-live-plan.json", + ); + expect(wait.run).toContain("timeout --signal=TERM --kill-after=30s 105m"); + expect(wait.run).toContain('gh run view "$RUN_ID" --repo "$GITHUB_REPOSITORY"'); + expect(wait.run).toContain("--json status,conclusion"); + expect(wait.run).toContain('if [[ "$state" != "$last_state" ]]'); + expect(wait.run).toContain("completed:success"); + expect(wait.run).toContain("completed:failure"); + expect(wait.run).toContain("sleep 10"); + expect(wait.run).toContain('if [ "$wait_status" -eq 124 ]'); + expect(wait.run).toContain('exit "$wait_status"'); + expect(wait.run).not.toContain("gh run watch"); + expect(wait.run).not.toContain("--json jobs"); + expect(wait.run).not.toContain("2>/dev/null"); + expect(wait["continue-on-error"]).toBe(true); + expect(download.if).toContain("always()"); + expect(download.run).toContain("timeout --signal=TERM --kill-after=30s 10m"); + expect(download.run).toContain('if [ "$download_status" -eq 124 ]'); + expect(download.run).toContain('--dir "${{ steps.workspace.outputs.work_dir }}/evidence"'); + expect(download["continue-on-error"]).toBe(true); + expect(finish.if).toContain("always()"); + expect(finish.run).toContain("tools/e2e/required-live.mts --mode finish"); + expect(finish.run).toContain('--state-hash "${{ steps.start.outputs.state_hash }}"'); + expect(finish.run).toContain('--check-id "${{ steps.start.outputs.check_id }}"'); + expect(finish.run).toContain('--run-id "${{ steps.start.outputs.run_id }}"'); + expect(fallback.if).toContain("always()"); + expect(fallback.if).toContain("steps.start.outputs.check_id != ''"); + expect(fallback.if).toContain("steps.start.outputs.finalized != 'true'"); + expect(fallback.if).toContain("steps.finish.outputs.finalized != 'true'"); + expect(fallback.run).toContain("tools/e2e/required-live.mts --mode abandon"); + expect(fallback.run).toContain('--run-id "${{ steps.start.outputs.run_id }}"'); + expect(cleanup.if).toContain("always() && steps.workspace.outputs.work_dir != ''"); + expect(cleanup.run).toContain('rm -rf -- "${{ steps.workspace.outputs.work_dir }}"'); + expect(collectStrings(workflow).some((value) => value.includes("/tmp/"))).toBe(false); + }); + + it("uses one child dispatch protocol and one correlated run title", () => { + const workflow = readYaml(E2E_PATH); + const inputs = workflow.on.workflow_dispatch.inputs; + + expect(inputs).toEqual( + expect.objectContaining({ + jobs: expect.any(Object), + pr_number: expect.any(Object), + checkout_sha: expect.any(Object), + plan_hash: expect.any(Object), + correlation_id: expect.any(Object), + }), + ); + expect(workflow["run-name"]).toContain( + "format('E2E PR #{0} required live {1}', inputs.pr_number, inputs.correlation_id)", + ); + }); + + it("executes child validation against the pull request current revision", () => { + const current = runChildValidation("a".repeat(40)); + const stale = runChildValidation("c".repeat(40)); + + expect(current.status).toBe(0); + expect(stale.status).toBe(1); + expect(stale.stdout).toContain("checkout_sha must match the pull request's current commit"); + }); + + it("logs each child state once and exits after success", () => { + const result = runWaitStep("success"); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(""); + expect(result.stdout.trim().split(/\r?\n/u)).toEqual([ + expect.stringContaining("status=in_progress"), + expect.stringContaining("status=completed conclusion=success"), + ]); + }); + + it("surfaces a terminal child failure", () => { + const result = runWaitStep("failure"); + + expect(result.status).toBe(1); + expect(result.stdout.match(/status=in_progress/gu)).toHaveLength(1); + expect(result.stderr).toContain("::error title=Required live run did not succeed::"); + expect(result.stderr).toContain("completed with conclusion failure"); + }); + + it("preserves GitHub CLI errors when status queries fail", () => { + const result = runWaitStep("query-failure"); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("simulated GitHub query failure"); + expect(result.stderr).toContain("::error title=Required live status query failed::"); + }); + + it("labels only the bounded wait exit as a timeout", () => { + const result = runWaitStep("timeout"); + + expect(result.status).toBe(124); + expect(result.stderr).toContain("::error title=Required live wait timed out::"); + expect(result.stderr).toContain("did not complete within 105 minutes"); + }); + + it("rejects an invalid child run ID before querying GitHub", () => { + const result = runWaitStep("success", { runId: "invalid" }); + + expect(result.status).toBe(1); + expect(result.ghCallCount).toBe(0); + expect(result.stderr).toContain("::error title=Invalid required live run ID::"); + }); + + it("fails closed for an unsupported child state", () => { + const result = runWaitStep("unsupported"); + + expect(result.status).toBe(1); + expect(result.ghCallCount).toBe(1); + expect(result.stderr).toContain("::error title=Unexpected required live state::"); + }); +}); diff --git a/test/required-live.test.ts b/test/required-live.test.ts new file mode 100644 index 00000000000..8a2e5dd3a32 --- /dev/null +++ b/test/required-live.test.ts @@ -0,0 +1,769 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; +import { buildRiskPlan, riskPlanRequiredJobIds } from "../tools/advisors/risk-plan.mts"; +import { + abandonRequiredLive, + assertCorrelatedWorkflowRun, + cancelRequiredLive, + classifyRequiredLiveEvidence, + dispatchRequiredLive, + expectedSignalShards, + findSignalFiles, + finishRequiredLive, + type PullRequest, + parseControllerCommand, + pullChangedFiles, + type RequiredLiveState, + startRequiredLive, + validateRequiredLiveState, + validateRiskPlan, + validateSignal, + validateWorkflowDispatchDetails, +} from "../tools/e2e/required-live.mts"; +import type { E2eRiskSignal } from "../tools/e2e/risk-signal.ts"; + +const HEAD_SHA = "a".repeat(40); +const BASE_SHA = "b".repeat(40); +const WORKFLOW_SHA = "d".repeat(40); +const CORRELATION_ID = "12345678-1234-4123-8123-123456789abc"; +const BROAD_FILES = [ + "src/lib/onboard.ts", + "src/lib/actions/upgrade-sandboxes.ts", + "src/lib/actions/sandbox/agents/apply.ts", + "src/lib/messaging/applier/agent-config.ts", + "src/lib/inference/health.ts", + "install.sh", + "src/lib/credentials/provider-list.ts", +] as const; +const BROAD_JOBS = [ + "cloud-onboard", + "credential-sanitization", + "security-posture", + "channels-add-remove", + "channels-stop-start", + "full-e2e", + "hermes-e2e", + "inference-routing", + "network-policy", + "onboard-repair", + "onboard-resume", + "state-backup-restore", + "upgrade-stale-sandbox", +] as const; + +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); +}); + +function githubResponse(value?: unknown, status = 200): Response { + return { + ok: status >= 200 && status < 300, + status, + json: async () => value, + text: async () => (value === undefined ? "" : JSON.stringify(value)), + } as Response; +} + +function sha256(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} + +function pullRequest(changedFiles = 1): PullRequest { + return { + number: 42, + state: "open", + changed_files: changedFiles, + head: { + ref: "feature/required-live", + sha: HEAD_SHA, + repo: { full_name: "NVIDIA/NemoClaw" }, + }, + base: { + sha: BASE_SHA, + repo: { full_name: "NVIDIA/NemoClaw" }, + }, + }; +} + +function pullRequestListItem(pull = pullRequest()): Omit { + const { changed_files: _changedFiles, ...item } = pull; + return item; +} + +function state(): RequiredLiveState { + const plan = buildRiskPlan({ headSha: HEAD_SHA, changedFiles: ["src/lib/onboard.ts"] }); + return { + version: 1, + commitSha: HEAD_SHA, + workflowSha: WORKFLOW_SHA, + planHash: plan.planHash, + correlationId: CORRELATION_ID, + prNumber: 42, + expectedJobs: ["onboard-repair", "onboard-resume"], + expectedShards: { + "onboard-repair": ["default"], + "onboard-resume": ["default"], + }, + }; +} + +function startCommand(workDir: string) { + const command = parseControllerCommand([ + "--mode", + "start", + "--head", + HEAD_SHA, + "--head-repo", + "NVIDIA/NemoClaw", + "--head-branch", + "feature/required-live", + "--workflow-sha", + WORKFLOW_SHA, + "--ci-conclusion", + "success", + "--work-dir", + workDir, + ]); + if (command.mode !== "start") throw new Error("unexpected command mode"); + return command; +} + +function signal( + gate: RequiredLiveState, + jobId: string, + shardId = "default", + overrides: Partial = {}, +): E2eRiskSignal { + return { + version: 1, + jobId, + shardId, + expectedSha: gate.commitSha, + testedSha: gate.commitSha, + planHash: gate.planHash, + correlationId: gate.correlationId, + passed: 1, + failed: 0, + skipped: 0, + pending: 0, + unhandledErrors: 0, + runReason: "passed", + ...overrides, + }; +} + +function workflowRun(gate: RequiredLiveState, overrides: Record = {}) { + return { + id: 23, + name: "E2E", + path: ".github/workflows/e2e.yaml", + workflow_id: 304268429, + event: "workflow_dispatch", + head_sha: gate.workflowSha, + status: "completed", + conclusion: "success", + display_title: `E2E PR #${gate.prNumber} required live ${gate.correlationId}`, + html_url: "https://github.com/NVIDIA/NemoClaw/actions/runs/23", + ...overrides, + }; +} + +describe("required live E2E controller", () => { + it("parses one lifecycle command set inside a private workspace", () => { + const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-required-live-")); + try { + expect( + parseControllerCommand([ + "--mode", + "start", + "--head", + HEAD_SHA, + "--head-repo", + "NVIDIA/NemoClaw", + "--head-branch", + "feature/required-live", + "--workflow-sha", + WORKFLOW_SHA, + "--ci-conclusion", + "success", + "--work-dir", + workDir, + ]), + ).toMatchObject({ + mode: "start", + planPath: path.join(workDir, "required-live-plan.json"), + statePath: path.join(workDir, "required-live-state.json"), + evidencePath: path.join(workDir, "evidence"), + }); + expect(parseControllerCommand(["--mode", "cancel", "--pr", "42"])).toEqual({ + mode: "cancel", + prNumber: 42, + }); + expect( + parseControllerCommand(["--mode", "abandon", "--check-id", "17", "--run-id", "23"]), + ).toEqual({ mode: "abandon", checkRunId: 17, childRunId: 23 }); + expect(() => + parseControllerCommand(["--mode", "cancel", "--pr", "9007199254740992"]), + ).toThrow(/safe integer range/u); + + fs.chmodSync(workDir, 0o755); + expect(() => parseControllerCommand(["--mode", "finish", "--work-dir", workDir])).toThrow( + /owned private absolute directory/u, + ); + } finally { + fs.chmodSync(workDir, 0o700); + fs.rmSync(workDir, { recursive: true, force: true }); + } + }); + + it("accepts only the current deterministic plan and bounded state", () => { + const plan = buildRiskPlan({ headSha: HEAD_SHA, changedFiles: ["src/lib/onboard.ts"] }); + const allowed = new Set(riskPlanRequiredJobIds(plan)); + const gate = state(); + + expect(validateRiskPlan(plan, allowed)).toEqual(plan); + expect(() => validateRiskPlan({ ...plan, version: 1 }, allowed)).toThrow( + /unsupported risk-plan version/u, + ); + expect(() => validateRiskPlan({ ...plan, planHash: "b".repeat(64) }, allowed)).toThrow( + /deterministic hash/u, + ); + expect(() => validateRiskPlan(plan, new Set())).toThrow(/unknown E2E job/u); + expect(validateRequiredLiveState(gate)).toEqual(gate); + expect(() => validateRequiredLiveState({ ...gate, prNumber: 0 })).toThrow(/PR number/u); + expect(() => validateRequiredLiveState({ ...gate, expectedShards: {} })).toThrow(/shard jobs/u); + }); + + it("paginates canonical pull request files and includes both names for renames", async () => { + const pageOne = Array.from({ length: 100 }, (_, index) => ({ + filename: `src/file-${index}.ts`, + ...(index === 0 ? { previous_filename: "src/old-name.ts" } : {}), + })); + const pageTwo = [{ filename: "src/file-100.ts" }]; + const fetchMock = vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => { + const url = String(input); + if (url.endsWith("page=1")) return githubResponse(pageOne); + if (url.endsWith("page=2")) return githubResponse(pageTwo); + throw new Error(`Unexpected request: ${url}`); + }); + + const files = await pullChangedFiles("NVIDIA/NemoClaw", pullRequest(101), "token"); + + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(files).toHaveLength(102); + expect(files.slice(0, 3)).toEqual(["src/old-name.ts", "src/file-0.ts", "src/file-1.ts"]); + await expect(pullChangedFiles("NVIDIA/NemoClaw", pullRequest(3001), "token")).rejects.toThrow( + /between 0 and 3000/u, + ); + }); + + it("fails closed for missing, duplicate, skipped, or failing evidence", () => { + const gate = state(); + const complete = gate.expectedJobs.map((job) => signal(gate, job)); + const classify = (signals: E2eRiskSignal[], workflowConclusion: string | null = "success") => + classifyRequiredLiveEvidence({ + workflowConclusion, + expectedJobs: gate.expectedJobs, + expectedShards: gate.expectedShards, + signals, + }); + + expect(classify(complete).conclusion).toBe("success"); + expect(classify([], "cancelled").conclusion).toBe("failure"); + expect(classify(complete.slice(0, 1)).title).toMatch(/missing evidence/u); + expect(classify([...complete, complete[0]!]).title).toMatch(/duplicate evidence/u); + expect( + classify([signal(gate, "onboard-repair", "default", { skipped: 1 }), complete[1]!]).title, + ).toMatch(/incomplete evidence/u); + expect( + classify([ + signal(gate, "onboard-repair", "default", { failed: 1, runReason: "failed" }), + complete[1]!, + ]).title, + ).toMatch(/test failures/u); + }); + + it("binds every signal to the revision, plan, correlation, job, and shard", () => { + const gate = state(); + const valid = signal(gate, "onboard-repair"); + + expect(validateSignal(valid, gate)).toEqual(valid); + expect(() => validateSignal({ ...valid, testedSha: BASE_SHA }, gate)).toThrow(/tested SHA/u); + expect(() => validateSignal({ ...valid, planHash: "c".repeat(64) }, gate)).toThrow( + /plan hash/u, + ); + expect(() => + validateSignal({ ...valid, correlationId: CORRELATION_ID.replace(/.$/u, "d") }, gate), + ).toThrow(/correlation/u); + expect(() => validateSignal({ ...valid, jobId: "other" }, gate)).toThrow(/unexpected/u); + }); + + it("derives shard policy from the checked-in workflow", () => { + expect(expectedSignalShards(["onboard-repair", "onboard-resume"])).toEqual({ + "onboard-repair": ["default"], + "onboard-resume": ["default"], + }); + const broadPlan = buildRiskPlan({ headSha: HEAD_SHA, changedFiles: BROAD_FILES }); + const broadShards = expectedSignalShards(riskPlanRequiredJobIds(broadPlan)); + expect(Object.keys(broadShards)).toHaveLength(13); + expect(Object.values(broadShards).flat()).toHaveLength(15); + expect(() => expectedSignalShards(["not-a-workflow-job"])).toThrow(/does not define/u); + }); + + it("dispatches every selected job through the five-field child protocol", async () => { + const jobs = ["onboard-repair", "onboard-resume", "full-e2e", "hermes-e2e"]; + const fetchMock = vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => { + if (String(input).endsWith("/git/ref/heads/main")) { + return githubResponse({ + ref: "refs/heads/main", + object: { type: "commit", sha: WORKFLOW_SHA }, + }); + } + return githubResponse({ + workflow_run_id: 23, + run_url: "https://api.github.com/repos/NVIDIA/NemoClaw/actions/runs/23", + html_url: "https://github.com/NVIDIA/NemoClaw/actions/runs/23", + }); + }); + + await expect( + dispatchRequiredLive({ + repository: "NVIDIA/NemoClaw", + token: "token", + jobs, + prNumber: 42, + commitSha: HEAD_SHA, + workflowSha: WORKFLOW_SHA, + planHash: "c".repeat(64), + correlationId: CORRELATION_ID, + }), + ).resolves.toBe(23); + expect(String(fetchMock.mock.calls[0]?.[0])).toContain("git/ref/heads/main"); + const request = fetchMock.mock.calls[1]!; + expect(String(request[0])).toContain("actions/workflows/e2e.yaml/dispatches"); + expect(JSON.parse(String(request[1]?.body))).toEqual({ + ref: "main", + inputs: { + jobs: jobs.join(","), + pr_number: "42", + checkout_sha: HEAD_SHA, + plan_hash: "c".repeat(64), + correlation_id: CORRELATION_ID, + }, + return_run_details: true, + }); + expect(() => + validateWorkflowDispatchDetails( + { + workflow_run_id: 23, + run_url: "https://api.github.com/repos/NVIDIA/NemoClaw/actions/runs/24", + html_url: "https://github.com/NVIDIA/NemoClaw/actions/runs/23", + }, + "NVIDIA/NemoClaw", + ), + ).toThrow(/mismatched workflow dispatch URLs/u); + }); + + it("refuses dispatch when main moved past the trusted workflow revision", async () => { + const fetchMock = vi.spyOn(globalThis, "fetch").mockResolvedValue( + githubResponse({ + ref: "refs/heads/main", + object: { type: "commit", sha: BASE_SHA }, + }), + ); + + await expect( + dispatchRequiredLive({ + repository: "NVIDIA/NemoClaw", + token: "token", + jobs: ["onboard-repair"], + prNumber: 42, + commitSha: HEAD_SHA, + workflowSha: WORKFLOW_SHA, + planHash: "c".repeat(64), + correlationId: CORRELATION_ID, + }), + ).rejects.toThrow(/no longer the current main revision/u); + expect(fetchMock).toHaveBeenCalledOnce(); + }); + + it("uses one child title for dispatch correlation and verification", () => { + const gate = state(); + const child = workflowRun(gate); + const identity = { + childRunId: 23, + correlationId: gate.correlationId, + prNumber: gate.prNumber, + repository: "NVIDIA/NemoClaw", + workflowSha: gate.workflowSha, + }; + + expect(() => assertCorrelatedWorkflowRun(child, identity)).not.toThrow(); + expect(() => + assertCorrelatedWorkflowRun({ ...child, display_title: "E2E unrelated" }, identity), + ).toThrow(/display_title/u); + }); + + it("runs the dispatch-to-evidence lifecycle and completes one successful check", async () => { + const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-required-live-lifecycle-")); + const outputPath = path.join(workDir, "github-output"); + fs.writeFileSync(outputPath, "", { mode: 0o600 }); + vi.stubEnv("GITHUB_TOKEN", "token"); + vi.stubEnv("GITHUB_REPOSITORY", "NVIDIA/NemoClaw"); + vi.stubEnv("GITHUB_OUTPUT", outputPath); + const requests: Array<{ url: string; method: string; body?: unknown }> = []; + let gate: RequiredLiveState | undefined; + vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => { + const url = String(input); + const method = init?.method ?? "GET"; + const body = init?.body ? JSON.parse(String(init.body)) : undefined; + requests.push({ url, method, body }); + if (url.endsWith("/check-runs") && method === "POST") return githubResponse({ id: 17 }); + if (url.includes("/pulls?state=open&head=")) { + return githubResponse([pullRequestListItem(pullRequest(BROAD_FILES.length))]); + } + if (url.includes("/pulls/42/files?")) { + return githubResponse(BROAD_FILES.map((filename) => ({ filename }))); + } + if (url.endsWith("/pulls/42")) return githubResponse(pullRequest(BROAD_FILES.length)); + if (url.endsWith("/git/ref/heads/main")) { + return githubResponse({ + ref: "refs/heads/main", + object: { type: "commit", sha: WORKFLOW_SHA }, + }); + } + if (url.endsWith("/actions/workflows/e2e.yaml/dispatches")) { + return githubResponse({ + workflow_run_id: 23, + run_url: "https://api.github.com/repos/NVIDIA/NemoClaw/actions/runs/23", + html_url: "https://github.com/NVIDIA/NemoClaw/actions/runs/23", + }); + } + if (url.endsWith("/actions/runs/23") && method === "GET") { + if (!gate) throw new Error("state was not loaded before finish"); + return githubResponse(workflowRun(gate)); + } + if (url.endsWith("/check-runs/17") && method === "PATCH") return githubResponse({}); + throw new Error(`Unexpected request: ${method} ${url}`); + }); + + try { + const command = startCommand(workDir); + await startRequiredLive(command); + gate = validateRequiredLiveState(JSON.parse(fs.readFileSync(command.statePath, "utf8"))); + for (const job of gate.expectedJobs) { + for (const shard of gate.expectedShards[job]!) { + const directory = path.join(command.evidencePath, `${job}-${shard}`); + fs.mkdirSync(directory, { recursive: true }); + fs.writeFileSync( + path.join(directory, "risk-signal.json"), + `${JSON.stringify(signal(gate, job, shard))}\n`, + ); + } + } + const outputs = Object.fromEntries( + fs + .readFileSync(outputPath, "utf8") + .trim() + .split("\n") + .map((line) => line.split("=", 2)), + ); + await finishRequiredLive({ + statePath: command.statePath, + stateHash: outputs.state_hash!, + evidencePath: command.evidencePath, + checkRunId: Number(outputs.check_id), + childRunId: Number(outputs.run_id), + }); + + expect(gate.expectedJobs).toEqual(BROAD_JOBS); + expect(requests.filter((request) => request.url.includes("/pulls?"))).toHaveLength(2); + expect(requests.filter((request) => request.url.endsWith("/pulls/42"))).toHaveLength(2); + const dispatch = requests.find((request) => request.url.endsWith("/dispatches")); + expect(dispatch?.body).toMatchObject({ + inputs: { + jobs: BROAD_JOBS.join(","), + pr_number: "42", + checkout_sha: HEAD_SHA, + plan_hash: gate.planHash, + correlation_id: gate.correlationId, + }, + }); + const checkUpdates = requests.filter( + (request) => request.url.endsWith("/check-runs/17") && request.method === "PATCH", + ); + expect(checkUpdates).toHaveLength(2); + expect(checkUpdates[0]?.body).toMatchObject({ + status: "in_progress", + output: { + title: "Running 13 required live E2E jobs", + summary: expect.stringContaining("upgrade-stale-sandbox"), + }, + }); + expect(checkUpdates[1]?.body).toMatchObject({ + status: "completed", + conclusion: "success", + output: { title: "Required live E2E passed" }, + }); + expect(fs.readFileSync(outputPath, "utf8")).toContain("finalized=true"); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + }); + + it("fails without dispatch when the pull request changes during planning", async () => { + const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-required-live-race-")); + const outputPath = path.join(workDir, "github-output"); + fs.writeFileSync(outputPath, "", { mode: 0o600 }); + vi.stubEnv("GITHUB_TOKEN", "token"); + vi.stubEnv("GITHUB_REPOSITORY", "NVIDIA/NemoClaw"); + vi.stubEnv("GITHUB_OUTPUT", outputPath); + const requests: Array<{ url: string; method: string; body?: unknown }> = []; + let listCalls = 0; + let detailCalls = 0; + const updatedPull = { + ...pullRequest(), + base: { ...pullRequest().base, sha: "c".repeat(40) }, + }; + vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => { + const url = String(input); + const method = init?.method ?? "GET"; + const body = init?.body ? JSON.parse(String(init.body)) : undefined; + requests.push({ url, method, body }); + if (url.endsWith("/check-runs") && method === "POST") return githubResponse({ id: 17 }); + if (url.includes("/pulls?state=open&head=")) { + listCalls += 1; + return githubResponse([pullRequestListItem(listCalls === 1 ? pullRequest() : updatedPull)]); + } + if (url.includes("/pulls/42/files?")) { + return githubResponse([{ filename: "src/lib/onboard.ts" }]); + } + if (url.endsWith("/pulls/42")) { + detailCalls += 1; + return githubResponse(detailCalls === 1 ? pullRequest() : updatedPull); + } + if (url.endsWith("/check-runs/17") && method === "PATCH") return githubResponse({}); + throw new Error(`Unexpected request: ${method} ${url}`); + }); + + try { + await expect(startRequiredLive(startCommand(workDir))).rejects.toThrow( + /changed while required live E2E was being prepared/u, + ); + expect(requests.some((request) => request.url.endsWith("/dispatches"))).toBe(false); + expect(requests.some((request) => request.url.endsWith("/git/ref/heads/main"))).toBe(false); + const finalUpdate = requests.find( + (request) => request.url.endsWith("/check-runs/17") && request.method === "PATCH", + ); + expect(finalUpdate?.body).toMatchObject({ status: "completed", conclusion: "failure" }); + expect(fs.readFileSync(outputPath, "utf8")).toContain("finalized=true"); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + }); + + it("cancels the child and closes the check when startup fails after dispatch", async () => { + const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-required-live-start-")); + const outputPath = path.join(workDir, "github-output"); + fs.writeFileSync(outputPath, "", { mode: 0o600 }); + vi.stubEnv("GITHUB_TOKEN", "token"); + vi.stubEnv("GITHUB_REPOSITORY", "NVIDIA/NemoClaw"); + vi.stubEnv("GITHUB_OUTPUT", outputPath); + const requests: Array<{ url: string; method: string; body?: unknown }> = []; + let checkPatches = 0; + vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => { + const url = String(input); + const method = init?.method ?? "GET"; + const body = init?.body ? JSON.parse(String(init.body)) : undefined; + requests.push({ url, method, body }); + if (url.endsWith("/check-runs") && method === "POST") return githubResponse({ id: 17 }); + if (url.includes("/pulls?state=open&head=")) return githubResponse([pullRequestListItem()]); + if (url.includes("/pulls/42/files?")) { + return githubResponse([{ filename: "src/lib/onboard.ts" }]); + } + if (url.endsWith("/pulls/42")) return githubResponse(pullRequest()); + if (url.endsWith("/git/ref/heads/main")) { + return githubResponse({ + ref: "refs/heads/main", + object: { type: "commit", sha: WORKFLOW_SHA }, + }); + } + if (url.endsWith("/actions/workflows/e2e.yaml/dispatches")) { + return githubResponse({ + workflow_run_id: 23, + run_url: "https://api.github.com/repos/NVIDIA/NemoClaw/actions/runs/23", + html_url: "https://github.com/NVIDIA/NemoClaw/actions/runs/23", + }); + } + if (url.endsWith("/actions/runs/23/cancel") && method === "POST") { + return githubResponse(undefined, 202); + } + if (url.endsWith("/check-runs/17") && method === "PATCH") { + checkPatches += 1; + return checkPatches === 1 + ? githubResponse({ message: "simulated update failure" }, 500) + : githubResponse({}); + } + throw new Error(`Unexpected request: ${method} ${url}`); + }); + + try { + await expect(startRequiredLive(startCommand(workDir))).rejects.toThrow( + /simulated update failure/u, + ); + expect(requests.some((request) => request.url.endsWith("/actions/runs/23/cancel"))).toBe( + true, + ); + const checkUpdates = requests.filter((request) => request.url.endsWith("/check-runs/17")); + expect(checkUpdates).toHaveLength(2); + expect(checkUpdates[1]?.body).toMatchObject({ status: "completed", conclusion: "failure" }); + expect(fs.readFileSync(outputPath, "utf8")).toContain("finalized=true"); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + }); + + it.each([ + { label: "missing evidence", status: "completed", expectCancellation: false }, + { label: "an unfinished child", status: "in_progress", expectCancellation: true }, + ])("closes the check as failure for $label", async ({ status, expectCancellation }) => { + const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-required-live-finish-")); + const outputPath = path.join(workDir, "github-output"); + const statePath = path.join(workDir, "required-live-state.json"); + const evidencePath = path.join(workDir, "evidence"); + const gate = state(); + const serializedState = `${JSON.stringify(gate, null, 2)}\n`; + fs.writeFileSync(outputPath, "", { mode: 0o600 }); + fs.writeFileSync(statePath, serializedState, { mode: 0o600 }); + fs.mkdirSync(evidencePath); + vi.stubEnv("GITHUB_TOKEN", "token"); + vi.stubEnv("GITHUB_REPOSITORY", "NVIDIA/NemoClaw"); + vi.stubEnv("GITHUB_OUTPUT", outputPath); + const requests: Array<{ url: string; method: string; body?: unknown }> = []; + vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => { + const url = String(input); + const method = init?.method ?? "GET"; + const body = init?.body ? JSON.parse(String(init.body)) : undefined; + requests.push({ url, method, body }); + if (url.endsWith("/actions/runs/23") && method === "GET") { + return githubResponse(workflowRun(gate, { status, conclusion: "success" })); + } + if (url.endsWith("/actions/runs/23/cancel") && method === "POST") { + return githubResponse(undefined, 202); + } + if (url.endsWith("/check-runs/17") && method === "PATCH") return githubResponse({}); + throw new Error(`Unexpected request: ${method} ${url}`); + }); + + try { + await expect( + finishRequiredLive({ + statePath, + stateHash: sha256(serializedState), + evidencePath, + checkRunId: 17, + childRunId: 23, + }), + ).rejects.toThrow(); + expect(requests.some((request) => request.url.endsWith("/actions/runs/23/cancel"))).toBe( + expectCancellation, + ); + const completion = requests.find((request) => request.url.endsWith("/check-runs/17")); + expect(completion?.body).toMatchObject({ status: "completed", conclusion: "failure" }); + expect(fs.readFileSync(outputPath, "utf8")).toContain("finalized=true"); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + }); + + it("cancels only active child runs for the pull request", async () => { + vi.stubEnv("GITHUB_TOKEN", "token"); + vi.stubEnv("GITHUB_REPOSITORY", "NVIDIA/NemoClaw"); + const gate = state(); + const fetchMock = vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => { + const url = String(input); + if (url.includes("/actions/workflows/e2e.yaml/runs?")) { + return githubResponse({ + workflow_runs: [ + workflowRun(gate, { status: "in_progress" }), + workflowRun(gate, { id: 24, status: "completed" }), + workflowRun(gate, { id: 25, status: "queued", display_title: "E2E manual" }), + ], + }); + } + if (url.endsWith("/actions/runs/23/cancel") && init?.method === "POST") { + return githubResponse(undefined, 202); + } + throw new Error(`Unexpected request: ${init?.method ?? "GET"} ${url}`); + }); + + await expect(cancelRequiredLive(42)).resolves.toBe(1); + expect( + fetchMock.mock.calls.filter(([input]) => String(input).endsWith("/cancel")), + ).toHaveLength(1); + }); + + it("cancels a known child and closes an abandoned check as failure", async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-required-live-abandon-")); + const outputPath = path.join(directory, "github-output"); + fs.writeFileSync(outputPath, "", { mode: 0o600 }); + vi.stubEnv("GITHUB_TOKEN", "token"); + vi.stubEnv("GITHUB_REPOSITORY", "NVIDIA/NemoClaw"); + vi.stubEnv("GITHUB_OUTPUT", outputPath); + const requests: Array<{ url: string; body?: unknown }> = []; + vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => { + requests.push({ + url: String(input), + body: init?.body ? JSON.parse(String(init.body)) : undefined, + }); + return githubResponse(undefined, String(input).endsWith("/cancel") ? 202 : 200); + }); + + try { + await abandonRequiredLive(17, 23); + expect(requests.map((request) => request.url)).toEqual([ + "https://api.github.com/repos/NVIDIA/NemoClaw/actions/runs/23/cancel", + "https://api.github.com/repos/NVIDIA/NemoClaw/check-runs/17", + ]); + expect(requests[1]?.body).toMatchObject({ status: "completed", conclusion: "failure" }); + expect(fs.readFileSync(outputPath, "utf8")).toContain("finalized=true"); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }); + + it("bounds recursive signal discovery and rejects symlinks", () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-required-live-evidence-")); + try { + const first = path.join(directory, "first"); + fs.mkdirSync(first); + fs.writeFileSync(path.join(first, "risk-signal.json"), "{}\n"); + expect(findSignalFiles(directory, { maxDepth: 2, maxEntries: 3, maxSignalFiles: 1 })).toEqual( + [path.join(first, "risk-signal.json")], + ); + + const second = path.join(directory, "second"); + fs.mkdirSync(second); + fs.writeFileSync(path.join(second, "risk-signal.json"), "{}\n"); + expect(() => + findSignalFiles(directory, { maxDepth: 2, maxEntries: 8, maxSignalFiles: 1 }), + ).toThrow(/signal-file limit/u); + + fs.rmSync(second, { recursive: true }); + fs.symlinkSync(first, path.join(directory, "linked")); + expect(() => + findSignalFiles(directory, { maxDepth: 2, maxEntries: 8, maxSignalFiles: 2 }), + ).toThrow(/symlinks/u); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }); +}); diff --git a/tools/advisors/risk-plan.mts b/tools/advisors/risk-plan.mts index d41f5f7550f..0921bd98b90 100644 --- a/tools/advisors/risk-plan.mts +++ b/tools/advisors/risk-plan.mts @@ -3,8 +3,7 @@ import { createHash } from "node:crypto"; -export const RISK_PLAN_VERSION = 1 as const; -export const DEFAULT_MAX_AUTOMATIC_JOBS = 3; +export const RISK_PLAN_VERSION = 2 as const; export type RiskTier = 0 | 1 | 2 | 3; export type RiskFamilyId = @@ -41,9 +40,6 @@ export type RiskPlan = { tier: RiskTier; families: RiskPlanFamily[]; requiredJobs: RiskPlanJob[]; - automaticJobs: string[]; - maxAutomaticJobs: number; - requiresManualExpansion: boolean; }; type RiskRule = Omit & { @@ -223,7 +219,6 @@ function planDigest(value: Omit): string { export function buildRiskPlan(options: { headSha: string; changedFiles: readonly string[]; - maxAutomaticJobs?: number; }): RiskPlan { const changedFiles = stableUnique(options.changedFiles); const runtimeFiles = changedFiles.filter(isRuntimeRelevant); @@ -263,8 +258,6 @@ export function buildRiskPlan(options: { const requiredJobs = [...jobs.values()].sort( (left, right) => right.tier - left.tier || left.id.localeCompare(right.id), ); - const maxAutomaticJobs = options.maxAutomaticJobs ?? DEFAULT_MAX_AUTOMATIC_JOBS; - const automaticJobs = requiredJobs.slice(0, maxAutomaticJobs).map((job) => job.id); const tier = families.reduce( (highest, family) => Math.max(highest, family.tier) as RiskTier, 0, @@ -276,9 +269,6 @@ export function buildRiskPlan(options: { tier, families, requiredJobs, - automaticJobs, - maxAutomaticJobs, - requiresManualExpansion: requiredJobs.length > maxAutomaticJobs, }; return { ...withoutHash, planHash: planDigest(withoutHash) }; diff --git a/tools/e2e-advisor/README.md b/tools/e2e-advisor/README.md index 11b6991f91f..b7ebed364ab 100644 --- a/tools/e2e-advisor/README.md +++ b/tools/e2e-advisor/README.md @@ -38,42 +38,57 @@ the trusted timing signal. `.github/workflows/e2e.yaml`, but the advisor job does not trigger those commands automatically. -## Post-merge shadow controller - -The model-independent `.github/workflows/post-merge-e2e-risk-gate-shadow.yaml` workflow -uses the same checked-in risk-plan policy after a commit lands on `main`. It -does not consume model output or the advisor artifact. Instead, -`tools/e2e-advisor/post-merge-risk-gate.mts` confirms the trusted controller checkout, -builds a new plan from the exact `github.event.before` and `github.event.after` range, -and dispatches at most three `automaticJobs` to `e2e.yaml` against the merged -commit. The controller opts into GitHub's workflow-dispatch run details and -uses the returned run ID as the sole child-run selector, so a lookalike run -cannot win a polling race for the same correlation ID. - -The child workflow validates that the exact checkout SHA equals the workflow's -own current `main` commit, verifies its reachability, and checks selective-job -inputs, plan hash, and correlation ID before E2E preparation or secret-bearing -jobs can run. If `main` advances before an older controller dispatches, that -child fails closed and the controller reports failure rather than executing the -older commit with current secrets. The shadow-only Vitest reporter -records the observed checkout SHA and pass, failure, skip, pending, and -unhandled-error counts for each job and matrix shard. The controller accepts -only signals bound to the expected SHA, plan hash, correlation ID, job, and -shard. - -The start step records a SHA-256 digest of its private controller state in a -GitHub step output. After child artifacts are downloaded, the finish step reads -that state once and verifies the digest before parsing it or classifying any -evidence, so downloaded files cannot change the dispatch state used for the -check result. - -The controller writes a check on the merged commit without posting a PR -comment or running the scheduled/manual scorecard. Complete unskipped evidence -reports success, selected E2E workflow or test failures for the merged commit report -failure, and incomplete, ambiguous, skipped, or cap-limited evidence that -requires manual expansion reports neutral. If no runtime risk family matches, -it reports success without dispatching E2E. This is post-merge shadow evidence, -not a required pre-merge check. +## Required live PR check + +The model-independent `.github/workflows/required-live-e2e.yaml` workflow owns the +required `E2E / Required Live` check for same-repository pull requests after +`CI / Pull Request` completes. +It does not consume model output or advisor artifacts. +Instead, `tools/e2e/required-live.mts` resolves the open pull request for the +triggering revision, reads its complete changed-file list, and builds a new plan +from the checked-in risk policy. +The controller dispatches every job in `requiredJobs` through `e2e.yaml`. +If no runtime risk family matches, it reports success without dispatching live E2E. + +The controller verifies the pull request identity again immediately before +dispatch. +It also records the trusted controller revision, requires that revision to +still be the current `main` revision immediately before dispatch, and accepts +only a child workflow run created from that same revision. +The child workflow validates that the pull request is still open, belongs to +the base repository, and still points to the requested checkout SHA before E2E +preparation or secret-bearing jobs can run. +It also requires selective jobs, an empty target fan-out, a valid plan hash, +and a valid correlation ID. +The controller uses GitHub's returned workflow run ID as the sole child-run +selector for waiting, evidence download, and completion. + +The Vitest reporter records the observed checkout SHA and pass, failure, skip, +pending, and unhandled-error counts for each selected job and matrix shard. +The checked workflow boundary requires every job named by the deterministic +policy to expose its matching job identity, attach the reporter to every +Vitest invocation, and always upload its evidence artifact. +The controller accepts only signals bound to the expected SHA, plan hash, +correlation ID, job, and shard. +It also records a SHA-256 digest of its private dispatch state and verifies that +digest before parsing downloaded evidence. + +The check has a binary result. +It succeeds only when the correlated workflow succeeds and every expected job +shard produces one complete, unskipped pass. +Workflow failures, failed tests, missing or duplicate signals, skipped or +pending tests, interrupted runs, and controller or evidence-validation errors +all fail the check. +Evidence download has its own 10-minute limit within the coordinator's +180-minute job budget, and exceeding that limit fails the check. +Pull request synchronization, reopening, or closure cancels active child runs +for that pull request, and the E2E workflow cancels a superseded child run when +a new revision is dispatched. + +E2E Advisor remains advisory. +It uses the same deterministic policy as a recommendation floor and may add +adjacent coverage, but its model output and availability never determine the +required check. ## Required secret @@ -98,11 +113,12 @@ dispatch commands; it does not trigger E2E workflows automatically. ## Artifacts - `e2e-advisor-prompt.md` — task prompt sent to the advisor. Diff, changed files, metadata, and schema are exposed through deterministic turn-scoped context tools and captured in the session transcript. -- `risk-plan.json` — deterministic risk families, invariants, and required jobs for the PR - head commit and changed-file set, plus a capped `automaticJobs` subset, - manual-expansion state, and the plan digest. Both E2E advisor projections consume the - required-job floor, while the separate post-merge controller rebuilds the plan and - dispatches the automatic subset. +- `risk-plan.json` — deterministic risk families, invariants, required jobs, + changed files, and the plan digest for the pull request revision. + Both E2E Advisor projections consume this required-job floor. + The required live controller independently rebuilds the plan from GitHub's + pull request file list and dispatches every selected job, so this advisor + artifact is not an input to the required check. - `e2e-advisor-raw-output.txt` — raw advisor transcript and diagnostics. - `e2e-advisor-result.json` — parsed advisor response or execution metadata. - `e2e-advisor-session.html` — exported advisor session transcript. @@ -134,6 +150,5 @@ secret. Run `npm install` first so the Pi SDK dependency is available. `tools/e2e-advisor/schema.json` defines the normalized coverage recommendation shape. `tools/e2e-advisor/targets-schema.json` defines the normalized target recommendation shape used by the `targets` and `jobs` dispatch commands. -The post-merge shadow check does not establish pre-merge enforcement. Any future required check -must verify complete E2E evidence for the same PR head commit without making model availability part -of the merge authority. +The required live check verifies complete E2E evidence for the same pull +request revision without making model availability part of merge authority. diff --git a/tools/e2e-advisor/post-merge-risk-gate.mts b/tools/e2e-advisor/post-merge-risk-gate.mts deleted file mode 100755 index cb75b049b3d..00000000000 --- a/tools/e2e-advisor/post-merge-risk-gate.mts +++ /dev/null @@ -1,965 +0,0 @@ -#!/usr/bin/env node - -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { execFileSync } from "node:child_process"; -import { createHash, randomUUID } from "node:crypto"; -import fs from "node:fs"; -import path from "node:path"; -import { pathToFileURL } from "node:url"; - -import YAML from "yaml"; - -import { githubApi } from "../advisors/github.mts"; -import { parseArgs } from "../advisors/io.mts"; -import { buildRiskPlan, type RiskPlan } from "../advisors/risk-plan.mts"; -import { readFreeStandingJobsInventory } from "../e2e/workflow-boundary.mts"; -import { readPrivateRegularFile, writePrivateRegularFile } from "./private-file.ts"; -import type { E2eRiskSignal } from "./risk-signal.ts"; - -const E2E_WORKFLOW = "e2e.yaml"; -const E2E_WORKFLOW_PATH = `.github/workflows/${E2E_WORKFLOW}`; -const CHECK_NAME = "E2E / Post-merge Risk Gate (shadow)"; -const SHA_PATTERN = /^[a-f0-9]{40}$/u; -const HASH_PATTERN = /^[a-f0-9]{64}$/u; -const JOB_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]*$/u; -const SHARD_PATTERN = /^(?:default|[A-Za-z0-9][A-Za-z0-9_-]*)$/u; -const CORRELATION_PATTERN = - /^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/u; -const MAX_PLAN_BYTES = 1024 * 1024; -const MAX_CONTROLLER_ERROR_CHARS = 512; -const DEFAULT_EVIDENCE_LIMITS = { - maxDepth: 8, - maxEntries: 4096, - maxSignalFiles: 12, -} as const; - -type ControllerPaths = { - planPath: string; - statePath: string; - evidencePath: string; -}; - -export type ControllerCommand = - | ({ mode: "start"; baseSha: string; commitSha: string } & ControllerPaths) - | ({ - mode: "finish"; - checkRunId: number; - childRunId: number; - stateHash: string; - } & ControllerPaths) - | { mode: "abandon"; checkRunId: number }; - -type CheckConclusion = "success" | "failure" | "neutral"; - -type WorkflowRun = { - id: number; - name: string; - path: string; - workflow_id: number; - event: string; - head_sha: string; - status: string; - conclusion: string | null; - created_at: string; - display_title: string; - html_url: string; -}; - -type CheckRun = { id: number }; - -type WorkflowDispatchDetails = { - workflow_run_id: number; - run_url: string; - html_url: string; -}; - -type WorkflowRunIdentity = { - childRunId: number; - correlationId: string; - repository: string; -}; - -export type RiskGateState = { - version: 1; - commitSha: string; - planHash: string; - correlationId: string; - expectedJobs: string[]; - expectedShards: Record; - requiresManualExpansion: boolean; -}; - -export type RiskEvidenceVerdict = { - conclusion: CheckConclusion; - title: string; - summary: string; -}; - -export function assertTrustedMainPush(options: { - eventName: string | undefined; - ref: string | undefined; - sha: string | undefined; - commitSha: string; -}): void { - if ( - options.eventName !== "push" || - options.ref !== "refs/heads/main" || - options.sha !== options.commitSha - ) { - throw new Error("post-merge risk dispatch requires the exact trusted main push context"); - } -} - -function isObjectRecord(value: unknown): value is Record { - return !!value && typeof value === "object" && !Array.isArray(value); -} - -function requiredArgument(value: string | undefined, name: string): string { - if (!value) throw new Error(`--${name} is required`); - return value; -} - -function parsePositiveId(value: string, name: string): number { - if (!/^[1-9][0-9]*$/u.test(value)) throw new Error(`${name} must be a positive integer`); - const parsed = Number(value); - if (!Number.isSafeInteger(parsed)) throw new Error(`${name} exceeds the safe integer range`); - return parsed; -} - -function parseHash(value: string | undefined, name: string): string { - const parsed = requiredArgument(value, name); - if (!HASH_PATTERN.test(parsed)) throw new Error(`--${name} must be a lowercase SHA-256 hash`); - return parsed; -} - -function sha256(value: string): string { - return createHash("sha256").update(value).digest("hex"); -} - -export function privateControllerPaths(workDir: string): ControllerPaths { - const resolved = path.resolve(workDir); - const stat = fs.lstatSync(resolved); - const currentUid = typeof process.getuid === "function" ? process.getuid() : null; - if ( - resolved !== workDir || - !stat.isDirectory() || - stat.isSymbolicLink() || - (stat.mode & 0o077) !== 0 || - (currentUid !== null && stat.uid !== currentUid) - ) { - throw new Error("--work-dir must be an owned private absolute directory"); - } - return { - planPath: path.join(resolved, "post-merge-risk-plan.json"), - statePath: path.join(resolved, "e2e-risk-gate-state.json"), - evidencePath: path.join(resolved, "evidence"), - }; -} - -export function parseControllerCommand(argv: string[]): ControllerCommand { - const args = parseArgs(argv); - if (args.mode === "start") { - return { - mode: "start", - baseSha: requiredArgument(args.base, "base"), - commitSha: requiredArgument(args.commit, "commit"), - ...privateControllerPaths(requiredArgument(args.workDir, "work-dir")), - }; - } - if (args.mode === "finish") { - return { - mode: "finish", - ...privateControllerPaths(requiredArgument(args.workDir, "work-dir")), - checkRunId: parsePositiveId(requiredArgument(args.checkId, "check-id"), "--check-id"), - childRunId: parsePositiveId(requiredArgument(args.runId, "run-id"), "--run-id"), - stateHash: parseHash(args.stateHash, "state-hash"), - }; - } - if (args.mode === "abandon") { - return { - mode: "abandon", - checkRunId: parsePositiveId(requiredArgument(args.checkId, "check-id"), "--check-id"), - }; - } - throw new Error("--mode must be start, finish, or abandon"); -} - -function readRegularJson(file: string, maxBytes = MAX_PLAN_BYTES): unknown { - return JSON.parse(readPrivateRegularFile(file, { maxBytes })!); -} - -export function validateRiskGateState(value: unknown): RiskGateState { - if (!isObjectRecord(value) || value.version !== 1) { - throw new Error("invalid risk-gate state version"); - } - if (typeof value.commitSha !== "string" || !SHA_PATTERN.test(value.commitSha)) { - throw new Error("risk-gate state commit SHA is invalid"); - } - if (typeof value.planHash !== "string" || !HASH_PATTERN.test(value.planHash)) { - throw new Error("risk-gate state plan hash is invalid"); - } - if (typeof value.correlationId !== "string" || !CORRELATION_PATTERN.test(value.correlationId)) { - throw new Error("risk-gate state correlation id is invalid"); - } - if ( - !Array.isArray(value.expectedJobs) || - value.expectedJobs.length < 1 || - value.expectedJobs.length > 3 || - !value.expectedJobs.every((job) => typeof job === "string" && JOB_PATTERN.test(job)) || - new Set(value.expectedJobs).size !== value.expectedJobs.length - ) { - throw new Error("risk-gate state expected jobs are invalid"); - } - if (!isObjectRecord(value.expectedShards)) { - throw new Error("risk-gate state shards are invalid"); - } - const shardJobs = Object.keys(value.expectedShards).sort(); - if (JSON.stringify(shardJobs) !== JSON.stringify([...value.expectedJobs].sort())) { - throw new Error("risk-gate state shard jobs do not match expected jobs"); - } - for (const job of value.expectedJobs) { - const shards = value.expectedShards[job]; - if ( - !Array.isArray(shards) || - shards.length < 1 || - new Set(shards).size !== shards.length || - !shards.every((shard) => typeof shard === "string" && SHARD_PATTERN.test(shard)) - ) { - throw new Error(`risk-gate state shards are invalid for ${job}`); - } - } - if (typeof value.requiresManualExpansion !== "boolean") { - throw new Error("risk-gate manual-expansion state is invalid"); - } - return value as RiskGateState; -} - -export function validateRiskPlan(value: unknown, allowedJobs: ReadonlySet): RiskPlan { - if (!isObjectRecord(value)) throw new Error("risk plan must be an object"); - if (value.version !== 1) throw new Error("unsupported risk-plan version"); - if (typeof value.headSha !== "string" || !SHA_PATTERN.test(value.headSha)) { - throw new Error("risk plan headSha must be a lowercase 40-character SHA"); - } - if ( - !Array.isArray(value.changedFiles) || - !value.changedFiles.every((file) => typeof file === "string") - ) { - throw new Error("risk plan changedFiles must be strings"); - } - if (value.maxAutomaticJobs !== 3) throw new Error("risk plan automatic-job cap must be 3"); - const rebuilt = buildRiskPlan({ - headSha: value.headSha, - changedFiles: value.changedFiles, - maxAutomaticJobs: value.maxAutomaticJobs, - }); - if (JSON.stringify(value) !== JSON.stringify(rebuilt)) { - throw new Error("risk plan does not match its deterministic hash and inputs"); - } - if (!HASH_PATTERN.test(rebuilt.planHash)) throw new Error("risk plan hash is invalid"); - const automatic = new Set(rebuilt.automaticJobs); - if (automatic.size !== rebuilt.automaticJobs.length) { - throw new Error("risk plan automatic jobs must be unique"); - } - for (const job of rebuilt.requiredJobs) { - if (!JOB_PATTERN.test(job.id) || !allowedJobs.has(job.id)) { - throw new Error(`risk plan names unknown E2E job: ${job.id}`); - } - } - return rebuilt; -} - -export function validateSignal( - value: unknown, - state: Pick< - RiskGateState, - "commitSha" | "planHash" | "correlationId" | "expectedJobs" | "expectedShards" - >, -): E2eRiskSignal { - if (!isObjectRecord(value) || value.version !== 1) { - throw new Error("invalid risk signal version"); - } - const signal = value as E2eRiskSignal; - if (!state.expectedJobs.includes(signal.jobId)) throw new Error("risk signal job is unexpected"); - if (!state.expectedShards[signal.jobId]?.includes(signal.shardId)) { - throw new Error("risk signal shard is unexpected"); - } - if (signal.expectedSha !== state.commitSha) throw new Error("risk signal SHA mismatch"); - if (signal.testedSha !== state.commitSha) throw new Error("risk signal tested SHA mismatch"); - if (signal.planHash !== state.planHash) throw new Error("risk signal plan hash mismatch"); - if (signal.correlationId !== state.correlationId) { - throw new Error("risk signal correlation mismatch"); - } - for (const key of ["passed", "failed", "skipped", "pending", "unhandledErrors"] as const) { - if (!Number.isSafeInteger(signal[key]) || signal[key] < 0) { - throw new Error(`risk signal ${key} must be a non-negative integer`); - } - } - if (!(["passed", "failed", "interrupted"] as const).includes(signal.runReason)) { - throw new Error("risk signal runReason is invalid"); - } - return signal; -} - -export function classifyRiskEvidence(options: { - workflowConclusion: string | null; - expectedJobs: readonly string[]; - expectedShards: Readonly>; - signals: readonly E2eRiskSignal[]; - requiresManualExpansion: boolean; -}): RiskEvidenceVerdict { - if ( - ["failure", "timed_out", "action_required", "startup_failure"].includes( - options.workflowConclusion ?? "", - ) - ) { - return { - conclusion: "failure", - title: "Selected E2E workflow failed", - summary: "The correlated workflow reported a failing terminal conclusion.", - }; - } - if (options.workflowConclusion !== "success") { - return { - conclusion: "neutral", - title: "Selected E2E workflow produced no complete signal", - summary: "The correlated workflow did not report a successful terminal conclusion.", - }; - } - const byJobShard = new Map(); - const duplicates = new Set(); - for (const signal of options.signals) { - const key = `${signal.jobId}:${signal.shardId}`; - if (byJobShard.has(key)) duplicates.add(key); - byJobShard.set(key, signal); - } - if (duplicates.size > 0) { - return { - conclusion: "neutral", - title: "Selected E2E jobs produced ambiguous evidence", - summary: "More than one signal was uploaded for at least one expected job shard.", - }; - } - const expectedEvidence = options.expectedJobs.flatMap((job) => - (options.expectedShards[job] ?? []).map((shard) => `${job}:${shard}`), - ); - const jobsWithoutShardPolicy = options.expectedJobs.filter( - (job) => (options.expectedShards[job]?.length ?? 0) === 0, - ); - if (jobsWithoutShardPolicy.length > 0) { - return { - conclusion: "neutral", - title: "Selected E2E jobs lack an evidence policy", - summary: "At least one selected job had no trusted shard policy.", - }; - } - const missing = expectedEvidence.filter((key) => !byJobShard.has(key)); - if (missing.length > 0) { - // Missing bound evidence is unverifiable, not proof that product behavior - // failed. Shadow checks become green only for complete evidence; neutral - // keeps incomplete infrastructure evidence from masquerading as a pass. - return { - conclusion: "neutral", - title: "Selected E2E jobs are missing test evidence", - summary: "At least one expected job shard did not upload a bound risk signal.", - }; - } - const failed = expectedEvidence.filter((key) => { - const signal = byJobShard.get(key)!; - return signal.failed > 0 || signal.unhandledErrors > 0 || signal.runReason === "failed"; - }); - if (failed.length > 0) { - return { - conclusion: "failure", - title: "Selected E2E jobs reported test failures", - summary: "At least one selected job shard reported a test failure or unhandled error.", - }; - } - const partial = expectedEvidence.filter((key) => { - const signal = byJobShard.get(key)!; - return ( - signal.passed < 1 || signal.skipped > 0 || signal.pending > 0 || signal.runReason !== "passed" - ); - }); - if (partial.length > 0) { - return { - conclusion: "neutral", - title: "Selected E2E jobs produced partial or skipped evidence", - summary: "At least one expected job shard did not produce a complete, unskipped pass.", - }; - } - if (options.requiresManualExpansion) { - return { - conclusion: "neutral", - title: "Automatic shadow subset passed; broader evidence is required", - summary: - "The risk plan exceeded the three-job automatic cap, so this passing subset is not complete merge evidence.", - }; - } - return { - conclusion: "success", - title: "All risk-selected E2E jobs passed", - summary: "Every expected job shard produced complete, unskipped evidence.", - }; -} - -function appendOutput(name: string, value: string): void { - const output = process.env.GITHUB_OUTPUT; - if (!output) return; - if (!/^(?:check_id|dispatched|finalized|run_id|state_hash)$/u.test(name)) { - throw new Error("invalid controller output name"); - } - const validValue = - name === "state_hash" ? HASH_PATTERN.test(value) : /^(?:true|false|[1-9][0-9]*)$/u.test(value); - if (!validValue) { - throw new Error("invalid controller output value"); - } - const descriptor = fs.openSync( - output, - fs.constants.O_WRONLY | fs.constants.O_APPEND | (fs.constants.O_NOFOLLOW ?? 0), - ); - try { - if (!fs.fstatSync(descriptor).isFile()) throw new Error("GITHUB_OUTPUT must be a regular file"); - // GitHub supplies this output file; values are restricted above to fixed - // booleans, positive decimal IDs, or a lowercase SHA-256 digest before the - // descriptor write. - // codeql[js/http-to-file-access] - fs.writeFileSync(descriptor, `${name}=${value}\n`, "utf8"); - } finally { - fs.closeSync(descriptor); - } -} - -async function createCheck( - repository: string, - token: string, - headSha: string, - title: string, - summary: string, -): Promise { - const check = await githubApi(`repos/${repository}/check-runs`, token, { - method: "POST", - body: { - name: CHECK_NAME, - head_sha: headSha, - status: "in_progress", - output: { title, summary }, - }, - userAgent: "nemoclaw-e2e-risk-gate", - }); - if (!Number.isSafeInteger(check.id) || check.id < 1) - throw new Error("GitHub returned an invalid check id"); - return check.id; -} - -async function completeCheck( - context: { repository: string; checkRunId: number }, - token: string, - verdict: RiskEvidenceVerdict, - detailsUrl?: string, -): Promise { - await githubApi(`repos/${context.repository}/check-runs/${context.checkRunId}`, token, { - method: "PATCH", - body: { - status: "completed", - conclusion: verdict.conclusion, - completed_at: new Date().toISOString(), - details_url: detailsUrl, - output: { title: verdict.title, summary: verdict.summary }, - }, - userAgent: "nemoclaw-e2e-risk-gate", - }); -} - -function controllerErrorMessage(error: unknown): string { - const message = error instanceof Error ? error.message : String(error); - const singleLine = message - .replace(/[\r\n\t]+/gu, " ") - .replace(/\s{2,}/gu, " ") - .trim(); - return singleLine.length > MAX_CONTROLLER_ERROR_CHARS - ? `${singleLine.slice(0, MAX_CONTROLLER_ERROR_CHARS - 3)}...` - : singleLine; -} - -async function completeNeutralAfterControllerError( - context: { repository: string; checkRunId: number }, - token: string, - title: string, - options: { error: unknown; detailsUrl?: string }, -): Promise { - const reason = controllerErrorMessage(options.error).replace(/`/gu, "'"); - try { - await completeCheck( - context, - token, - { - conclusion: "neutral", - title, - summary: - "The shadow controller could not produce complete, trustworthy evidence." + - `\n\nController error: \`${reason}\``, - }, - options.detailsUrl, - ); - return true; - } catch (error) { - console.error( - `failed to close shadow check after controller error: ${error instanceof Error ? error.message : String(error)}`, - ); - return false; - } -} - -export function changedFilesBetween( - baseSha: string, - commitSha: string, - workspace = process.cwd(), -): string[] { - if (!SHA_PATTERN.test(baseSha) || !SHA_PATTERN.test(commitSha)) { - throw new Error("base and tested commits must be lowercase 40-character SHAs"); - } - const checkedOutSha = execFileSync("git", ["rev-parse", "--verify", "HEAD"], { - cwd: workspace, - encoding: "utf8", - stdio: ["ignore", "pipe", "pipe"], - }).trim(); - if (checkedOutSha !== commitSha) { - throw new Error("trusted controller checkout does not match the tested commit"); - } - const output = execFileSync( - "git", - ["diff", "--no-renames", "--name-only", "-z", baseSha, commitSha], - { - cwd: workspace, - encoding: "utf8", - maxBuffer: 16 * 1024 * 1024, - stdio: ["ignore", "pipe", "pipe"], - }, - ); - const files = output.split("\0").filter(Boolean); - if (files.length > 5000) throw new Error("post-merge risk plan exceeds 5000 changed files"); - if (files.some((file) => file.startsWith("/") || file.split("/").includes(".."))) { - throw new Error("post-merge diff contains an unsafe repository path"); - } - return files; -} - -export function expectedRiskSignalShards( - jobIds: readonly string[], - workflowPath = ".github/workflows/e2e.yaml", -): Record { - const workflow = YAML.parse(fs.readFileSync(workflowPath, "utf8")) as unknown; - const jobs = isObjectRecord(workflow) && isObjectRecord(workflow.jobs) ? workflow.jobs : {}; - return Object.fromEntries( - jobIds.map((jobId) => { - const job = isObjectRecord(jobs[jobId]) ? jobs[jobId] : {}; - const strategy = isObjectRecord(job.strategy) ? job.strategy : {}; - const matrix = isObjectRecord(strategy.matrix) ? strategy.matrix : null; - let shards = ["default"]; - if (matrix) { - const keys = Object.keys(matrix); - if (keys.length === 1 && Array.isArray(matrix.agent)) { - shards = matrix.agent.filter((value): value is string => typeof value === "string"); - if (shards.length !== matrix.agent.length) { - throw new Error(`${jobId} risk matrix agent values must be strings`); - } - } else if (keys.length === 1 && Array.isArray(matrix.include)) { - shards = matrix.include.map((entry) => { - if (!isObjectRecord(entry) || typeof entry.agent !== "string") { - throw new Error(`${jobId} risk matrix include entries must name an agent`); - } - return entry.agent; - }); - } else { - throw new Error(`${jobId} uses an unsupported risk-evidence matrix`); - } - } - if ( - shards.length === 0 || - new Set(shards).size !== shards.length || - shards.some((shard) => !SHARD_PATTERN.test(shard)) - ) { - throw new Error(`${jobId} risk evidence shards must be unique safe identifiers`); - } - return [jobId, shards]; - }), - ); -} - -export function validateWorkflowDispatchDetails( - value: unknown, - repository: string, -): WorkflowDispatchDetails { - if (!isObjectRecord(value)) { - throw new Error("GitHub returned invalid workflow dispatch details"); - } - const runId = value.workflow_run_id; - if (!Number.isSafeInteger(runId) || (runId as number) < 1) { - throw new Error("GitHub returned an invalid dispatched workflow run id"); - } - const expectedApiUrl = `https://api.github.com/repos/${repository}/actions/runs/${runId}`; - const expectedHtmlUrl = `https://github.com/${repository}/actions/runs/${runId}`; - if (value.run_url !== expectedApiUrl || value.html_url !== expectedHtmlUrl) { - throw new Error("GitHub returned mismatched workflow dispatch URLs"); - } - return value as WorkflowDispatchDetails; -} - -function diagnosticValue(value: unknown): string { - const serialized = JSON.stringify(value) ?? String(value); - return serialized.length > 256 ? `${serialized.slice(0, 253)}...` : serialized; -} - -export function assertCorrelatedWorkflowRun( - child: WorkflowRun, - identity: WorkflowRunIdentity, -): void { - const childRunUrl = `https://github.com/${identity.repository}/actions/runs/${identity.childRunId}`; - const mismatches: string[] = []; - const requireEqual = (field: string, expected: unknown, actual: unknown): void => { - if (actual !== expected) { - mismatches.push( - `${field} expected=${diagnosticValue(expected)} actual=${diagnosticValue(actual)}`, - ); - } - }; - - requireEqual("id", identity.childRunId, child.id); - requireEqual("path", E2E_WORKFLOW_PATH, child.path); - requireEqual("event", "workflow_dispatch", child.event); - requireEqual("html_url", childRunUrl, child.html_url); - requireEqual("display_title", `E2E risk ${identity.correlationId}`, child.display_title); - if (!SHA_PATTERN.test(child.head_sha)) { - mismatches.push( - `head_sha expected="40 lowercase hexadecimal characters" actual=${diagnosticValue(child.head_sha)}`, - ); - } - if (!Number.isSafeInteger(child.workflow_id) || child.workflow_id < 1) { - mismatches.push( - `workflow_id expected="positive safe integer" actual=${diagnosticValue(child.workflow_id)}`, - ); - } - if (mismatches.length > 0) { - throw new Error( - `correlated E2E workflow identity mismatch: ${mismatches.join("; ")}; observed run_name=${diagnosticValue(child.name)} workflow_id=${diagnosticValue(child.workflow_id)}`, - ); - } -} - -export async function dispatchRiskWorkflow(options: { - repository: string; - token: string; - jobs: readonly string[]; - commitSha: string; - planHash: string; - correlationId: string; -}): Promise { - if ( - !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(options.repository) || - !options.token || - options.jobs.length < 1 || - options.jobs.length > 3 || - new Set(options.jobs).size !== options.jobs.length || - options.jobs.some((job) => !JOB_PATTERN.test(job)) || - !SHA_PATTERN.test(options.commitSha) || - !HASH_PATTERN.test(options.planHash) || - !CORRELATION_PATTERN.test(options.correlationId) - ) { - throw new Error("risk workflow dispatch inputs are invalid"); - } - const details = await githubApi( - `repos/${options.repository}/actions/workflows/${E2E_WORKFLOW}/dispatches`, - options.token, - { - method: "POST", - body: { - ref: "main", - inputs: { - jobs: options.jobs.join(","), - checkout_sha: options.commitSha, - risk_plan_hash: options.planHash, - risk_correlation: options.correlationId, - risk_shadow: "true", - }, - // GitHub REST 2022-11-28 otherwise returns no run identity. - return_run_details: true, - }, - userAgent: "nemoclaw-e2e-risk-gate", - }, - ); - return validateWorkflowDispatchDetails(details, options.repository).workflow_run_id; -} - -async function start(options: { - baseSha: string; - commitSha: string; - planPath: string; - statePath: string; -}): Promise { - const token = process.env.GITHUB_TOKEN ?? ""; - const repository = process.env.GITHUB_REPOSITORY ?? ""; - if (!token || !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository)) { - throw new Error("GITHUB_TOKEN and a safe GITHUB_REPOSITORY are required"); - } - assertTrustedMainPush({ - eventName: process.env.GITHUB_EVENT_NAME, - ref: process.env.GITHUB_REF, - sha: process.env.GITHUB_SHA, - commitSha: options.commitSha, - }); - - // The inventory and controller are both read from the exact trusted main - // commit. A second copied allowlist would drift without adding a trust - // boundary: compromising this inventory already means compromising main. - const allowedJobs = new Set(readFreeStandingJobsInventory().allowedJobs); - const plan = validateRiskPlan( - buildRiskPlan({ - headSha: options.commitSha, - changedFiles: changedFilesBetween(options.baseSha, options.commitSha), - }), - allowedJobs, - ); - writePrivateRegularFile(options.planPath, `${JSON.stringify(plan, null, 2)}\n`); - - const checkRunId = await createCheck( - repository, - token, - options.commitSha, - "Post-merge risk-selected E2E is being dispatched", - `Plan ${plan.planHash.slice(0, 12)} selected ${plan.automaticJobs.length} automatic job(s).`, - ); - appendOutput("check_id", String(checkRunId)); - try { - const expectedShards = expectedRiskSignalShards(plan.automaticJobs); - if (plan.requiredJobs.length === 0) { - await completeCheck({ repository, checkRunId }, token, { - conclusion: "success", - title: "No post-merge runtime E2E required", - summary: "The deterministic risk plan matched no runtime regression family.", - }); - appendOutput("dispatched", "false"); - appendOutput("finalized", "true"); - console.log( - `Completed shadow check without dispatch: plan=${plan.planHash.slice(0, 12)} selected_jobs=none`, - ); - return; - } - - const correlationId = randomUUID(); - if (!CORRELATION_PATTERN.test(correlationId)) { - throw new Error("generated correlation id is invalid"); - } - const childRunId = await dispatchRiskWorkflow({ - repository, - token, - jobs: plan.automaticJobs, - commitSha: options.commitSha, - planHash: plan.planHash, - correlationId, - }); - const state: RiskGateState = { - version: 1, - commitSha: options.commitSha, - planHash: plan.planHash, - correlationId, - expectedJobs: plan.automaticJobs, - expectedShards, - requiresManualExpansion: plan.requiresManualExpansion, - }; - const serializedState = `${JSON.stringify(state, null, 2)}\n`; - writePrivateRegularFile(options.statePath, serializedState); - appendOutput("state_hash", sha256(serializedState)); - appendOutput("run_id", String(childRunId)); - appendOutput("dispatched", "true"); - console.log( - `Dispatched risk-selected E2E run ${childRunId}: plan=${plan.planHash.slice(0, 12)} selected_jobs=${plan.automaticJobs.join(",")} url=https://github.com/${repository}/actions/runs/${childRunId}`, - ); - } catch (error) { - const finalized = await completeNeutralAfterControllerError( - { repository, checkRunId }, - token, - "Risk-selected E2E could not be dispatched", - { error }, - ); - if (finalized) appendOutput("finalized", "true"); - throw error; - } -} - -export function findSignalFiles( - root: string, - limits: { - maxDepth: number; - maxEntries: number; - maxSignalFiles: number; - } = DEFAULT_EVIDENCE_LIMITS, -): string[] { - if (!fs.existsSync(root)) return []; - if ( - !Number.isSafeInteger(limits.maxDepth) || - limits.maxDepth < 0 || - !Number.isSafeInteger(limits.maxEntries) || - limits.maxEntries < 1 || - !Number.isSafeInteger(limits.maxSignalFiles) || - limits.maxSignalFiles < 1 - ) { - throw new Error("risk evidence traversal limits are invalid"); - } - const rootStat = fs.lstatSync(root); - if (!rootStat.isDirectory() || rootStat.isSymbolicLink()) { - throw new Error("risk evidence root must be a directory, not a symlink"); - } - const files: string[] = []; - let entriesVisited = 0; - const visit = (directory: string, depth: number): void => { - const handle = fs.opendirSync(directory); - try { - let entry = handle.readSync(); - while (entry !== null) { - entriesVisited += 1; - if (entriesVisited > limits.maxEntries) { - throw new Error("risk evidence exceeds the entry limit"); - } - const full = path.join(directory, entry.name); - if (entry.isSymbolicLink()) throw new Error("risk evidence must not contain symlinks"); - if (entry.isDirectory()) { - if (depth >= limits.maxDepth) throw new Error("risk evidence exceeds the depth limit"); - visit(full, depth + 1); - } else if (entry.isFile() && entry.name === "risk-signal.json") { - files.push(full); - if (files.length > limits.maxSignalFiles) { - throw new Error("risk evidence exceeds the signal-file limit"); - } - } - entry = handle.readSync(); - } - } finally { - handle.closeSync(); - } - }; - visit(root, 0); - return files.sort((left, right) => left.localeCompare(right)); -} - -export async function finishRiskGate(options: { - statePath: string; - stateHash: string; - evidencePath: string; - checkRunId: number; - childRunId: number; -}): Promise { - const token = process.env.GITHUB_TOKEN ?? ""; - const repository = process.env.GITHUB_REPOSITORY ?? ""; - if (!token || !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository)) { - throw new Error("GITHUB_TOKEN and a safe GITHUB_REPOSITORY are required"); - } - const { checkRunId, childRunId } = options; - const childRunUrl = `https://github.com/${repository}/actions/runs/${childRunId}`; - const context = { repository, checkRunId }; - try { - if (!HASH_PATTERN.test(options.stateHash)) throw new Error("controller state hash is invalid"); - const serializedState = readPrivateRegularFile(options.statePath, { - maxBytes: MAX_PLAN_BYTES, - })!; - if (sha256(serializedState) !== options.stateHash) { - throw new Error("controller state changed after E2E dispatch"); - } - const state = validateRiskGateState(JSON.parse(serializedState)); - const child = await githubApi( - `repos/${repository}/actions/runs/${childRunId}`, - token, - { userAgent: "nemoclaw-e2e-risk-gate" }, - ); - assertCorrelatedWorkflowRun(child, { - childRunId, - correlationId: state.correlationId, - repository, - }); - console.log( - `Verified correlated E2E run ${childRunId}: conclusion=${child.conclusion ?? "none"} url=${childRunUrl}`, - ); - const signals = - child.conclusion === "success" - ? findSignalFiles(options.evidencePath).map((file) => - validateSignal(readRegularJson(file), state), - ) - : []; - const verdict = classifyRiskEvidence({ - workflowConclusion: child.conclusion, - expectedJobs: state.expectedJobs, - expectedShards: state.expectedShards, - signals, - requiresManualExpansion: state.requiresManualExpansion, - }); - await completeCheck(context, token, verdict, childRunUrl); - console.log(`Completed shadow check: conclusion=${verdict.conclusion} title=${verdict.title}`); - } catch (error) { - const finalized = await completeNeutralAfterControllerError( - context, - token, - "Risk-selected E2E evidence could not be verified", - { error, detailsUrl: childRunUrl }, - ); - if (finalized) appendOutput("finalized", "true"); - throw error; - } -} - -async function abandon(checkRunId: number): Promise { - const token = process.env.GITHUB_TOKEN ?? ""; - const repository = process.env.GITHUB_REPOSITORY ?? ""; - if (!token || !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository)) { - throw new Error("GITHUB_TOKEN and a safe GITHUB_REPOSITORY are required"); - } - await completeCheck({ repository, checkRunId }, token, { - conclusion: "neutral", - title: "Risk-selected E2E controller stopped early", - summary: - "The shadow controller stopped before it could produce complete evidence. Inspect the controller workflow for details.", - }); -} - -function reportControllerError(error: unknown): void { - const message = controllerErrorMessage(error); - console.error(message); - if (process.env.GITHUB_ACTIONS === "true") { - const escaped = message.replace(/%/gu, "%25").replace(/\r/gu, "%0D").replace(/\n/gu, "%0A"); - console.error(`::error title=Post-merge E2E risk gate controller failed::${escaped}`); - } -} - -async function main(): Promise { - const command = parseControllerCommand(process.argv.slice(2)); - if (command.mode === "start") { - await start({ - baseSha: command.baseSha, - commitSha: command.commitSha, - planPath: command.planPath, - statePath: command.statePath, - }); - return; - } - if (command.mode === "finish") { - await finishRiskGate({ - statePath: command.statePath, - stateHash: command.stateHash, - evidencePath: command.evidencePath, - checkRunId: command.checkRunId, - childRunId: command.childRunId, - }); - return; - } - if (command.mode === "abandon") { - await abandon(command.checkRunId); - return; - } -} - -if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - main().catch((error: unknown) => { - reportControllerError(error); - process.exit(1); - }); -} diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index 35dd668a698..3eb3d16d466 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -6,6 +6,7 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import YAML from "yaml"; +import { RISK_RULES } from "../advisors/risk-plan.mts"; const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); const DEFAULT_WORKFLOW_PATH = join(REPO_ROOT, ".github", "workflows", "e2e.yaml"); @@ -14,6 +15,8 @@ const META_JOBS = new Set(["report-to-pr", "scorecard"]); const FULL_SHA_ACTION = /^[^\s@]+@[0-9a-f]{40}$/u; const GITHUB_SCRIPT_NODE24_ACTION = "actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3"; +const REQUIRED_LIVE_REPORTER = "test/e2e/risk-signal-reporter.ts"; +const E2E_ARTIFACT_ACTION = "NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@"; const ISSUE_API_REFERENCE = /\bgithub\.rest\.issues\b/u; const ISSUE_MUTATION_BEYOND_COMMENT = /github\.rest\.issues\.(?:addAssignees|addLabels|create|deleteComment|lock|removeAssignees|removeLabel|setLabels|unlock|update|updateComment)\s*\(/u; @@ -44,8 +47,14 @@ type WorkflowJob = { }; export type OperationsWorkflow = { + concurrency?: { + "cancel-in-progress"?: unknown; + group?: unknown; + }; + env?: Record; jobs: Record; permissions?: WorkflowPermissions; + "run-name"?: unknown; on?: { workflow_dispatch?: { inputs?: Record>; @@ -101,6 +110,128 @@ function requireNode24GithubScript(errors: string[], step: WorkflowStep, owner: } } +function validateRequiredLiveDispatch(errors: string[], workflow: OperationsWorkflow): void { + const inputs = workflow.on?.workflow_dispatch?.inputs ?? {}; + for (const name of ["jobs", "pr_number", "checkout_sha", "plan_hash", "correlation_id"]) { + const input = inputs[name]; + if (input?.type !== "string" || input.default !== "") { + errors.push(`workflow_dispatch ${name} must be an optional string with an empty default`); + } + } + const expectedEnvironment = { + NEMOCLAW_E2E_CORRELATION_ID: "${{ inputs.correlation_id }}", + NEMOCLAW_E2E_EXPECTED_SHA: "${{ inputs.checkout_sha }}", + NEMOCLAW_E2E_PLAN_HASH: "${{ inputs.plan_hash }}", + NEMOCLAW_E2E_SHARD: "default", + }; + for (const [name, value] of Object.entries(expectedEnvironment)) { + if (workflow.env?.[name] !== value) { + errors.push(`E2E workflow must bind ${name} to required-live metadata`); + } + } + const runName = String(workflow["run-name"] ?? ""); + for (const fragment of ["inputs.checkout_sha", "inputs.pr_number", "inputs.correlation_id"]) { + if (!runName.includes(fragment)) errors.push(`required-live run name must include ${fragment}`); + } + const concurrencyGroup = String(workflow.concurrency?.group ?? ""); + if ( + !concurrencyGroup.includes("inputs.checkout_sha") || + !concurrencyGroup.includes("inputs.pr_number") + ) { + errors.push("required-live concurrency must be scoped to its pull request"); + } + if (workflow.concurrency?.["cancel-in-progress"] !== "${{ inputs.checkout_sha != '' }}") { + errors.push("required-live concurrency must cancel obsolete pull request runs"); + } + + const matrixJob = workflow.jobs["generate-matrix"] ?? {}; + const steps = matrixJob.steps ?? []; + const validationIndex = steps.findIndex( + (step) => step.name === "Validate required-live dispatch", + ); + const prepareIndex = steps.findIndex((step) => step.name === "Prepare E2E workspace"); + const validation = validationIndex >= 0 ? steps[validationIndex] : {}; + if (validation.if !== "${{ inputs.checkout_sha != '' }}") { + errors.push("required-live validation must be activated only by checkout_sha"); + } + if (validationIndex < 0 || prepareIndex < 0 || validationIndex >= prepareIndex) { + errors.push("required-live validation must run before workspace preparation"); + } + const expectedStepEnvironment = { + CHECKOUT_SHA: "${{ inputs.checkout_sha }}", + JOBS: "${{ inputs.jobs }}", + PLAN_HASH: "${{ inputs.plan_hash }}", + PR_NUMBER: "${{ inputs.pr_number }}", + CORRELATION_ID: "${{ inputs.correlation_id }}", + TARGETS: "${{ inputs.targets }}", + }; + for (const [name, value] of Object.entries(expectedStepEnvironment)) { + if (validation.env?.[name] !== value) { + errors.push(`required-live validation must bind ${name}`); + } + } + const validationScript = String(validation.run ?? ""); + for (const fragment of [ + '"$WORKFLOW_EVENT" == "workflow_dispatch"', + '"$WORKFLOW_REF" == "refs/heads/main"', + '"$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA"', + '"$PR_NUMBER" =~ ^[1-9][0-9]*$', + '[[ -n "$JOBS" && -z "$TARGETS" ]]', + "https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}", + "'.state'", + "'.head.repo.full_name // \"\"'", + "'.head.sha'", + ]) { + if (!validationScript.includes(fragment)) { + errors.push(`required-live validation must retain ${fragment}`); + } + } + + for (const [jobName, job] of Object.entries(workflow.jobs)) { + for (const step of job.steps ?? []) { + if ( + step.uses?.startsWith("actions/checkout@") && + step.with?.ref !== "${{ inputs.checkout_sha || github.sha }}" + ) { + errors.push(`${jobName} checkout must use the selected immutable commit`); + } + } + } +} + +function validateRequiredLiveEvidenceProducers( + errors: string[], + workflow: OperationsWorkflow, +): void { + const requiredJobs = new Set(RISK_RULES.flatMap((rule) => rule.requiredJobs)); + for (const jobId of requiredJobs) { + const job = workflow.jobs[jobId]; + if (!job) { + errors.push(`required-live plan job is missing from E2E workflow: ${jobId}`); + continue; + } + if (job.env?.E2E_JOB !== "1" || job.env?.E2E_TARGET_ID !== jobId) { + errors.push(`${jobId} must expose matching required-live job identity`); + } + if (typeof job.env?.E2E_ARTIFACT_DIR !== "string" || !job.env.E2E_ARTIFACT_DIR) { + errors.push(`${jobId} must expose a required-live artifact directory`); + } + const vitestSteps = (job.steps ?? []).filter((step) => + String(step.run ?? "").includes("npx vitest"), + ); + if ( + vitestSteps.length === 0 || + vitestSteps.some((step) => !String(step.run).includes(REQUIRED_LIVE_REPORTER)) + ) { + errors.push(`${jobId} must attach the required-live reporter to every Vitest invocation`); + } + const uploads = (job.steps ?? []).filter((step) => step.uses?.startsWith(E2E_ARTIFACT_ACTION)); + if (uploads.length !== 1 || uploads[0]?.if !== "always()") { + errors.push(`${jobId} must always upload one required-live evidence artifact`); + } + } +} + function validateAggregation(errors: string[], workflow: OperationsWorkflow): void { const executionJobs = Object.keys(workflow.jobs).filter((name) => !META_JOBS.has(name)); const reportNeeds = needs(workflow.jobs["report-to-pr"] ?? {}); @@ -150,7 +281,7 @@ function validateIssueRoutingRetirement(errors: string[], workflow: OperationsWo } if ( job.if !== - "${{ always() && github.event_name == 'workflow_dispatch' && !inputs.risk_shadow }}" + "${{ always() && github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '' }}" ) { errors.push("report-to-pr must run only for manual workflow dispatches"); } @@ -215,7 +346,7 @@ function validateScorecard(errors: string[], workflow: OperationsWorkflow): void const permissions = permissionMap(job.permissions); if ( job.if !== - "${{ always() && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && !inputs.risk_shadow)) }}" + "${{ always() && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '')) }}" ) { errors.push("scorecard must run after scheduled and manual E2E executions"); } @@ -415,6 +546,8 @@ export function validateE2eOperationsWorkflow( advisorPath = DEFAULT_ADVISOR_PATH, ): string[] { const errors: string[] = []; + validateRequiredLiveDispatch(errors, workflow); + validateRequiredLiveEvidenceProducers(errors, workflow); validateAggregation(errors, workflow); validateIssueRoutingRetirement(errors, workflow); validateScorecard(errors, workflow); diff --git a/tools/e2e-advisor/private-file.ts b/tools/e2e/private-file.ts similarity index 100% rename from tools/e2e-advisor/private-file.ts rename to tools/e2e/private-file.ts diff --git a/tools/e2e/required-live.mts b/tools/e2e/required-live.mts new file mode 100755 index 00000000000..3d18e869b02 --- /dev/null +++ b/tools/e2e/required-live.mts @@ -0,0 +1,1261 @@ +#!/usr/bin/env node + +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash, randomUUID } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +import YAML from "yaml"; + +import { githubApi, githubRestPaginated } from "../advisors/github.mts"; +import { parseArgs } from "../advisors/io.mts"; +import { + buildRiskPlan, + RISK_PLAN_VERSION, + type RiskPlan, + riskPlanRequiredJobIds, +} from "../advisors/risk-plan.mts"; +import { readPrivateRegularFile, writePrivateRegularFile } from "./private-file.ts"; +import type { E2eRiskSignal } from "./risk-signal.ts"; +import { readFreeStandingJobsInventory } from "./workflow-boundary.mts"; + +const E2E_WORKFLOW = "e2e.yaml"; +const E2E_WORKFLOW_PATH = `.github/workflows/${E2E_WORKFLOW}`; +const CHECK_NAME = "E2E / Required Live"; +const USER_AGENT = "nemoclaw-required-live"; +const SHA_PATTERN = /^[a-f0-9]{40}$/u; +const HASH_PATTERN = /^[a-f0-9]{64}$/u; +const REPOSITORY_PATTERN = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u; +const JOB_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]*$/u; +const SHARD_PATTERN = /^(?:default|[A-Za-z0-9][A-Za-z0-9_-]*)$/u; +const CORRELATION_PATTERN = + /^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/u; +const RUN_REASONS = new Set(["passed", "failed", "interrupted"]); +const MAX_PLAN_BYTES = 1024 * 1024; +const MAX_CONTROLLER_ERROR_CHARS = 512; +const MAX_PR_FILES = 3000; +const MAX_ACTIVE_RUN_PAGES = 10; +const EVIDENCE_LIMITS = { + maxDepth: 8, + maxEntries: 4096, +} as const; + +type ControllerPaths = { + planPath: string; + statePath: string; + evidencePath: string; +}; + +export type ControllerCommand = + | ({ + mode: "start"; + headSha: string; + headRepository: string; + headBranch: string; + workflowSha: string; + ciConclusion: string; + } & ControllerPaths) + | ({ + mode: "finish"; + checkRunId: number; + childRunId: number; + stateHash: string; + } & ControllerPaths) + | { mode: "abandon"; checkRunId: number; childRunId?: number } + | { mode: "cancel"; prNumber: number }; + +type CheckConclusion = "success" | "failure"; + +export type PullRequest = { + number: number; + state: string; + changed_files: number; + head: { ref: string; sha: string; repo: { full_name: string } | null }; + base: { sha: string; repo: { full_name: string } }; +}; + +type PullRequestListItem = Omit; + +type PullRequestFile = { filename: string; previous_filename?: string }; + +type WorkflowRun = { + id: number; + name: string; + path: string; + workflow_id: number; + event: string; + head_sha: string; + status: string; + conclusion: string | null; + display_title: string; + html_url: string; +}; + +type WorkflowRunsResponse = { workflow_runs: WorkflowRun[] }; +type CheckRun = { id: number }; +type GitReference = { ref: string; object: { type: string; sha: string } }; + +type WorkflowDispatchDetails = { + workflow_run_id: number; + run_url: string; + html_url: string; +}; + +type WorkflowRunIdentity = { + childRunId: number; + correlationId: string; + prNumber: number; + repository: string; + workflowSha: string; +}; + +export type RequiredLiveState = { + version: 1; + commitSha: string; + workflowSha: string; + planHash: string; + correlationId: string; + prNumber: number; + expectedJobs: string[]; + expectedShards: Record; +}; + +export type RequiredLiveVerdict = { + conclusion: CheckConclusion; + title: string; + summary: string; +}; + +function isObjectRecord(value: unknown): value is Record { + return !!value && typeof value === "object" && !Array.isArray(value); +} + +function requiredArgument(value: string | undefined, name: string): string { + if (!value) throw new Error(`--${name} is required`); + return value; +} + +function parsePositiveId(value: string, name: string): number { + if (!/^[1-9][0-9]*$/u.test(value)) throw new Error(`${name} must be a positive integer`); + const parsed = Number(value); + if (!Number.isSafeInteger(parsed)) throw new Error(`${name} exceeds the safe integer range`); + return parsed; +} + +function parseHash(value: string | undefined, name: string): string { + const parsed = requiredArgument(value, name); + if (!HASH_PATTERN.test(parsed)) throw new Error(`--${name} must be a lowercase SHA-256 hash`); + return parsed; +} + +function sha256(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} + +function assertRepository(value: string, name: string): void { + if (!REPOSITORY_PATTERN.test(value)) throw new Error(`${name} must be an owner/repository name`); +} + +function assertBranch(value: string): void { + if ( + value.length > 255 || + /[\u0000-\u001f\u007f\\]/u.test(value) || + value.startsWith("/") || + value.endsWith("/") || + value.includes("..") || + value.includes("@{") + ) { + throw new Error("head branch is invalid"); + } +} + +function assertRepositoryPath(value: string): void { + if ( + value.length === 0 || + value.length > 4096 || + value.startsWith("/") || + value.includes("\\") || + /[\u0000\r\n]/u.test(value) || + value.split("/").some((part) => part === "." || part === "..") + ) { + throw new Error("pull request files contain an unsafe repository path"); + } +} + +function tokenAndRepository(): { token: string; repository: string } { + const token = process.env.GITHUB_TOKEN ?? ""; + const repository = process.env.GITHUB_REPOSITORY ?? ""; + if (!token) throw new Error("GITHUB_TOKEN is required"); + assertRepository(repository, "GITHUB_REPOSITORY"); + return { token, repository }; +} + +export function privateControllerPaths(workDir: string): ControllerPaths { + const resolved = path.resolve(workDir); + const stat = fs.lstatSync(resolved); + const currentUid = typeof process.getuid === "function" ? process.getuid() : null; + if ( + resolved !== workDir || + !stat.isDirectory() || + stat.isSymbolicLink() || + (stat.mode & 0o077) !== 0 || + (currentUid !== null && stat.uid !== currentUid) + ) { + throw new Error("--work-dir must be an owned private absolute directory"); + } + return { + planPath: path.join(resolved, "required-live-plan.json"), + statePath: path.join(resolved, "required-live-state.json"), + evidencePath: path.join(resolved, "evidence"), + }; +} + +export function parseControllerCommand(argv: string[]): ControllerCommand { + const args = parseArgs(argv); + if (args.mode === "start") { + return { + mode: "start", + headSha: requiredArgument(args.head, "head"), + headRepository: requiredArgument(args.headRepo, "head-repo"), + headBranch: requiredArgument(args.headBranch, "head-branch"), + workflowSha: requiredArgument(args.workflowSha, "workflow-sha"), + ciConclusion: requiredArgument(args.ciConclusion, "ci-conclusion"), + ...privateControllerPaths(requiredArgument(args.workDir, "work-dir")), + }; + } + if (args.mode === "finish") { + return { + mode: "finish", + ...privateControllerPaths(requiredArgument(args.workDir, "work-dir")), + checkRunId: parsePositiveId(requiredArgument(args.checkId, "check-id"), "--check-id"), + childRunId: parsePositiveId(requiredArgument(args.runId, "run-id"), "--run-id"), + stateHash: parseHash(args.stateHash, "state-hash"), + }; + } + if (args.mode === "abandon") { + return { + mode: "abandon", + checkRunId: parsePositiveId(requiredArgument(args.checkId, "check-id"), "--check-id"), + childRunId: args.runId ? parsePositiveId(args.runId, "--run-id") : undefined, + }; + } + if (args.mode === "cancel") { + return { + mode: "cancel", + prNumber: parsePositiveId(requiredArgument(args.pr, "pr"), "--pr"), + }; + } + throw new Error("--mode must be start, finish, abandon, or cancel"); +} + +function readRegularJson(file: string, maxBytes = MAX_PLAN_BYTES): unknown { + return JSON.parse(readPrivateRegularFile(file, { maxBytes })!); +} + +export function validateRequiredLiveState(value: unknown): RequiredLiveState { + if (!isObjectRecord(value) || value.version !== 1) { + throw new Error("invalid required-live state version"); + } + if (typeof value.commitSha !== "string" || !SHA_PATTERN.test(value.commitSha)) { + throw new Error("required-live state commit SHA is invalid"); + } + if (typeof value.workflowSha !== "string" || !SHA_PATTERN.test(value.workflowSha)) { + throw new Error("required-live state workflow SHA is invalid"); + } + if (typeof value.planHash !== "string" || !HASH_PATTERN.test(value.planHash)) { + throw new Error("required-live state plan hash is invalid"); + } + if (typeof value.correlationId !== "string" || !CORRELATION_PATTERN.test(value.correlationId)) { + throw new Error("required-live state correlation id is invalid"); + } + if (!Number.isSafeInteger(value.prNumber) || (value.prNumber as number) < 1) { + throw new Error("required-live state PR number is invalid"); + } + if ( + !Array.isArray(value.expectedJobs) || + value.expectedJobs.length < 1 || + !value.expectedJobs.every((job) => typeof job === "string" && JOB_PATTERN.test(job)) || + new Set(value.expectedJobs).size !== value.expectedJobs.length + ) { + throw new Error("required-live state expected jobs are invalid"); + } + if (!isObjectRecord(value.expectedShards)) { + throw new Error("required-live state shards are invalid"); + } + const shardJobs = Object.keys(value.expectedShards).sort(); + if (JSON.stringify(shardJobs) !== JSON.stringify([...value.expectedJobs].sort())) { + throw new Error("required-live state shard jobs do not match expected jobs"); + } + for (const job of value.expectedJobs) { + const shards = value.expectedShards[job]; + if ( + !Array.isArray(shards) || + shards.length < 1 || + new Set(shards).size !== shards.length || + !shards.every((shard) => typeof shard === "string" && SHARD_PATTERN.test(shard)) + ) { + throw new Error(`required-live state shards are invalid for ${job}`); + } + } + return value as RequiredLiveState; +} + +export function validateRiskPlan(value: unknown, allowedJobs: ReadonlySet): RiskPlan { + if (!isObjectRecord(value)) throw new Error("risk plan must be an object"); + if (value.version !== RISK_PLAN_VERSION) throw new Error("unsupported risk-plan version"); + if (typeof value.headSha !== "string" || !SHA_PATTERN.test(value.headSha)) { + throw new Error("risk plan headSha must be a lowercase 40-character SHA"); + } + if ( + !Array.isArray(value.changedFiles) || + !value.changedFiles.every((file) => typeof file === "string") + ) { + throw new Error("risk plan changedFiles must be strings"); + } + for (const file of value.changedFiles) assertRepositoryPath(file as string); + const rebuilt = buildRiskPlan({ + headSha: value.headSha, + changedFiles: value.changedFiles as string[], + }); + if (JSON.stringify(value) !== JSON.stringify(rebuilt)) { + throw new Error("risk plan does not match its deterministic hash and inputs"); + } + if (!HASH_PATTERN.test(rebuilt.planHash)) throw new Error("risk plan hash is invalid"); + const selectedJobs = riskPlanRequiredJobIds(rebuilt); + if (new Set(selectedJobs).size !== selectedJobs.length) { + throw new Error("risk plan required jobs must be unique"); + } + for (const job of selectedJobs) { + if (!JOB_PATTERN.test(job) || !allowedJobs.has(job)) { + throw new Error(`risk plan names unknown E2E job: ${job}`); + } + } + return rebuilt; +} + +export function validateSignal( + value: unknown, + state: Pick< + RequiredLiveState, + "commitSha" | "planHash" | "correlationId" | "expectedJobs" | "expectedShards" + >, +): E2eRiskSignal { + if (!isObjectRecord(value) || value.version !== 1) { + throw new Error("invalid E2E signal version"); + } + const signal = value as E2eRiskSignal; + if (!state.expectedJobs.includes(signal.jobId)) throw new Error("E2E signal job is unexpected"); + if (!state.expectedShards[signal.jobId]?.includes(signal.shardId)) { + throw new Error("E2E signal shard is unexpected"); + } + if (signal.expectedSha !== state.commitSha) throw new Error("E2E signal SHA mismatch"); + if (signal.testedSha !== state.commitSha) throw new Error("E2E signal tested SHA mismatch"); + if (signal.planHash !== state.planHash) throw new Error("E2E signal plan hash mismatch"); + if (signal.correlationId !== state.correlationId) { + throw new Error("E2E signal correlation mismatch"); + } + for (const key of ["passed", "failed", "skipped", "pending", "unhandledErrors"] as const) { + if (!Number.isSafeInteger(signal[key]) || signal[key] < 0) { + throw new Error(`E2E signal ${key} must be a non-negative integer`); + } + } + if (!RUN_REASONS.has(signal.runReason)) { + throw new Error("E2E signal runReason is invalid"); + } + return signal; +} + +export function classifyRequiredLiveEvidence(options: { + workflowConclusion: string | null; + expectedJobs: readonly string[]; + expectedShards: Readonly>; + signals: readonly E2eRiskSignal[]; +}): RequiredLiveVerdict { + if (options.workflowConclusion !== "success") { + return { + conclusion: "failure", + title: "Required live E2E did not complete successfully", + summary: `The correlated E2E workflow concluded ${options.workflowConclusion ?? "without a result"}.`, + }; + } + const expectedEvidence = options.expectedJobs.flatMap((job) => + (options.expectedShards[job] ?? []).map((shard) => `${job}:${shard}`), + ); + if ( + options.expectedJobs.length === 0 || + options.expectedJobs.some((job) => (options.expectedShards[job]?.length ?? 0) === 0) + ) { + return { + conclusion: "failure", + title: "Required live E2E lacks an evidence policy", + summary: "At least one selected job has no trusted shard policy.", + }; + } + const byJobShard = new Map(); + for (const signal of options.signals) { + const key = `${signal.jobId}:${signal.shardId}`; + if (byJobShard.has(key)) { + return { + conclusion: "failure", + title: "Required live E2E produced duplicate evidence", + summary: `More than one signal was uploaded for ${key}.`, + }; + } + byJobShard.set(key, signal); + } + const missing = expectedEvidence.filter((key) => !byJobShard.has(key)); + if (missing.length > 0) { + return { + conclusion: "failure", + title: "Required live E2E is missing evidence", + summary: `Missing bound signals for: ${missing.join(", ")}.`, + }; + } + const failed = expectedEvidence.filter((key) => { + const signal = byJobShard.get(key)!; + return signal.failed > 0 || signal.unhandledErrors > 0 || signal.runReason === "failed"; + }); + if (failed.length > 0) { + return { + conclusion: "failure", + title: "Required live E2E reported test failures", + summary: `Failing signals: ${failed.join(", ")}.`, + }; + } + const partial = expectedEvidence.filter((key) => { + const signal = byJobShard.get(key)!; + return ( + signal.passed < 1 || signal.skipped > 0 || signal.pending > 0 || signal.runReason !== "passed" + ); + }); + if (partial.length > 0) { + return { + conclusion: "failure", + title: "Required live E2E produced incomplete evidence", + summary: `Incomplete or skipped signals: ${partial.join(", ")}.`, + }; + } + return { + conclusion: "success", + title: "Required live E2E passed", + summary: "Every expected job shard produced a complete, unskipped pass.", + }; +} + +function appendOutput(name: string, value: string): void { + const output = process.env.GITHUB_OUTPUT; + if (!output) return; + if (!/^(?:check_id|dispatched|finalized|run_id|state_hash)$/u.test(name)) { + throw new Error("invalid controller output name"); + } + const validValue = + name === "state_hash" ? HASH_PATTERN.test(value) : /^(?:true|false|[1-9][0-9]*)$/u.test(value); + if (!validValue) throw new Error("invalid controller output value"); + const descriptor = fs.openSync( + output, + fs.constants.O_WRONLY | fs.constants.O_APPEND | (fs.constants.O_NOFOLLOW ?? 0), + ); + try { + if (!fs.fstatSync(descriptor).isFile()) throw new Error("GITHUB_OUTPUT must be a regular file"); + fs.writeFileSync(descriptor, `${name}=${value}\n`, "utf8"); + } finally { + fs.closeSync(descriptor); + } +} + +async function createCheck( + repository: string, + token: string, + headSha: string, + title: string, + summary: string, +): Promise { + const check = await githubApi(`repos/${repository}/check-runs`, token, { + method: "POST", + body: { + name: CHECK_NAME, + head_sha: headSha, + status: "in_progress", + output: { title, summary }, + }, + userAgent: USER_AGENT, + }); + if (!Number.isSafeInteger(check.id) || check.id < 1) { + throw new Error("GitHub returned an invalid check id"); + } + return check.id; +} + +async function completeCheck( + context: { repository: string; checkRunId: number }, + token: string, + verdict: RequiredLiveVerdict, + detailsUrl?: string, +): Promise { + await githubApi(`repos/${context.repository}/check-runs/${context.checkRunId}`, token, { + method: "PATCH", + body: { + status: "completed", + conclusion: verdict.conclusion, + completed_at: new Date().toISOString(), + details_url: detailsUrl, + output: { title: verdict.title, summary: verdict.summary }, + }, + userAgent: USER_AGENT, + }); +} + +async function updateRunningCheck( + context: { repository: string; checkRunId: number }, + token: string, + options: { childRunId: number; jobs: readonly string[]; planHash: string }, +): Promise { + const childRunUrl = `https://github.com/${context.repository}/actions/runs/${options.childRunId}`; + await githubApi(`repos/${context.repository}/check-runs/${context.checkRunId}`, token, { + method: "PATCH", + body: { + status: "in_progress", + details_url: childRunUrl, + output: { + title: `Running ${options.jobs.length} required live E2E ${options.jobs.length === 1 ? "job" : "jobs"}`, + summary: `Plan ${options.planHash} selected: ${options.jobs.join(", ")}.`, + }, + }, + userAgent: USER_AGENT, + }); +} + +function controllerErrorMessage(error: unknown): string { + const message = error instanceof Error ? error.message : String(error); + const singleLine = message + .replace(/[\r\n\t]+/gu, " ") + .replace(/\s{2,}/gu, " ") + .trim(); + return singleLine.length > MAX_CONTROLLER_ERROR_CHARS + ? `${singleLine.slice(0, MAX_CONTROLLER_ERROR_CHARS - 3)}...` + : singleLine; +} + +async function completeFailureAfterControllerError( + context: { repository: string; checkRunId: number }, + token: string, + title: string, + options: { error: unknown; detailsUrl?: string }, +): Promise { + const reason = controllerErrorMessage(options.error).replace(/`/gu, "'"); + try { + await completeCheck( + context, + token, + { + conclusion: "failure", + title, + summary: `The required-live controller could not produce trustworthy evidence.\n\nController error: \`${reason}\``, + }, + options.detailsUrl, + ); + return true; + } catch (error) { + console.error( + `Failed to close required-live check after controller error: ${controllerErrorMessage(error)}`, + ); + return false; + } +} + +function validatePullRequestIdentity(value: unknown): PullRequestListItem { + if ( + !isObjectRecord(value) || + !Number.isSafeInteger(value.number) || + (value.number as number) < 1 + ) { + throw new Error("GitHub returned an invalid pull request number"); + } + if (value.state !== "open") throw new Error("GitHub returned invalid pull request state"); + if (!isObjectRecord(value.head) || !isObjectRecord(value.base)) { + throw new Error("GitHub returned invalid pull request refs"); + } + const head = value.head; + const base = value.base; + if ( + typeof head.ref !== "string" || + typeof head.sha !== "string" || + !SHA_PATTERN.test(head.sha) || + !isObjectRecord(head.repo) || + typeof head.repo.full_name !== "string" || + !REPOSITORY_PATTERN.test(head.repo.full_name) || + typeof base.sha !== "string" || + !SHA_PATTERN.test(base.sha) || + !isObjectRecord(base.repo) || + typeof base.repo.full_name !== "string" || + !REPOSITORY_PATTERN.test(base.repo.full_name) + ) { + throw new Error("GitHub returned invalid pull request identity"); + } + return value as PullRequestListItem; +} + +function validatePullRequest(value: unknown): PullRequest { + const identity = validatePullRequestIdentity(value); + if (!isObjectRecord(value) || !Number.isSafeInteger(value.changed_files)) { + throw new Error("GitHub returned an invalid pull request changed-file count"); + } + return { ...identity, changed_files: value.changed_files as number }; +} + +function pullIdentity(pull: PullRequestListItem): Record { + return { + number: pull.number, + state: pull.state, + headRef: pull.head.ref, + headSha: pull.head.sha, + headRepository: pull.head.repo?.full_name, + baseSha: pull.base.sha, + baseRepository: pull.base.repo.full_name, + }; +} + +export async function resolvePullRequest(options: { + repository: string; + token: string; + headSha: string; + headRepository: string; + headBranch: string; +}): Promise { + assertRepository(options.repository, "repository"); + assertRepository(options.headRepository, "head repository"); + if (!options.token) throw new Error("GitHub token is required"); + if (!SHA_PATTERN.test(options.headSha)) throw new Error("head SHA is invalid"); + assertBranch(options.headBranch); + const owner = options.headRepository.split("/", 1)[0]!; + const query = encodeURIComponent(`${owner}:${options.headBranch}`); + const response = await githubApi( + `repos/${options.repository}/pulls?state=open&head=${query}&per_page=100`, + options.token, + { userAgent: USER_AGENT }, + ); + if (!Array.isArray(response)) throw new Error("GitHub returned an invalid pull request list"); + const matches = response + .map(validatePullRequestIdentity) + .filter( + (pull) => + pull.head.sha === options.headSha && + pull.head.ref === options.headBranch && + pull.head.repo?.full_name === options.headRepository && + pull.base.repo.full_name === options.repository, + ); + if (matches.length !== 1) { + throw new Error( + `Expected one open pull request for the triggering revision; found ${matches.length}`, + ); + } + const detail = validatePullRequest( + await githubApi( + `repos/${options.repository}/pulls/${matches[0]!.number}`, + options.token, + { + userAgent: USER_AGENT, + }, + ), + ); + if (JSON.stringify(pullIdentity(matches[0]!)) !== JSON.stringify(pullIdentity(detail))) { + throw new Error("Pull request identity changed while its details were being resolved"); + } + return detail; +} + +export async function pullChangedFiles( + repository: string, + pull: PullRequest, + token: string, +): Promise { + assertRepository(repository, "repository"); + if (!token) throw new Error("GitHub token is required"); + if ( + !Number.isSafeInteger(pull.changed_files) || + pull.changed_files < 0 || + pull.changed_files > MAX_PR_FILES + ) { + throw new Error(`Pull request changed-file count must be between 0 and ${MAX_PR_FILES}`); + } + const files = await githubRestPaginated( + `repos/${repository}/pulls/${pull.number}/files`, + token, + MAX_PR_FILES, + ); + if (files.length !== pull.changed_files) { + throw new Error( + `Pull request file listing is incomplete: expected ${pull.changed_files}, received ${files.length}`, + ); + } + const changed: string[] = []; + const seen = new Set(); + for (const entry of files) { + if (!isObjectRecord(entry) || typeof entry.filename !== "string") { + throw new Error("GitHub returned an invalid pull request file entry"); + } + const names = [entry.previous_filename, entry.filename].filter( + (name): name is string => typeof name === "string", + ); + for (const name of names) { + assertRepositoryPath(name); + if (!seen.has(name)) { + seen.add(name); + changed.push(name); + } + } + } + return changed; +} + +function assertPullUnchanged(before: PullRequest, after: PullRequest): void { + if ( + JSON.stringify({ ...pullIdentity(before), changedFiles: before.changed_files }) !== + JSON.stringify({ ...pullIdentity(after), changedFiles: after.changed_files }) + ) { + throw new Error("Pull request changed while required live E2E was being prepared"); + } +} + +export function expectedSignalShards( + jobIds: readonly string[], + workflowPath = ".github/workflows/e2e.yaml", +): Record { + const workflow = YAML.parse(fs.readFileSync(workflowPath, "utf8")) as unknown; + const jobs = isObjectRecord(workflow) && isObjectRecord(workflow.jobs) ? workflow.jobs : {}; + return Object.fromEntries( + jobIds.map((jobId) => { + if (!isObjectRecord(jobs[jobId])) throw new Error(`E2E workflow does not define ${jobId}`); + const job = jobs[jobId]; + const strategy = isObjectRecord(job.strategy) ? job.strategy : {}; + const matrix = isObjectRecord(strategy.matrix) ? strategy.matrix : null; + let shards = ["default"]; + if (matrix) { + const keys = Object.keys(matrix); + if (keys.length === 1 && Array.isArray(matrix.agent)) { + shards = matrix.agent.filter((value): value is string => typeof value === "string"); + if (shards.length !== matrix.agent.length) { + throw new Error(`${jobId} matrix agent values must be strings`); + } + } else if (keys.length === 1 && Array.isArray(matrix.include)) { + shards = matrix.include.map((entry) => { + if (!isObjectRecord(entry) || typeof entry.agent !== "string") { + throw new Error(`${jobId} matrix include entries must name an agent`); + } + return entry.agent; + }); + } else { + throw new Error(`${jobId} uses an unsupported evidence matrix`); + } + } + if ( + shards.length === 0 || + new Set(shards).size !== shards.length || + shards.some((shard) => !SHARD_PATTERN.test(shard)) + ) { + throw new Error(`${jobId} evidence shards must be unique safe identifiers`); + } + return [jobId, shards]; + }), + ); +} + +export function validateWorkflowDispatchDetails( + value: unknown, + repository: string, +): WorkflowDispatchDetails { + if (!isObjectRecord(value)) throw new Error("GitHub returned invalid workflow dispatch details"); + const runId = value.workflow_run_id; + if (!Number.isSafeInteger(runId) || (runId as number) < 1) { + throw new Error("GitHub returned an invalid dispatched workflow run id"); + } + const expectedApiUrl = `https://api.github.com/repos/${repository}/actions/runs/${runId}`; + const expectedHtmlUrl = `https://github.com/${repository}/actions/runs/${runId}`; + if (value.run_url !== expectedApiUrl || value.html_url !== expectedHtmlUrl) { + throw new Error("GitHub returned mismatched workflow dispatch URLs"); + } + return value as WorkflowDispatchDetails; +} + +function diagnosticValue(value: unknown): string { + const serialized = JSON.stringify(value) ?? String(value); + return serialized.length > 256 ? `${serialized.slice(0, 253)}...` : serialized; +} + +export function assertCorrelatedWorkflowRun( + child: WorkflowRun, + identity: WorkflowRunIdentity, +): void { + const childRunUrl = `https://github.com/${identity.repository}/actions/runs/${identity.childRunId}`; + const mismatches: string[] = []; + const requireEqual = (field: string, expected: unknown, actual: unknown): void => { + if (actual !== expected) { + mismatches.push( + `${field} expected=${diagnosticValue(expected)} actual=${diagnosticValue(actual)}`, + ); + } + }; + requireEqual("id", identity.childRunId, child.id); + requireEqual("path", E2E_WORKFLOW_PATH, child.path); + requireEqual("event", "workflow_dispatch", child.event); + requireEqual("html_url", childRunUrl, child.html_url); + requireEqual( + "display_title", + `E2E PR #${identity.prNumber} required live ${identity.correlationId}`, + child.display_title, + ); + requireEqual("head_sha", identity.workflowSha, child.head_sha); + if (!Number.isSafeInteger(child.workflow_id) || child.workflow_id < 1) { + mismatches.push( + `workflow_id expected="positive safe integer" actual=${diagnosticValue(child.workflow_id)}`, + ); + } + if (mismatches.length > 0) { + throw new Error( + `Correlated E2E workflow identity mismatch: ${mismatches.join("; ")}; observed run_name=${diagnosticValue(child.name)} workflow_id=${diagnosticValue(child.workflow_id)}`, + ); + } +} + +export async function dispatchRequiredLive(options: { + repository: string; + token: string; + jobs: readonly string[]; + prNumber: number; + commitSha: string; + workflowSha: string; + planHash: string; + correlationId: string; +}): Promise { + assertRepository(options.repository, "repository"); + if ( + !options.token || + options.jobs.length < 1 || + new Set(options.jobs).size !== options.jobs.length || + options.jobs.some((job) => !JOB_PATTERN.test(job)) || + !Number.isSafeInteger(options.prNumber) || + options.prNumber < 1 || + !SHA_PATTERN.test(options.commitSha) || + !SHA_PATTERN.test(options.workflowSha) || + !HASH_PATTERN.test(options.planHash) || + !CORRELATION_PATTERN.test(options.correlationId) + ) { + throw new Error("required-live workflow dispatch inputs are invalid"); + } + const main = await githubApi( + `repos/${options.repository}/git/ref/heads/main`, + options.token, + { userAgent: USER_AGENT }, + ); + if ( + !main || + main.ref !== "refs/heads/main" || + main.object?.type !== "commit" || + main.object.sha !== options.workflowSha + ) { + throw new Error( + `Trusted workflow revision ${options.workflowSha} is no longer the current main revision`, + ); + } + const details = await githubApi( + `repos/${options.repository}/actions/workflows/${E2E_WORKFLOW}/dispatches`, + options.token, + { + method: "POST", + body: { + ref: "main", + inputs: { + jobs: options.jobs.join(","), + pr_number: String(options.prNumber), + checkout_sha: options.commitSha, + plan_hash: options.planHash, + correlation_id: options.correlationId, + }, + return_run_details: true, + }, + userAgent: USER_AGENT, + }, + ); + return validateWorkflowDispatchDetails(details, options.repository).workflow_run_id; +} + +async function cancelChildRun(repository: string, token: string, runId: number): Promise { + try { + await githubApi(`repos/${repository}/actions/runs/${runId}/cancel`, token, { + method: "POST", + userAgent: USER_AGENT, + }); + } catch (error) { + if (/failed: 409\b/u.test(controllerErrorMessage(error))) return; + throw error; + } +} + +export async function startRequiredLive( + command: Extract, +): Promise { + const { token, repository } = tokenAndRepository(); + if (!SHA_PATTERN.test(command.headSha)) throw new Error("triggering head SHA is invalid"); + if (!SHA_PATTERN.test(command.workflowSha)) throw new Error("trusted workflow SHA is invalid"); + assertRepository(command.headRepository, "triggering head repository"); + assertBranch(command.headBranch); + + const checkRunId = await createCheck( + repository, + token, + command.headSha, + "Required live E2E is evaluating this revision", + "The controller is validating the pull request and building its deterministic live-test plan.", + ); + appendOutput("check_id", String(checkRunId)); + + let finalized = false; + let childRunId: number | undefined; + try { + if (command.ciConclusion !== "success") { + await completeCheck({ repository, checkRunId }, token, { + conclusion: "failure", + title: "Pull request CI did not pass", + summary: `CI / Pull Request concluded ${command.ciConclusion || "without a result"}; live E2E was not dispatched.`, + }); + appendOutput("dispatched", "false"); + appendOutput("finalized", "true"); + finalized = true; + throw new Error(`CI / Pull Request concluded ${command.ciConclusion || "without a result"}`); + } + + const pull = await resolvePullRequest({ + repository, + token, + headSha: command.headSha, + headRepository: command.headRepository, + headBranch: command.headBranch, + }); + if (command.headRepository !== repository || pull.head.repo?.full_name !== repository) { + throw new Error("Required live E2E can run only for branches in the base repository"); + } + + const changedFiles = await pullChangedFiles(repository, pull, token); + const allowedJobs = new Set(readFreeStandingJobsInventory().allowedJobs); + const plan = validateRiskPlan( + buildRiskPlan({ headSha: command.headSha, changedFiles }), + allowedJobs, + ); + writePrivateRegularFile(command.planPath, `${JSON.stringify(plan, null, 2)}\n`); + const jobs = riskPlanRequiredJobIds(plan); + const currentPull = await resolvePullRequest({ + repository, + token, + headSha: command.headSha, + headRepository: command.headRepository, + headBranch: command.headBranch, + }); + assertPullUnchanged(pull, currentPull); + if (jobs.length === 0) { + await completeCheck({ repository, checkRunId }, token, { + conclusion: "success", + title: "No required live E2E selected", + summary: "The deterministic plan matched no live runtime regression family.", + }); + appendOutput("dispatched", "false"); + appendOutput("finalized", "true"); + finalized = true; + console.log( + `Required live E2E completed without dispatch: pr=${pull.number} plan=${plan.planHash}`, + ); + return; + } + + const expectedShards = expectedSignalShards(jobs); + const correlationId = randomUUID(); + if (!CORRELATION_PATTERN.test(correlationId)) { + throw new Error("generated correlation id is invalid"); + } + childRunId = await dispatchRequiredLive({ + repository, + token, + jobs, + prNumber: pull.number, + commitSha: command.headSha, + workflowSha: command.workflowSha, + planHash: plan.planHash, + correlationId, + }); + appendOutput("run_id", String(childRunId)); + const state: RequiredLiveState = { + version: 1, + commitSha: command.headSha, + workflowSha: command.workflowSha, + planHash: plan.planHash, + correlationId, + prNumber: pull.number, + expectedJobs: jobs, + expectedShards, + }; + const serializedState = `${JSON.stringify(state, null, 2)}\n`; + writePrivateRegularFile(command.statePath, serializedState); + await updateRunningCheck({ repository, checkRunId }, token, { + childRunId, + jobs, + planHash: plan.planHash, + }); + appendOutput("state_hash", sha256(serializedState)); + appendOutput("dispatched", "true"); + console.log( + `Required live E2E dispatched: pr=${pull.number} run=${childRunId} plan=${plan.planHash} jobs=${jobs.join(",")} url=https://github.com/${repository}/actions/runs/${childRunId}`, + ); + } catch (error) { + let reportedError = error; + if (!finalized && childRunId) { + try { + await cancelChildRun(repository, token, childRunId); + } catch (cancelError) { + reportedError = new Error( + `${controllerErrorMessage(error)}; child cancellation failed: ${controllerErrorMessage(cancelError)}`, + ); + } + } + if (!finalized) { + const closed = await completeFailureAfterControllerError( + { repository, checkRunId }, + token, + "Required live E2E could not start", + { error: reportedError }, + ); + if (closed) appendOutput("finalized", "true"); + } + throw reportedError; + } +} + +export function findSignalFiles( + root: string, + limits: { maxDepth: number; maxEntries: number; maxSignalFiles: number }, +): string[] { + if (!fs.existsSync(root)) return []; + if ( + !Number.isSafeInteger(limits.maxDepth) || + limits.maxDepth < 0 || + !Number.isSafeInteger(limits.maxEntries) || + limits.maxEntries < 1 || + !Number.isSafeInteger(limits.maxSignalFiles) || + limits.maxSignalFiles < 1 + ) { + throw new Error("E2E evidence traversal limits are invalid"); + } + const rootStat = fs.lstatSync(root); + if (!rootStat.isDirectory() || rootStat.isSymbolicLink()) { + throw new Error("E2E evidence root must be a directory, not a symlink"); + } + const files: string[] = []; + let entriesVisited = 0; + const visit = (directory: string, depth: number): void => { + const handle = fs.opendirSync(directory); + try { + let entry = handle.readSync(); + while (entry !== null) { + entriesVisited += 1; + if (entriesVisited > limits.maxEntries) { + throw new Error("E2E evidence exceeds the entry limit"); + } + const full = path.join(directory, entry.name); + if (entry.isSymbolicLink()) throw new Error("E2E evidence must not contain symlinks"); + if (entry.isDirectory()) { + if (depth >= limits.maxDepth) throw new Error("E2E evidence exceeds the depth limit"); + visit(full, depth + 1); + } else if (entry.isFile() && entry.name === "risk-signal.json") { + files.push(full); + if (files.length > limits.maxSignalFiles) { + throw new Error("E2E evidence exceeds the signal-file limit"); + } + } + entry = handle.readSync(); + } + } finally { + handle.closeSync(); + } + }; + visit(root, 0); + return files.sort((left, right) => left.localeCompare(right)); +} + +export async function finishRequiredLive(options: { + statePath: string; + stateHash: string; + evidencePath: string; + checkRunId: number; + childRunId: number; +}): Promise { + const { token, repository } = tokenAndRepository(); + const childRunUrl = `https://github.com/${repository}/actions/runs/${options.childRunId}`; + const context = { repository, checkRunId: options.checkRunId }; + let finalized = false; + try { + if (!HASH_PATTERN.test(options.stateHash)) throw new Error("controller state hash is invalid"); + const serializedState = readPrivateRegularFile(options.statePath, { + maxBytes: MAX_PLAN_BYTES, + })!; + if (sha256(serializedState) !== options.stateHash) { + throw new Error("controller state changed after E2E dispatch"); + } + const state = validateRequiredLiveState(JSON.parse(serializedState)); + const child = await githubApi( + `repos/${repository}/actions/runs/${options.childRunId}`, + token, + { userAgent: USER_AGENT }, + ); + assertCorrelatedWorkflowRun(child, { + childRunId: options.childRunId, + correlationId: state.correlationId, + prNumber: state.prNumber, + repository, + workflowSha: state.workflowSha, + }); + if (child.status !== "completed") { + await cancelChildRun(repository, token, options.childRunId); + console.log( + `Cancelled unfinished required live E2E during finalization: run=${options.childRunId} status=${child.status} url=${childRunUrl}`, + ); + } + const workflowConclusion = + child.status === "completed" ? child.conclusion : `unfinished (${child.status})`; + const expectedSignalCount = Object.values(state.expectedShards).reduce( + (total, shards) => total + shards.length, + 0, + ); + const signals = + workflowConclusion === "success" + ? findSignalFiles(options.evidencePath, { + ...EVIDENCE_LIMITS, + maxSignalFiles: expectedSignalCount + 1, + }).map((file) => validateSignal(readRegularJson(file), state)) + : []; + const verdict = classifyRequiredLiveEvidence({ + workflowConclusion, + expectedJobs: state.expectedJobs, + expectedShards: state.expectedShards, + signals, + }); + await completeCheck(context, token, verdict, childRunUrl); + appendOutput("finalized", "true"); + finalized = true; + console.log( + `Required live E2E completed: run=${options.childRunId} conclusion=${verdict.conclusion} title=${verdict.title} url=${childRunUrl}`, + ); + if (verdict.conclusion === "failure") throw new Error(verdict.title); + } catch (error) { + if (!finalized) { + const closed = await completeFailureAfterControllerError( + context, + token, + "Required live E2E evidence could not be verified", + { error, detailsUrl: childRunUrl }, + ); + if (closed) appendOutput("finalized", "true"); + } + throw error; + } +} + +export async function abandonRequiredLive(checkRunId: number, childRunId?: number): Promise { + const { token, repository } = tokenAndRepository(); + let cancellationError: unknown; + if (childRunId) { + try { + await cancelChildRun(repository, token, childRunId); + } catch (error) { + cancellationError = error; + } + } + const cancellationSummary = cancellationError + ? ` Child cancellation also failed: ${controllerErrorMessage(cancellationError)}.` + : ""; + await completeCheck({ repository, checkRunId }, token, { + conclusion: "failure", + title: "Required live E2E controller stopped early", + summary: `The controller stopped before it could produce complete evidence.${cancellationSummary}`, + }); + appendOutput("finalized", "true"); + if (cancellationError) throw cancellationError; +} + +export async function cancelRequiredLive(prNumber: number): Promise { + const { token, repository } = tokenAndRepository(); + if (!Number.isSafeInteger(prNumber) || prNumber < 1) throw new Error("PR number is invalid"); + const titlePrefix = `E2E PR #${prNumber} required live `; + const active: WorkflowRun[] = []; + for (let page = 1; page <= MAX_ACTIVE_RUN_PAGES; page += 1) { + const response = await githubApi( + `repos/${repository}/actions/workflows/${E2E_WORKFLOW}/runs?event=workflow_dispatch&per_page=100&page=${page}`, + token, + { userAgent: USER_AGENT }, + ); + if (!response || !Array.isArray(response.workflow_runs)) { + throw new Error("GitHub returned an invalid workflow run list"); + } + active.push( + ...response.workflow_runs.filter( + (run) => run.display_title.startsWith(titlePrefix) && run.status !== "completed", + ), + ); + if (response.workflow_runs.length < 100) break; + if (page === MAX_ACTIVE_RUN_PAGES) { + throw new Error("Required-live run listing exceeded its page limit"); + } + } + for (const run of active) { + if (!Number.isSafeInteger(run.id) || run.id < 1) { + throw new Error("GitHub returned an invalid active workflow run id"); + } + await cancelChildRun(repository, token, run.id); + console.log( + `Cancelled superseded required live E2E: pr=${prNumber} run=${run.id} url=https://github.com/${repository}/actions/runs/${run.id}`, + ); + } + if (active.length === 0) { + console.log(`No active required live E2E runs found for PR #${prNumber}`); + } + return active.length; +} + +function reportControllerError(error: unknown): void { + const message = controllerErrorMessage(error); + console.error(message); + if (process.env.GITHUB_ACTIONS === "true") { + const escaped = message.replace(/%/gu, "%25").replace(/\r/gu, "%0D").replace(/\n/gu, "%0A"); + console.error(`::error title=Required live E2E controller failed::${escaped}`); + } +} + +async function main(): Promise { + const command = parseControllerCommand(process.argv.slice(2)); + if (command.mode === "start") { + await startRequiredLive(command); + return; + } + if (command.mode === "finish") { + await finishRequiredLive({ + statePath: command.statePath, + stateHash: command.stateHash, + evidencePath: command.evidencePath, + checkRunId: command.checkRunId, + childRunId: command.childRunId, + }); + return; + } + if (command.mode === "abandon") { + await abandonRequiredLive(command.checkRunId, command.childRunId); + return; + } + await cancelRequiredLive(command.prNumber); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error: unknown) => { + reportControllerError(error); + process.exit(1); + }); +} diff --git a/tools/e2e-advisor/risk-signal.ts b/tools/e2e/risk-signal.ts similarity index 100% rename from tools/e2e-advisor/risk-signal.ts rename to tools/e2e/risk-signal.ts diff --git a/tools/pr-review-advisor/README.md b/tools/pr-review-advisor/README.md index e4331a08c51..2c7cd8ef873 100644 --- a/tools/pr-review-advisor/README.md +++ b/tools/pr-review-advisor/README.md @@ -56,9 +56,11 @@ before the failure so later runs and reviewers do not lose substantive review hi The workflow is advisory and must not be configured as a required status check. It uses the deterministic plan as review context but does not run its jobs. E2E Advisor emits the corresponding plan-backed recommendations separately and likewise does not dispatch E2E. Model availability must -not become the authority for whether a pull request can merge. After a commit lands, a separate -model-independent shadow controller rebuilds the plan from the exact `main` push range and runs its -capped automatic subset. That post-merge check does not make PR Review Advisor a merge gate. +not become the authority for whether a pull request can merge. The separate model-independent +required live controller rebuilds the plan from GitHub's changed-file list for the current pull +request revision and dispatches every required job after `CI / Pull Request` completes. It does not +consume PR Review Advisor or E2E Advisor output. The `E2E / Required Live` check therefore remains +independent of both model advisors and does not make PR Review Advisor a merge gate. Required-check status is point-in-time context, not a settled-CI gate. Earlier `PR_REVIEW_ADVISOR_WAIT_*` workflow variables were inert and have been removed; any future waiting @@ -82,7 +84,8 @@ Authors and coding agents should follow the shared [PR CI and Automated Review F - During rollout, non-default advisor lanes may see an older trusted `main` checkout that has the workflow matrix but not the matching model/configurable-comment support. The workflow treats that as trusted-main rollout skew, writes low-confidence skip artifacts in the lane-specific artifact directory, and suppresses that lane's sticky PR comment. Do not run PR-controlled advisor code to bypass this gate; remove the gate only after the trusted `main` implementation always supports the parallel advisor lane and configurable sticky markers. - The checked-in risk plan is deterministic and additive. PR Review Advisor reviews every listed invariant and required job for missing evidence. Both E2E Advisor result normalizers restore any - listed job that a model omits or downgrades. + listed job that a model omits or downgrades. The required live controller separately dispatches + every listed job without consuming either advisor's normalized result. ## Required secret diff --git a/tools/pr-review-advisor/analyze.mts b/tools/pr-review-advisor/analyze.mts index 8e5350dd23d..ce48fb3ea29 100755 --- a/tools/pr-review-advisor/analyze.mts +++ b/tools/pr-review-advisor/analyze.mts @@ -2141,7 +2141,6 @@ function buildReconciliationTurnContext( tier: context.riskPlan.tier, familyIds: context.riskPlan.families.map((family) => family.id), requiredJobIds: context.riskPlan.requiredJobs.map((job) => job.id), - requiresManualExpansion: context.riskPlan.requiresManualExpansion, }, linkedIssues: (context.github?.linkedIssues ?? []).map(({ number, fetchError }) => ({ number, @@ -2173,9 +2172,6 @@ export function buildRiskPlanReviewContext(plan: RiskPlan): Record Date: Fri, 10 Jul 2026 15:30:29 -0700 Subject: [PATCH 2/7] test(e2e): keep required live fixtures linear --- test/required-live.test.ts | 388 +++++++++++++++++----------- test/support/github-fetch-router.ts | 49 ++++ 2 files changed, 286 insertions(+), 151 deletions(-) create mode 100644 test/support/github-fetch-router.ts diff --git a/test/required-live.test.ts b/test/required-live.test.ts index 8a2e5dd3a32..6b61c24bf68 100644 --- a/test/required-live.test.ts +++ b/test/required-live.test.ts @@ -28,6 +28,11 @@ import { validateWorkflowDispatchDetails, } from "../tools/e2e/required-live.mts"; import type { E2eRiskSignal } from "../tools/e2e/risk-signal.ts"; +import { + createGitHubFetchRouter, + githubFetchRoute, + type RecordedGitHubRequest, +} from "./support/github-fetch-router.ts"; const HEAD_SHA = "a".repeat(40); const BASE_SHA = "b".repeat(40); @@ -132,8 +137,8 @@ function startCommand(workDir: string) { "--work-dir", workDir, ]); - if (command.mode !== "start") throw new Error("unexpected command mode"); - return command; + expect(command.mode).toBe("start"); + return command as Extract, { mode: "start" }>; } function signal( @@ -248,12 +253,18 @@ describe("required live E2E controller", () => { ...(index === 0 ? { previous_filename: "src/old-name.ts" } : {}), })); const pageTwo = [{ filename: "src/file-100.ts" }]; - const fetchMock = vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => { - const url = String(input); - if (url.endsWith("page=1")) return githubResponse(pageOne); - if (url.endsWith("page=2")) return githubResponse(pageTwo); - throw new Error(`Unexpected request: ${url}`); - }); + const fetchMock = vi.spyOn(globalThis, "fetch").mockImplementation( + createGitHubFetchRouter([ + githubFetchRoute( + ({ url }) => url.endsWith("page=1"), + () => githubResponse(pageOne), + ), + githubFetchRoute( + ({ url }) => url.endsWith("page=2"), + () => githubResponse(pageTwo), + ), + ]), + ); const files = await pullChangedFiles("NVIDIA/NemoClaw", pullRequest(101), "token"); @@ -320,19 +331,27 @@ describe("required live E2E controller", () => { it("dispatches every selected job through the five-field child protocol", async () => { const jobs = ["onboard-repair", "onboard-resume", "full-e2e", "hermes-e2e"]; - const fetchMock = vi.spyOn(globalThis, "fetch").mockImplementation(async (input) => { - if (String(input).endsWith("/git/ref/heads/main")) { - return githubResponse({ - ref: "refs/heads/main", - object: { type: "commit", sha: WORKFLOW_SHA }, - }); - } - return githubResponse({ - workflow_run_id: 23, - run_url: "https://api.github.com/repos/NVIDIA/NemoClaw/actions/runs/23", - html_url: "https://github.com/NVIDIA/NemoClaw/actions/runs/23", - }); - }); + const fetchMock = vi.spyOn(globalThis, "fetch").mockImplementation( + createGitHubFetchRouter([ + githubFetchRoute( + ({ url }) => url.endsWith("/git/ref/heads/main"), + () => + githubResponse({ + ref: "refs/heads/main", + object: { type: "commit", sha: WORKFLOW_SHA }, + }), + ), + githubFetchRoute( + ({ url }) => url.endsWith("/actions/workflows/e2e.yaml/dispatches"), + () => + githubResponse({ + workflow_run_id: 23, + run_url: "https://api.github.com/repos/NVIDIA/NemoClaw/actions/runs/23", + html_url: "https://github.com/NVIDIA/NemoClaw/actions/runs/23", + }), + ), + ]), + ); await expect( dispatchRequiredLive({ @@ -419,41 +438,59 @@ describe("required live E2E controller", () => { vi.stubEnv("GITHUB_TOKEN", "token"); vi.stubEnv("GITHUB_REPOSITORY", "NVIDIA/NemoClaw"); vi.stubEnv("GITHUB_OUTPUT", outputPath); - const requests: Array<{ url: string; method: string; body?: unknown }> = []; + const requests: RecordedGitHubRequest[] = []; let gate: RequiredLiveState | undefined; - vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => { - const url = String(input); - const method = init?.method ?? "GET"; - const body = init?.body ? JSON.parse(String(init.body)) : undefined; - requests.push({ url, method, body }); - if (url.endsWith("/check-runs") && method === "POST") return githubResponse({ id: 17 }); - if (url.includes("/pulls?state=open&head=")) { - return githubResponse([pullRequestListItem(pullRequest(BROAD_FILES.length))]); - } - if (url.includes("/pulls/42/files?")) { - return githubResponse(BROAD_FILES.map((filename) => ({ filename }))); - } - if (url.endsWith("/pulls/42")) return githubResponse(pullRequest(BROAD_FILES.length)); - if (url.endsWith("/git/ref/heads/main")) { - return githubResponse({ - ref: "refs/heads/main", - object: { type: "commit", sha: WORKFLOW_SHA }, - }); - } - if (url.endsWith("/actions/workflows/e2e.yaml/dispatches")) { - return githubResponse({ - workflow_run_id: 23, - run_url: "https://api.github.com/repos/NVIDIA/NemoClaw/actions/runs/23", - html_url: "https://github.com/NVIDIA/NemoClaw/actions/runs/23", - }); - } - if (url.endsWith("/actions/runs/23") && method === "GET") { - if (!gate) throw new Error("state was not loaded before finish"); - return githubResponse(workflowRun(gate)); - } - if (url.endsWith("/check-runs/17") && method === "PATCH") return githubResponse({}); - throw new Error(`Unexpected request: ${method} ${url}`); - }); + vi.spyOn(globalThis, "fetch").mockImplementation( + createGitHubFetchRouter( + [ + githubFetchRoute( + ({ url, method }) => url.endsWith("/check-runs") && method === "POST", + () => githubResponse({ id: 17 }), + ), + githubFetchRoute( + ({ url }) => url.includes("/pulls?state=open&head="), + () => githubResponse([pullRequestListItem(pullRequest(BROAD_FILES.length))]), + ), + githubFetchRoute( + ({ url }) => url.includes("/pulls/42/files?"), + () => githubResponse(BROAD_FILES.map((filename) => ({ filename }))), + ), + githubFetchRoute( + ({ url }) => url.endsWith("/pulls/42"), + () => githubResponse(pullRequest(BROAD_FILES.length)), + ), + githubFetchRoute( + ({ url }) => url.endsWith("/git/ref/heads/main"), + () => + githubResponse({ + ref: "refs/heads/main", + object: { type: "commit", sha: WORKFLOW_SHA }, + }), + ), + githubFetchRoute( + ({ url }) => url.endsWith("/actions/workflows/e2e.yaml/dispatches"), + () => + githubResponse({ + workflow_run_id: 23, + run_url: "https://api.github.com/repos/NVIDIA/NemoClaw/actions/runs/23", + html_url: "https://github.com/NVIDIA/NemoClaw/actions/runs/23", + }), + ), + githubFetchRoute( + ({ url, method }) => url.endsWith("/actions/runs/23") && method === "GET", + () => { + expect(gate).toBeDefined(); + return githubResponse(workflowRun(gate!)); + }, + ), + githubFetchRoute( + ({ url, method }) => url.endsWith("/check-runs/17") && method === "PATCH", + () => githubResponse({}), + ), + ], + requests, + ), + ); try { const command = startCommand(workDir); @@ -526,33 +563,48 @@ describe("required live E2E controller", () => { vi.stubEnv("GITHUB_TOKEN", "token"); vi.stubEnv("GITHUB_REPOSITORY", "NVIDIA/NemoClaw"); vi.stubEnv("GITHUB_OUTPUT", outputPath); - const requests: Array<{ url: string; method: string; body?: unknown }> = []; + const requests: RecordedGitHubRequest[] = []; let listCalls = 0; let detailCalls = 0; const updatedPull = { ...pullRequest(), base: { ...pullRequest().base, sha: "c".repeat(40) }, }; - vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => { - const url = String(input); - const method = init?.method ?? "GET"; - const body = init?.body ? JSON.parse(String(init.body)) : undefined; - requests.push({ url, method, body }); - if (url.endsWith("/check-runs") && method === "POST") return githubResponse({ id: 17 }); - if (url.includes("/pulls?state=open&head=")) { - listCalls += 1; - return githubResponse([pullRequestListItem(listCalls === 1 ? pullRequest() : updatedPull)]); - } - if (url.includes("/pulls/42/files?")) { - return githubResponse([{ filename: "src/lib/onboard.ts" }]); - } - if (url.endsWith("/pulls/42")) { - detailCalls += 1; - return githubResponse(detailCalls === 1 ? pullRequest() : updatedPull); - } - if (url.endsWith("/check-runs/17") && method === "PATCH") return githubResponse({}); - throw new Error(`Unexpected request: ${method} ${url}`); - }); + vi.spyOn(globalThis, "fetch").mockImplementation( + createGitHubFetchRouter( + [ + githubFetchRoute( + ({ url, method }) => url.endsWith("/check-runs") && method === "POST", + () => githubResponse({ id: 17 }), + ), + githubFetchRoute( + ({ url }) => url.includes("/pulls?state=open&head="), + () => { + listCalls += 1; + return githubResponse([ + pullRequestListItem(listCalls === 1 ? pullRequest() : updatedPull), + ]); + }, + ), + githubFetchRoute( + ({ url }) => url.includes("/pulls/42/files?"), + () => githubResponse([{ filename: "src/lib/onboard.ts" }]), + ), + githubFetchRoute( + ({ url }) => url.endsWith("/pulls/42"), + () => { + detailCalls += 1; + return githubResponse(detailCalls === 1 ? pullRequest() : updatedPull); + }, + ), + githubFetchRoute( + ({ url, method }) => url.endsWith("/check-runs/17") && method === "PATCH", + () => githubResponse({}), + ), + ], + requests, + ), + ); try { await expect(startRequiredLive(startCommand(workDir))).rejects.toThrow( @@ -577,43 +629,61 @@ describe("required live E2E controller", () => { vi.stubEnv("GITHUB_TOKEN", "token"); vi.stubEnv("GITHUB_REPOSITORY", "NVIDIA/NemoClaw"); vi.stubEnv("GITHUB_OUTPUT", outputPath); - const requests: Array<{ url: string; method: string; body?: unknown }> = []; + const requests: RecordedGitHubRequest[] = []; let checkPatches = 0; - vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => { - const url = String(input); - const method = init?.method ?? "GET"; - const body = init?.body ? JSON.parse(String(init.body)) : undefined; - requests.push({ url, method, body }); - if (url.endsWith("/check-runs") && method === "POST") return githubResponse({ id: 17 }); - if (url.includes("/pulls?state=open&head=")) return githubResponse([pullRequestListItem()]); - if (url.includes("/pulls/42/files?")) { - return githubResponse([{ filename: "src/lib/onboard.ts" }]); - } - if (url.endsWith("/pulls/42")) return githubResponse(pullRequest()); - if (url.endsWith("/git/ref/heads/main")) { - return githubResponse({ - ref: "refs/heads/main", - object: { type: "commit", sha: WORKFLOW_SHA }, - }); - } - if (url.endsWith("/actions/workflows/e2e.yaml/dispatches")) { - return githubResponse({ - workflow_run_id: 23, - run_url: "https://api.github.com/repos/NVIDIA/NemoClaw/actions/runs/23", - html_url: "https://github.com/NVIDIA/NemoClaw/actions/runs/23", - }); - } - if (url.endsWith("/actions/runs/23/cancel") && method === "POST") { - return githubResponse(undefined, 202); - } - if (url.endsWith("/check-runs/17") && method === "PATCH") { - checkPatches += 1; - return checkPatches === 1 - ? githubResponse({ message: "simulated update failure" }, 500) - : githubResponse({}); - } - throw new Error(`Unexpected request: ${method} ${url}`); - }); + vi.spyOn(globalThis, "fetch").mockImplementation( + createGitHubFetchRouter( + [ + githubFetchRoute( + ({ url, method }) => url.endsWith("/check-runs") && method === "POST", + () => githubResponse({ id: 17 }), + ), + githubFetchRoute( + ({ url }) => url.includes("/pulls?state=open&head="), + () => githubResponse([pullRequestListItem()]), + ), + githubFetchRoute( + ({ url }) => url.includes("/pulls/42/files?"), + () => githubResponse([{ filename: "src/lib/onboard.ts" }]), + ), + githubFetchRoute( + ({ url }) => url.endsWith("/pulls/42"), + () => githubResponse(pullRequest()), + ), + githubFetchRoute( + ({ url }) => url.endsWith("/git/ref/heads/main"), + () => + githubResponse({ + ref: "refs/heads/main", + object: { type: "commit", sha: WORKFLOW_SHA }, + }), + ), + githubFetchRoute( + ({ url }) => url.endsWith("/actions/workflows/e2e.yaml/dispatches"), + () => + githubResponse({ + workflow_run_id: 23, + run_url: "https://api.github.com/repos/NVIDIA/NemoClaw/actions/runs/23", + html_url: "https://github.com/NVIDIA/NemoClaw/actions/runs/23", + }), + ), + githubFetchRoute( + ({ url, method }) => url.endsWith("/actions/runs/23/cancel") && method === "POST", + () => githubResponse(undefined, 202), + ), + githubFetchRoute( + ({ url, method }) => url.endsWith("/check-runs/17") && method === "PATCH", + () => { + checkPatches += 1; + return checkPatches === 1 + ? githubResponse({ message: "simulated update failure" }, 500) + : githubResponse({}); + }, + ), + ], + requests, + ), + ); try { await expect(startRequiredLive(startCommand(workDir))).rejects.toThrow( @@ -647,21 +717,26 @@ describe("required live E2E controller", () => { vi.stubEnv("GITHUB_TOKEN", "token"); vi.stubEnv("GITHUB_REPOSITORY", "NVIDIA/NemoClaw"); vi.stubEnv("GITHUB_OUTPUT", outputPath); - const requests: Array<{ url: string; method: string; body?: unknown }> = []; - vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => { - const url = String(input); - const method = init?.method ?? "GET"; - const body = init?.body ? JSON.parse(String(init.body)) : undefined; - requests.push({ url, method, body }); - if (url.endsWith("/actions/runs/23") && method === "GET") { - return githubResponse(workflowRun(gate, { status, conclusion: "success" })); - } - if (url.endsWith("/actions/runs/23/cancel") && method === "POST") { - return githubResponse(undefined, 202); - } - if (url.endsWith("/check-runs/17") && method === "PATCH") return githubResponse({}); - throw new Error(`Unexpected request: ${method} ${url}`); - }); + const requests: RecordedGitHubRequest[] = []; + vi.spyOn(globalThis, "fetch").mockImplementation( + createGitHubFetchRouter( + [ + githubFetchRoute( + ({ url, method }) => url.endsWith("/actions/runs/23") && method === "GET", + () => githubResponse(workflowRun(gate, { status, conclusion: "success" })), + ), + githubFetchRoute( + ({ url, method }) => url.endsWith("/actions/runs/23/cancel") && method === "POST", + () => githubResponse(undefined, 202), + ), + githubFetchRoute( + ({ url, method }) => url.endsWith("/check-runs/17") && method === "PATCH", + () => githubResponse({}), + ), + ], + requests, + ), + ); try { await expect( @@ -688,22 +763,25 @@ describe("required live E2E controller", () => { vi.stubEnv("GITHUB_TOKEN", "token"); vi.stubEnv("GITHUB_REPOSITORY", "NVIDIA/NemoClaw"); const gate = state(); - const fetchMock = vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => { - const url = String(input); - if (url.includes("/actions/workflows/e2e.yaml/runs?")) { - return githubResponse({ - workflow_runs: [ - workflowRun(gate, { status: "in_progress" }), - workflowRun(gate, { id: 24, status: "completed" }), - workflowRun(gate, { id: 25, status: "queued", display_title: "E2E manual" }), - ], - }); - } - if (url.endsWith("/actions/runs/23/cancel") && init?.method === "POST") { - return githubResponse(undefined, 202); - } - throw new Error(`Unexpected request: ${init?.method ?? "GET"} ${url}`); - }); + const fetchMock = vi.spyOn(globalThis, "fetch").mockImplementation( + createGitHubFetchRouter([ + githubFetchRoute( + ({ url }) => url.includes("/actions/workflows/e2e.yaml/runs?"), + () => + githubResponse({ + workflow_runs: [ + workflowRun(gate, { status: "in_progress" }), + workflowRun(gate, { id: 24, status: "completed" }), + workflowRun(gate, { id: 25, status: "queued", display_title: "E2E manual" }), + ], + }), + ), + githubFetchRoute( + ({ url, method }) => url.endsWith("/actions/runs/23/cancel") && method === "POST", + () => githubResponse(undefined, 202), + ), + ]), + ); await expect(cancelRequiredLive(42)).resolves.toBe(1); expect( @@ -718,14 +796,22 @@ describe("required live E2E controller", () => { vi.stubEnv("GITHUB_TOKEN", "token"); vi.stubEnv("GITHUB_REPOSITORY", "NVIDIA/NemoClaw"); vi.stubEnv("GITHUB_OUTPUT", outputPath); - const requests: Array<{ url: string; body?: unknown }> = []; - vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => { - requests.push({ - url: String(input), - body: init?.body ? JSON.parse(String(init.body)) : undefined, - }); - return githubResponse(undefined, String(input).endsWith("/cancel") ? 202 : 200); - }); + const requests: RecordedGitHubRequest[] = []; + vi.spyOn(globalThis, "fetch").mockImplementation( + createGitHubFetchRouter( + [ + githubFetchRoute( + ({ url, method }) => url.endsWith("/actions/runs/23/cancel") && method === "POST", + () => githubResponse(undefined, 202), + ), + githubFetchRoute( + ({ url, method }) => url.endsWith("/check-runs/17") && method === "PATCH", + () => githubResponse(undefined), + ), + ], + requests, + ), + ); try { await abandonRequiredLive(17, 23); diff --git a/test/support/github-fetch-router.ts b/test/support/github-fetch-router.ts new file mode 100644 index 00000000000..3c8995404f5 --- /dev/null +++ b/test/support/github-fetch-router.ts @@ -0,0 +1,49 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export interface RecordedGitHubRequest { + body?: unknown; + method: string; + url: string; +} + +export interface GitHubFetchRoute { + matches: (request: RecordedGitHubRequest) => boolean; + respond: (request: RecordedGitHubRequest) => Promise | Response; +} + +async function readRequestBody(input: Parameters[0], init?: RequestInit) { + if (init?.body !== undefined) return String(init.body); + if (input instanceof Request) return input.clone().text(); + return ""; +} + +export function githubFetchRoute( + matches: GitHubFetchRoute["matches"], + respond: GitHubFetchRoute["respond"], +): GitHubFetchRoute { + return { matches, respond }; +} + +export function createGitHubFetchRouter( + routes: readonly GitHubFetchRoute[], + requests?: RecordedGitHubRequest[], +): typeof fetch { + return (async (input, init) => { + const requestInput = input instanceof Request ? input : undefined; + const serializedBody = await readRequestBody(input, init); + const request: RecordedGitHubRequest = { + url: requestInput?.url ?? String(input), + method: init?.method ?? requestInput?.method ?? "GET", + body: serializedBody === "" ? undefined : JSON.parse(serializedBody), + }; + requests?.push(request); + const matchingRoutes = routes.filter((candidate) => candidate.matches(request)); + if (matchingRoutes.length !== 1) { + throw new Error( + `Expected one route for ${request.method} ${request.url}, matched ${matchingRoutes.length}`, + ); + } + return matchingRoutes[0]!.respond(request); + }) as typeof fetch; +} From 466b4fe50c7f50e5566e15dd6a5ea13863de50f3 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Fri, 10 Jul 2026 15:34:37 -0700 Subject: [PATCH 3/7] fix(ci): isolate untrusted workflow metadata --- .github/workflows/required-live-e2e.yaml | 21 ++++-- test/required-live-workflow.test.ts | 83 +++++++++++++++++++++--- tools/e2e/required-live.mts | 3 + 3 files changed, 91 insertions(+), 16 deletions(-) diff --git a/.github/workflows/required-live-e2e.yaml b/.github/workflows/required-live-e2e.yaml index c2c75c6d935..e83791d268c 100644 --- a/.github/workflows/required-live-e2e.yaml +++ b/.github/workflows/required-live-e2e.yaml @@ -38,10 +38,11 @@ jobs: - name: Cancel superseded required live runs env: GITHUB_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} run: >- node --experimental-strip-types tools/e2e/required-live.mts --mode cancel - --pr "${{ github.event.pull_request.number }}" + --pr "$PR_NUMBER" coordinate: if: ${{ github.event_name == 'workflow_run' && github.repository == 'NVIDIA/NemoClaw' && github.event.workflow_run.event == 'pull_request' }} @@ -83,16 +84,22 @@ jobs: - id: start name: Start required live evaluation env: + CI_CONCLUSION: ${{ github.event.workflow_run.conclusion }} GITHUB_TOKEN: ${{ github.token }} + HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + HEAD_REPOSITORY: ${{ github.event.workflow_run.head_repository.full_name }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + WORK_DIR: ${{ steps.workspace.outputs.work_dir }} run: >- node --experimental-strip-types tools/e2e/required-live.mts --mode start - --head "${{ github.event.workflow_run.head_sha }}" - --head-repo "${{ github.event.workflow_run.head_repository.full_name }}" - --head-branch "${{ github.event.workflow_run.head_branch }}" - --workflow-sha "${{ github.workflow_sha }}" - --ci-conclusion "${{ github.event.workflow_run.conclusion }}" - --work-dir "${{ steps.workspace.outputs.work_dir }}" + --head "$HEAD_SHA" + --head-repo "$HEAD_REPOSITORY" + --head-branch "$HEAD_BRANCH" + --workflow-sha "$WORKFLOW_SHA" + --ci-conclusion "$CI_CONCLUSION" + --work-dir "$WORK_DIR" - name: Upload required live plan if: ${{ always() && steps.workspace.outputs.work_dir != '' }} diff --git a/test/required-live-workflow.test.ts b/test/required-live-workflow.test.ts index 96feaefd76f..0e57d89c748 100644 --- a/test/required-live-workflow.test.ts +++ b/test/required-live-workflow.test.ts @@ -121,6 +121,45 @@ exec "$@" } } +function runStartStep(headBranch: string) { + const workflow = readYaml(REQUIRED_LIVE_PATH); + const start = step(workflow.jobs.coordinate, "Start required live evaluation"); + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-required-live-start-step-")); + const binDir = path.join(tempDir, "bin"); + const argumentsPath = path.join(tempDir, "node-arguments"); + fs.mkdirSync(binDir); + fs.writeFileSync( + path.join(binDir, "node"), + '#!/usr/bin/env bash\nset -euo pipefail\nprintf \'%s\\0\' "$@" > "$FAKE_NODE_ARGUMENTS"\n', + { mode: 0o755 }, + ); + + try { + const result = spawnSync("bash", ["-e", "-o", "pipefail", "-c", start.run!], { + encoding: "utf8", + env: { + ...process.env, + CI_CONCLUSION: "success", + FAKE_NODE_ARGUMENTS: argumentsPath, + GITHUB_TOKEN: "token", + HEAD_BRANCH: headBranch, + HEAD_REPOSITORY: "NVIDIA/NemoClaw", + HEAD_SHA: "a".repeat(40), + PATH: `${binDir}:${process.env.PATH ?? ""}`, + WORKFLOW_SHA: "d".repeat(40), + WORK_DIR: tempDir, + }, + timeout: 5_000, + }); + return { + arguments: fs.readFileSync(argumentsPath, "utf8").split("\0").slice(0, -1), + result, + }; + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +} + function runChildValidation(currentPullSha: string) { const workflow = readYaml(E2E_PATH); const validation = step(workflow.jobs["generate-matrix"], "Validate required-live dispatch"); @@ -243,8 +282,27 @@ describe("required live E2E workflow", () => { const cancelStep = step(cancel, "Cancel superseded required live runs"); expect(cancelStep.run).toContain("tools/e2e/required-live.mts --mode cancel"); - expect(cancelStep.run).toContain('--pr "${{ github.event.pull_request.number }}"'); + expect(cancelStep.run).toContain('--pr "$PR_NUMBER"'); + expect(cancelStep.run).not.toContain("${{ github.event."); expect(cancelStep.env?.GITHUB_TOKEN).toBe("${{ github.token }}"); + expect(cancelStep.env?.PR_NUMBER).toBe("${{ github.event.pull_request.number }}"); + }); + + it.each([ + ["a single quote", "feature/'quoted"], + ["a double quote", 'feature/"quoted'], + ["command substitution", "feature/$(printf injected)"], + ["a semicolon", "feature/branch;printf injected"], + ["whitespace", "feature/space name"], + ["a newline", "feature/line\nname"], + ])("passes branch text containing $label as one inert shell argument", (_label, headBranch) => { + const execution = runStartStep(headBranch); + const branchFlag = execution.arguments.indexOf("--head-branch"); + + expect(execution.result.status).toBe(0); + expect(execution.result.stderr).toBe(""); + expect(execution.arguments.filter((argument) => argument === "--head-branch")).toHaveLength(1); + expect(execution.arguments[branchFlag + 1]).toBe(headBranch); }); it("coordinates one check lifecycle around a bounded child run", () => { @@ -268,14 +326,21 @@ describe("required live E2E workflow", () => { expect(workspace.run).toContain('mktemp -d "${RUNNER_TEMP}/nemoclaw-required-live.XXXXXX"'); expect(workspace.run).toContain('chmod 700 "$work_dir"'); expect(start.run).toContain("tools/e2e/required-live.mts --mode start"); - expect(start.run).toContain('--head "${{ github.event.workflow_run.head_sha }}"'); - expect(start.run).toContain( - '--head-repo "${{ github.event.workflow_run.head_repository.full_name }}"', - ); - expect(start.run).toContain('--head-branch "${{ github.event.workflow_run.head_branch }}"'); - expect(start.run).toContain('--workflow-sha "${{ github.workflow_sha }}"'); - expect(start.run).toContain('--ci-conclusion "${{ github.event.workflow_run.conclusion }}"'); - expect(start.run).toContain('--work-dir "${{ steps.workspace.outputs.work_dir }}"'); + expect(start.run).toContain('--head "$HEAD_SHA"'); + expect(start.run).toContain('--head-repo "$HEAD_REPOSITORY"'); + expect(start.run).toContain('--head-branch "$HEAD_BRANCH"'); + expect(start.run).toContain('--workflow-sha "$WORKFLOW_SHA"'); + expect(start.run).toContain('--ci-conclusion "$CI_CONCLUSION"'); + expect(start.run).toContain('--work-dir "$WORK_DIR"'); + expect(start.run).not.toContain("${{ github.event."); + expect(start.env).toMatchObject({ + CI_CONCLUSION: "${{ github.event.workflow_run.conclusion }}", + HEAD_BRANCH: "${{ github.event.workflow_run.head_branch }}", + HEAD_REPOSITORY: "${{ github.event.workflow_run.head_repository.full_name }}", + HEAD_SHA: "${{ github.event.workflow_run.head_sha }}", + WORKFLOW_SHA: "${{ github.workflow_sha }}", + WORK_DIR: "${{ steps.workspace.outputs.work_dir }}", + }); expect(start.run).not.toContain("--mode initialize"); expect(upload.if).toContain("steps.workspace.outputs.work_dir != ''"); expect(upload.with?.path).toBe( diff --git a/tools/e2e/required-live.mts b/tools/e2e/required-live.mts index 3d18e869b02..5993d18c61a 100755 --- a/tools/e2e/required-live.mts +++ b/tools/e2e/required-live.mts @@ -460,6 +460,9 @@ function appendOutput(name: string, value: string): void { ); try { if (!fs.fstatSync(descriptor).isFile()) throw new Error("GITHUB_OUTPUT must be a regular file"); + // lgtm[js/network-data-to-file] Values are reduced to a strict single-line allowlist above, + // and the trusted runner-owned output file is opened without following symlinks. + // lgtm[js/http-to-file-access] fs.writeFileSync(descriptor, `${name}=${value}\n`, "utf8"); } finally { fs.closeSync(descriptor); From e6366d69790ef8780b7cf132be427fea72395020 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Fri, 10 Jul 2026 15:47:02 -0700 Subject: [PATCH 4/7] docs(e2e): clarify required dispatch inputs --- test/e2e/README.md | 2 +- tools/e2e-advisor/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index 7bd4b156d37..f1d849caf06 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -76,7 +76,7 @@ checks out the pull request revision supplied through `checkout_sha`. Before E2E preparation or selected jobs can use repository secrets, the child workflow verifies that the pull request is still open, comes from `NVIDIA/NemoClaw`, and still points to that revision. -It also accepts only selective job dispatches with an empty target fan-out and +It also accepts only selective job dispatches without the `targets` input and valid plan and correlation metadata. GitHub returns the dispatched workflow's run ID directly, and the controller uses that ID as the sole child-run selector for waiting, evidence download, diff --git a/tools/e2e-advisor/README.md b/tools/e2e-advisor/README.md index b7ebed364ab..dfadf98c1f2 100644 --- a/tools/e2e-advisor/README.md +++ b/tools/e2e-advisor/README.md @@ -58,7 +58,7 @@ only a child workflow run created from that same revision. The child workflow validates that the pull request is still open, belongs to the base repository, and still points to the requested checkout SHA before E2E preparation or secret-bearing jobs can run. -It also requires selective jobs, an empty target fan-out, a valid plan hash, +It also requires selective jobs with no `targets` input, a valid plan hash, and a valid correlation ID. The controller uses GitHub's returned workflow run ID as the sole child-run selector for waiting, evidence download, and completion. From 3bee59c2a48c08aa931fcad443564c359742c27e Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Fri, 10 Jul 2026 16:26:32 -0700 Subject: [PATCH 5/7] refactor(e2e): simplify PR gate terminology --- .github/workflows/e2e.yaml | 18 +- ...equired-live-e2e.yaml => pr-e2e-gate.yaml} | 62 +++---- docs/about/release-notes.mdx | 2 +- test/e2e-risk-signal-reporter.test.ts | 2 +- test/e2e/README.md | 68 +++---- test/e2e/docs/README.md | 38 ++-- .../e2e-operations-workflow-boundary.test.ts | 22 +-- ...w.test.ts => pr-e2e-gate-workflow.test.ts} | 89 +++++---- ...uired-live.test.ts => pr-e2e-gate.test.ts} | 170 +++++++++++------- tools/e2e-advisor/README.md | 72 ++------ tools/e2e/operations-workflow-boundary.mts | 47 +++-- .../{required-live.mts => pr-e2e-gate.mts} | 170 +++++++++--------- tools/pr-review-advisor/README.md | 11 +- 13 files changed, 369 insertions(+), 402 deletions(-) rename .github/workflows/{required-live-e2e.yaml => pr-e2e-gate.yaml} (74%) rename test/{required-live-workflow.test.ts => pr-e2e-gate-workflow.test.ts} (83%) rename test/{required-live.test.ts => pr-e2e-gate.test.ts} (87%) rename tools/e2e/{required-live.mts => pr-e2e-gate.mts} (86%) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 1d1853c3541..9b8f7f0f51c 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 name: E2E -run-name: "${{ inputs.checkout_sha != '' && format('E2E PR #{0} required live {1}', inputs.pr_number, inputs.correlation_id) || format('E2E {0}', github.ref_name) }}" +run-name: "${{ inputs.checkout_sha != '' && format('E2E PR #{0} ({1})', inputs.pr_number, inputs.correlation_id) || format('E2E {0}', github.ref_name) }}" on: schedule: @@ -35,17 +35,17 @@ on: default: false type: boolean checkout_sha: - description: Immutable pull request commit selected by the trusted required-live controller. + description: PR head commit selected by the controller. required: false default: "" type: string plan_hash: - description: Deterministic required-live plan hash for evidence correlation. + description: SHA-256 of the selected E2E plan. required: false default: "" type: string correlation_id: - description: UUIDv4 correlation id for a required-live run. + description: Run correlation ID (UUIDv4). required: false default: "" type: string @@ -55,7 +55,7 @@ permissions: pull-requests: read concurrency: - group: e2e-${{ github.ref }}-${{ inputs.checkout_sha != '' && format('pr-{0}', inputs.pr_number) || inputs.targets || 'supported' }}-${{ inputs.checkout_sha != '' && 'required-live' || inputs.jobs || 'all-jobs' }} + group: e2e-${{ github.ref }}-${{ inputs.checkout_sha != '' && format('pr-{0}', inputs.pr_number) || inputs.targets || 'supported' }}-${{ inputs.checkout_sha != '' && 'pr-gate' || inputs.jobs || 'all-jobs' }} cancel-in-progress: ${{ inputs.checkout_sha != '' }} env: @@ -78,7 +78,7 @@ jobs: fetch-depth: 0 persist-credentials: false - - name: Validate required-live dispatch + - name: Validate controller dispatch if: ${{ inputs.checkout_sha != '' }} env: CHECKOUT_SHA: ${{ inputs.checkout_sha }} @@ -92,13 +92,13 @@ jobs: WORKFLOW_REF: ${{ github.ref }} run: | set -euo pipefail - [[ "$WORKFLOW_EVENT" == "workflow_dispatch" && "$WORKFLOW_REF" == "refs/heads/main" ]] || { echo "::error::required-live runs require a workflow_dispatch from main"; exit 1; } + [[ "$WORKFLOW_EVENT" == "workflow_dispatch" && "$WORKFLOW_REF" == "refs/heads/main" ]] || { echo "::error::PR E2E runs must be dispatched from main"; exit 1; } [[ "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ ]] || { echo "::error::checkout_sha must be a lowercase 40-character SHA"; exit 1; } [[ "$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]] || { echo "::error::checked-out commit does not match checkout_sha"; exit 1; } [[ "$PLAN_HASH" =~ ^[a-f0-9]{64}$ ]] || { echo "::error::plan_hash must be a lowercase SHA-256"; exit 1; } [[ "$CORRELATION_ID" =~ ^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$ ]] || { echo "::error::correlation_id must be a lowercase UUIDv4"; exit 1; } [[ "$PR_NUMBER" =~ ^[1-9][0-9]*$ ]] || { echo "::error::pr_number must be a positive integer"; exit 1; } - [[ -n "$JOBS" && -z "$TARGETS" ]] || { echo "::error::required-live runs require selective jobs and forbid targets/fan-out"; exit 1; } + [[ -n "$JOBS" && -z "$TARGETS" ]] || { echo "::error::PR E2E runs require jobs and do not accept targets"; exit 1; } pull_json="$(curl --fail --silent --show-error --proto '=https' \ --header "Authorization: Bearer ${GITHUB_TOKEN}" \ @@ -107,7 +107,7 @@ jobs: "https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}")" [[ "$(jq -r '.state' <<< "$pull_json")" == "open" ]] || { echo "::error::pull request must still be open"; exit 1; } [[ "$(jq -r '.head.repo.full_name // ""' <<< "$pull_json")" == "$GITHUB_REPOSITORY" ]] || { echo "::error::pull request must originate from this repository"; exit 1; } - [[ "$(jq -r '.head.sha' <<< "$pull_json")" == "$CHECKOUT_SHA" ]] || { echo "::error::checkout_sha must match the pull request's current commit"; exit 1; } + [[ "$(jq -r '.head.sha' <<< "$pull_json")" == "$CHECKOUT_SHA" ]] || { echo "::error::checkout_sha must match the PR head commit"; exit 1; } - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 diff --git a/.github/workflows/required-live-e2e.yaml b/.github/workflows/pr-e2e-gate.yaml similarity index 74% rename from .github/workflows/required-live-e2e.yaml rename to .github/workflows/pr-e2e-gate.yaml index e83791d268c..e021c998042 100644 --- a/.github/workflows/required-live-e2e.yaml +++ b/.github/workflows/pr-e2e-gate.yaml @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -name: E2E / Required Live +name: E2E / PR Gate on: workflow_run: @@ -21,7 +21,7 @@ jobs: actions: write contents: read steps: - - name: Checkout trusted controller + - name: Checkout controller uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: ref: ${{ github.workflow_sha }} @@ -32,15 +32,15 @@ jobs: with: node-version: "22" - - name: Install trusted controller dependencies + - name: Install controller dependencies run: npm ci --ignore-scripts - - name: Cancel superseded required live runs + - name: Cancel superseded E2E runs env: GITHUB_TOKEN: ${{ github.token }} PR_NUMBER: ${{ github.event.pull_request.number }} run: >- - node --experimental-strip-types tools/e2e/required-live.mts + node --experimental-strip-types tools/e2e/pr-e2e-gate.mts --mode cancel --pr "$PR_NUMBER" @@ -54,11 +54,11 @@ jobs: contents: read pull-requests: read concurrency: - group: required-live-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }} - # Let an older coordinator observe child cancellation and close its check. + group: pr-e2e-gate-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }} + # Let the previous coordinator observe E2E cancellation and close its check. cancel-in-progress: false steps: - - name: Checkout trusted controller + - name: Checkout controller uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: ref: ${{ github.workflow_sha }} @@ -69,20 +69,20 @@ jobs: with: node-version: "22" - - name: Install trusted controller dependencies + - name: Install controller dependencies run: npm ci --ignore-scripts - id: workspace - name: Create private controller workspace + name: Create private workspace shell: bash run: | set -euo pipefail - work_dir="$(mktemp -d "${RUNNER_TEMP}/nemoclaw-required-live.XXXXXX")" + work_dir="$(mktemp -d "${RUNNER_TEMP}/nemoclaw-pr-e2e-gate.XXXXXX")" chmod 700 "$work_dir" printf 'work_dir=%s\n' "$work_dir" >> "$GITHUB_OUTPUT" - id: start - name: Start required live evaluation + name: Start evaluation env: CI_CONCLUSION: ${{ github.event.workflow_run.conclusion }} GITHUB_TOKEN: ${{ github.token }} @@ -92,7 +92,7 @@ jobs: WORKFLOW_SHA: ${{ github.workflow_sha }} WORK_DIR: ${{ steps.workspace.outputs.work_dir }} run: >- - node --experimental-strip-types tools/e2e/required-live.mts + node --experimental-strip-types tools/e2e/pr-e2e-gate.mts --mode start --head "$HEAD_SHA" --head-repo "$HEAD_REPOSITORY" @@ -101,17 +101,17 @@ jobs: --ci-conclusion "$CI_CONCLUSION" --work-dir "$WORK_DIR" - - name: Upload required live plan + - name: Upload risk plan if: ${{ always() && steps.workspace.outputs.work_dir != '' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: required-live-plan-${{ github.event.workflow_run.head_sha }} - path: ${{ steps.workspace.outputs.work_dir }}/required-live-plan.json + name: pr-e2e-risk-plan-${{ github.event.workflow_run.head_sha }} + path: ${{ steps.workspace.outputs.work_dir }}/risk-plan.json if-no-files-found: ignore retention-days: 14 - id: wait - name: Wait for required live run + name: Wait for E2E run if: ${{ steps.start.outputs.dispatched == 'true' }} continue-on-error: true env: @@ -123,7 +123,7 @@ jobs: set -euo pipefail if [[ ! "$RUN_ID" =~ ^[1-9][0-9]*$ ]]; then - printf '::error title=Invalid required live run ID::The controller did not provide a positive numeric run ID.\n' >&2 + printf '::error title=Invalid run ID::The controller did not provide a positive numeric run ID.\n' >&2 exit 1 fi @@ -135,7 +135,7 @@ jobs: --json status,conclusion \ --jq '.status + ":" + (if (.conclusion == null or .conclusion == "") then "none" else .conclusion end)' )"; then - printf '::error title=Required live status query failed::Unable to query run %s. %s\n' \ + printf '::error title=Run status query failed::Unable to query run %s. %s\n' \ "$RUN_ID" "$run_url" >&2 exit 1 fi @@ -143,19 +143,19 @@ jobs: if [[ "$state" != "$last_state" ]]; then case "$state" in queued:none | in_progress:none | requested:none | waiting:none | pending:none) - printf 'Required live run %s status=%s url=%s\n' \ + printf 'Run %s status=%s url=%s\n' \ "$RUN_ID" "${state%%:*}" "$run_url" ;; completed:success) - printf 'Required live run %s status=completed conclusion=success url=%s\n' \ + printf 'Run %s status=completed conclusion=success url=%s\n' \ "$RUN_ID" "$run_url" ;; completed:failure | completed:cancelled | completed:timed_out | completed:action_required | completed:neutral | completed:skipped | completed:stale | completed:startup_failure) - printf '::error title=Required live run did not succeed::Run %s completed with conclusion %s. %s\n' \ + printf '::error title=E2E run failed::Run %s completed with conclusion %s. %s\n' \ "$RUN_ID" "${state#*:}" "$run_url" >&2 ;; *) - printf '::error title=Unexpected required live state::Run %s returned an unsupported status/conclusion pair. %s\n' \ + printf '::error title=Unexpected run state::Run %s returned an unsupported status/conclusion pair. %s\n' \ "$RUN_ID" "$run_url" >&2 ;; esac @@ -177,12 +177,12 @@ jobs: WAIT if [ "$wait_status" -eq 124 ]; then - printf '::error title=Required live wait timed out::The run did not complete within 105 minutes.\n' >&2 + printf '::error title=E2E run timed out::The run did not complete within 105 minutes.\n' >&2 fi exit "$wait_status" - id: evidence - name: Download required live evidence + name: Download evidence if: ${{ always() && steps.start.outputs.dispatched == 'true' }} continue-on-error: true env: @@ -195,33 +195,33 @@ jobs: gh run download "$RUN_ID" --repo "$GITHUB_REPOSITORY" \ --dir "${{ steps.workspace.outputs.work_dir }}/evidence" || download_status=$? if [ "$download_status" -eq 124 ]; then - printf '::error title=Required live evidence download timed out::Artifact download exceeded 10 minutes.\n' >&2 + printf '::error title=Evidence download timed out::Artifact download exceeded 10 minutes.\n' >&2 fi exit "$download_status" - id: finish - name: Finish required live evaluation + name: Verify evidence if: ${{ always() && steps.start.outputs.dispatched == 'true' }} env: GITHUB_TOKEN: ${{ github.token }} run: >- - node --experimental-strip-types tools/e2e/required-live.mts + node --experimental-strip-types tools/e2e/pr-e2e-gate.mts --mode finish --work-dir "${{ steps.workspace.outputs.work_dir }}" --state-hash "${{ steps.start.outputs.state_hash }}" --check-id "${{ steps.start.outputs.check_id }}" --run-id "${{ steps.start.outputs.run_id }}" - - name: Close incomplete required live check + - name: Close incomplete check if: ${{ always() && steps.start.outputs.check_id != '' && steps.start.outputs.finalized != 'true' && steps.finish.outputs.finalized != 'true' }} env: GITHUB_TOKEN: ${{ github.token }} run: >- - node --experimental-strip-types tools/e2e/required-live.mts + node --experimental-strip-types tools/e2e/pr-e2e-gate.mts --mode abandon --check-id "${{ steps.start.outputs.check_id }}" --run-id "${{ steps.start.outputs.run_id }}" - - name: Remove private controller workspace + - name: Remove private workspace if: ${{ always() && steps.workspace.outputs.work_dir != '' }} run: rm -rf -- "${{ steps.workspace.outputs.work_dir }}" diff --git a/docs/about/release-notes.mdx b/docs/about/release-notes.mdx index e4ecc91df92..8db02d3e063 100644 --- a/docs/about/release-notes.mdx +++ b/docs/about/release-notes.mdx @@ -42,7 +42,7 @@ The release also refreshes quickstarts and variant rendering so OpenClaw, Hermes Resume recovery now follows one explicit finite-state path, pending route reservations survive resume, sandbox create-failure reporting is separated from create-step handling, BuildKit progress no longer forces plain output, and null-name resume sessions are covered so canceled or malformed session state does not send users down the wrong recovery path. For more information, refer to [NemoClaw Quickstart with OpenClaw](../get-started/quickstart), [NemoClaw CLI Commands Reference](../reference/commands), [Manage Sandbox Lifecycle](../manage-sandboxes/lifecycle), and [Troubleshooting](../reference/troubleshooting). - Documentation and release validation are more deterministic. - The docs define extension taxonomy and SDK readiness gates, streamline the agent quickstarts, clarify legacy k3s sandbox resources, preserve list spacing in generated agent-variant pages, and add release-train risk planning with a deterministic required live E2E check for pull requests, queued Jetson dispatch guards, TUI idle regression coverage, and reusable live-readiness polling primitives. + The docs define extension taxonomy and SDK readiness gates, streamline the agent quickstarts, clarify legacy k3s sandbox resources, preserve list spacing in generated agent-variant pages, and add release-train risk planning with a PR E2E check, queued Jetson dispatch guards, TUI idle regression coverage, and reusable live-readiness polling primitives. For more information, refer to [Extension Taxonomy and SDK Readiness](../reference/extension-taxonomy-sdk-readiness), [NemoClaw Quickstart with OpenClaw](../get-started/quickstart), [Architecture Details](../reference/architecture), and the [NemoClaw E2E README](https://github.com/NVIDIA/NemoClaw/blob/main/test/e2e/README.md). ## v0.0.78 diff --git a/test/e2e-risk-signal-reporter.test.ts b/test/e2e-risk-signal-reporter.test.ts index 2be2eaf2abd..61cf4e654cc 100644 --- a/test/e2e-risk-signal-reporter.test.ts +++ b/test/e2e-risk-signal-reporter.test.ts @@ -45,7 +45,7 @@ describe("E2E risk signal reporter", () => { expect(configuredEnvironment({})).toBeNull(); }); - it("fails closed when required-live metadata is incomplete", () => { + it("fails closed when run metadata is incomplete", () => { expect(() => configuredEnvironment({ NEMOCLAW_E2E_EXPECTED_SHA: EXPECTED_SHA })).toThrow( /E2E_ARTIFACT_DIR/u, ); diff --git a/test/e2e/README.md b/test/e2e/README.md index f1d849caf06..4c7530d5030 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -10,8 +10,8 @@ before those targets run; local runners must provide it themselves. - `.github/workflows/e2e.yaml` is the scheduled, manually dispatchable, and selectively dispatched live target workflow. -- `.github/workflows/required-live-e2e.yaml` is the trusted pull request - controller that owns the required `E2E / Required Live` check. +- `.github/workflows/pr-e2e-gate.yaml` is the PR controller for + `E2E / PR Gate`. - `.github/workflows/e2e-branch-validation.yaml` provisions Brev instances and runs focused E2E targets from source on a clean machine. - Platform workflows such as macOS, WSL, Ollama proxy, sandbox image, and @@ -55,32 +55,23 @@ artifact so baseline aggregation stays stable. Older issue references to Vitest target artifacts under `e2e-artifacts/vitest/` map to this consolidated `e2e-artifacts/live/` registry-target artifact layout. -## Required live PR check - -When `CI / Pull Request` completes for a same-repository pull request, the -trusted `.github/workflows/required-live-e2e.yaml` workflow creates the -`E2E / Required Live` check for that revision. -The model-independent controller resolves the open pull request, reads its -complete changed-file list from GitHub, and builds the deterministic risk plan. -If runtime regression families match, it dispatches every selected -`requiredJobs` entry through `e2e.yaml`. -If no family matches, the check succeeds without dispatching live E2E. - -The controller verifies that the pull request did not change while the plan -was prepared. -It records the trusted workflow revision, requires that revision to remain the -current `main` revision immediately before dispatch, and accepts only a child -workflow run created from that same revision. -The `e2e.yaml` workflow definition stays on `main`, while each selected job -checks out the pull request revision supplied through `checkout_sha`. -Before E2E preparation or selected jobs can use repository secrets, the child -workflow verifies that the pull request is still open, comes from -`NVIDIA/NemoClaw`, and still points to that revision. -It also accepts only selective job dispatches without the `targets` input and -valid plan and correlation metadata. -GitHub returns the dispatched workflow's run ID directly, and the controller -uses that ID as the sole child-run selector for waiting, evidence download, -and completion. +## PR E2E check + +When `CI / Pull Request` completes for a PR from this repository, +`.github/workflows/pr-e2e-gate.yaml` creates `E2E / PR Gate` for the PR head +commit. The controller reads all changed files and builds the deterministic +risk plan. If a runtime risk family matches, it dispatches every selected +`requiredJobs` entry through `e2e.yaml`; otherwise the check passes without an +E2E run. + +Before dispatch, the controller verifies that the PR is unchanged and that +`main` still points to its workflow commit. It accepts only an E2E run using +that commit. Each selected job checks out `checkout_sha`. Before preparation or +secret-bearing jobs can run, `e2e.yaml` verifies that the PR remains open, +belongs to `NVIDIA/NemoClaw`, and still has that head commit. The dispatch +includes selected jobs and valid plan and correlation metadata, but not +`targets`. The controller uses GitHub's returned run ID for waiting, evidence +download, and completion. The Vitest reporter writes one `risk-signal.json` for each selected job and matrix shard. @@ -89,28 +80,23 @@ matching job identity, attach the reporter to every Vitest invocation, and always upload its evidence artifact. Each signal binds the observed checkout SHA, expected SHA, plan hash, correlation ID, and pass, failure, skip, pending, and unhandled-error counts. -The controller retains `required-live-plan-` for 14 days, while each +The controller retains `pr-e2e-risk-plan-` for 14 days, while each signal travels in the selected job's existing E2E artifact. Its private dispatch state is protected by a SHA-256 digest that is verified before downloaded evidence is classified. -The required check has a binary result. -It succeeds only when the correlated E2E workflow succeeds and every expected -job shard produces one complete, unskipped pass. -Workflow or test failures, missing or duplicate signals, skipped or pending -tests, interrupted runs, and controller or evidence-validation errors fail the -check. +When the plan selects jobs, the check passes only when the E2E run succeeds and +every expected job shard uploads one complete passing signal with no skips or +pending tests. Every other dispatched outcome fails. The coordinator has a 180-minute job budget and gives evidence download its own 10-minute limit, so a stalled download fails instead of consuming the remaining coordination time. -Required-live dispatches suppress PR comments and the scheduled or manual +These dispatches suppress PR comments and the scheduled or manual scorecard, including scorecard Slack reporting. -Pull request synchronization, reopening, or closure cancels active child runs -for that pull request. -The E2E workflow also cancels a superseded child run when a new revision is -dispatched, while the earlier controller remains available to close its check -as failed. +Synchronizing, reopening, or closing the PR cancels its active E2E runs. A new +dispatch also cancels the previous run, while the previous controller remains +available to close its check as failed. The controller does not read PR Review Advisor or E2E Advisor output, so model availability and recommendations are not part of merge authority. diff --git a/test/e2e/docs/README.md b/test/e2e/docs/README.md index c7e68d815ab..f4f155f0334 100644 --- a/test/e2e/docs/README.md +++ b/test/e2e/docs/README.md @@ -106,21 +106,17 @@ test/e2e/ ## CI Entry Points - `tools/advisors/risk-plan.mts` is the small deterministic selection policy - shared by PR Review Advisor, E2E Advisor, and the model-independent required - live controller. It maps changed runtime surfaces to invariant families and + shared by PR Review Advisor, E2E Advisor, and the PR E2E controller. It maps + changed runtime surfaces to invariant families and canonical `e2e.yaml` jobs; it is not a second test runner or migration-status - ledger. The advisors use it as recommendation context, while the required - controller applies it independently without model output. - -- `.github/workflows/required-live-e2e.yaml` owns the required - `E2E / Required Live` check for same-repository pull request revisions after - `CI / Pull Request` completes. `tools/e2e/required-live.mts` builds a plan - from GitHub's complete pull request file list, dispatches every selected job, - and validates `risk-signal.json` evidence for every expected job and matrix - shard. It also requires its trusted workflow revision to remain current on - `main` immediately before dispatch and verifies that the child run uses that - revision. Pull request synchronization, reopening, or closure cancels active - child runs for that pull request. + ledger. The advisors use it as recommendation context, while the controller + applies it independently without model output. + +- `.github/workflows/pr-e2e-gate.yaml` owns `E2E / PR Gate` for PRs from this + repository after `CI / Pull Request` completes. The controller builds the + risk plan from GitHub's complete file list, dispatches every selected job, + and verifies each expected `risk-signal.json`. See + [NemoClaw E2E CI](../README.md) for the full lifecycle. - `.github/workflows/e2e.yaml` runs selected or all supported live E2E targets and uploads an explicit artifact allowlist with @@ -133,16 +129,10 @@ test/e2e/ These per-target timing summaries are artifact evidence only. The Slack and GitHub scorecard timing comparison remains scoped to the dedicated `cloud-onboard` artifact. - Required-live dispatches require an open pull request from the base - repository whose current revision matches `checkout_sha` before preparation. - The controller uses GitHub's returned workflow-dispatch run ID as the sole - child-run selector for waiting, evidence download, and completion, attaches - `test/e2e/risk-signal-reporter.ts` to live Vitest invocations, and suppresses - PR reporting and scorecards. The workflow boundary requires every job named - by the deterministic policy to expose matching job identity, attach that - reporter to every Vitest invocation, and always upload one evidence artifact. - The check succeeds only for one complete, unskipped passing signal from every - expected job shard; every other outcome is a failure. + PR E2E dispatches validate the PR head commit and controller metadata before + preparation, attach `test/e2e/risk-signal-reporter.ts` to live Vitest + invocations, and suppress PR reporting and scorecards. The workflow boundary + requires every selected job shard to upload its evidence artifact. - `.github/workflows/e2e-branch-validation.yaml`, `macos-e2e.yaml`, `wsl-e2e.yaml`, `ollama-proxy-e2e.yaml`, and `regression-e2e.yaml` call focused E2E targets directly for their E2E coverage. diff --git a/test/e2e/support/e2e-operations-workflow-boundary.test.ts b/test/e2e/support/e2e-operations-workflow-boundary.test.ts index 6b8ebe26285..44f68f62cc9 100644 --- a/test/e2e/support/e2e-operations-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-operations-workflow-boundary.test.ts @@ -57,7 +57,7 @@ describe("E2E operations workflow boundary", () => { ); }); - it("keeps PR reporting and scorecards disabled for required-live runs", () => { + it("keeps PR reporting and scorecards disabled for PR E2E runs", () => { const workflow = readE2eOperationsWorkflow(); workflow.jobs["report-to-pr"].if = "${{ always() && github.event_name == 'workflow_dispatch' }}"; @@ -72,13 +72,13 @@ describe("E2E operations workflow boundary", () => { ); }); - it("rejects required-live protocol and pull request validation drift", () => { + it("rejects controller protocol and PR validation drift", () => { const workflow = readE2eOperationsWorkflow(); delete workflow.on?.workflow_dispatch?.inputs?.plan_hash; workflow.env!.NEMOCLAW_E2E_PLAN_HASH = "${{ inputs.checkout_sha }}"; workflow.concurrency!["cancel-in-progress"] = false; const validation = workflow.jobs["generate-matrix"].steps!.find( - (step) => step.name === "Validate required-live dispatch", + (step) => step.name === "Validate controller dispatch", )!; validation.if = "${{ inputs.plan_hash != '' }}"; validation.run = "echo unchecked"; @@ -90,11 +90,11 @@ describe("E2E operations workflow boundary", () => { expect(validateE2eOperationsWorkflow(workflow)).toEqual( expect.arrayContaining([ "workflow_dispatch plan_hash must be an optional string with an empty default", - "E2E workflow must bind NEMOCLAW_E2E_PLAN_HASH to required-live metadata", - "required-live concurrency must cancel obsolete pull request runs", - "required-live validation must be activated only by checkout_sha", - 'required-live validation must retain "$PR_NUMBER" =~ ^[1-9][0-9]*$', - "generate-matrix checkout must use the selected immutable commit", + "E2E workflow must bind NEMOCLAW_E2E_PLAN_HASH to controller metadata", + "PR E2E concurrency must cancel obsolete runs", + "Controller validation must be activated only by checkout_sha", + 'Controller validation must retain "$PR_NUMBER" =~ ^[1-9][0-9]*$', + "generate-matrix checkout must use the selected PR commit", ]), ); }); @@ -112,9 +112,9 @@ describe("E2E operations workflow boundary", () => { expect(validateE2eOperationsWorkflow(workflow)).toEqual( expect.arrayContaining([ - "cloud-onboard must expose matching required-live job identity", - "cloud-onboard must attach the required-live reporter to every Vitest invocation", - "cloud-onboard must always upload one required-live evidence artifact", + "cloud-onboard must expose matching E2E job identity", + "cloud-onboard must attach the risk-signal reporter to every Vitest invocation", + "cloud-onboard must always upload one evidence artifact", ]), ); }); diff --git a/test/required-live-workflow.test.ts b/test/pr-e2e-gate-workflow.test.ts similarity index 83% rename from test/required-live-workflow.test.ts rename to test/pr-e2e-gate-workflow.test.ts index 0e57d89c748..5c9d6961437 100644 --- a/test/required-live-workflow.test.ts +++ b/test/pr-e2e-gate-workflow.test.ts @@ -14,7 +14,7 @@ import { type WorkflowStep, } from "./helpers/e2e-workflow-contract.ts"; -const REQUIRED_LIVE_PATH = ".github/workflows/required-live-e2e.yaml"; +const PR_GATE_PATH = ".github/workflows/pr-e2e-gate.yaml"; const E2E_PATH = ".github/workflows/e2e.yaml"; type CoordinatorJob = WorkflowJob & { @@ -60,9 +60,9 @@ function runWaitStep( scenario: "success" | "failure" | "query-failure" | "timeout" | "unsupported", options: { runId?: string } = {}, ) { - const workflow = readYaml(REQUIRED_LIVE_PATH); - const wait = step(workflow.jobs.coordinate, "Wait for required live run"); - const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-required-live-wait-")); + const workflow = readYaml(PR_GATE_PATH); + const wait = step(workflow.jobs.coordinate, "Wait for E2E run"); + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pr-e2e-gate-wait-")); const binDir = path.join(tempDir, "bin"); const callCountPath = path.join(tempDir, "gh-call-count"); fs.mkdirSync(binDir); @@ -122,9 +122,9 @@ exec "$@" } function runStartStep(headBranch: string) { - const workflow = readYaml(REQUIRED_LIVE_PATH); - const start = step(workflow.jobs.coordinate, "Start required live evaluation"); - const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-required-live-start-step-")); + const workflow = readYaml(PR_GATE_PATH); + const start = step(workflow.jobs.coordinate, "Start evaluation"); + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pr-e2e-gate-start-step-")); const binDir = path.join(tempDir, "bin"); const argumentsPath = path.join(tempDir, "node-arguments"); fs.mkdirSync(binDir); @@ -162,8 +162,8 @@ function runStartStep(headBranch: string) { function runChildValidation(currentPullSha: string) { const workflow = readYaml(E2E_PATH); - const validation = step(workflow.jobs["generate-matrix"], "Validate required-live dispatch"); - const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-required-live-child-")); + const validation = step(workflow.jobs["generate-matrix"], "Validate controller dispatch"); + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pr-e2e-gate-child-")); const binDir = path.join(tempDir, "bin"); fs.mkdirSync(binDir); fs.writeFileSync( @@ -215,13 +215,13 @@ esac } } -describe("required live E2E workflow", () => { - it("runs only from trusted lifecycle events with least-privilege jobs", () => { - const workflow = readYaml(REQUIRED_LIVE_PATH); +describe("PR E2E gate workflow", () => { + it("limits triggers and job permissions", () => { + const workflow = readYaml(PR_GATE_PATH); const cancel = workflow.jobs["cancel-superseded"]; const coordinate = workflow.jobs.coordinate; - expect(workflow.name).toBe("E2E / Required Live"); + expect(workflow.name).toBe("E2E / PR Gate"); expect(workflow.on).toEqual({ workflow_run: { workflows: ["CI / Pull Request"], @@ -246,7 +246,7 @@ describe("required live E2E workflow", () => { }); it("pins both controller checkouts and installs without lifecycle scripts or caches", () => { - const workflow = readYaml(REQUIRED_LIVE_PATH); + const workflow = readYaml(PR_GATE_PATH); const allSteps = Object.values(workflow.jobs).flatMap((job) => job.steps ?? []); const checkouts = allSteps.filter((candidate) => candidate.uses?.startsWith("actions/checkout@"), @@ -255,7 +255,7 @@ describe("required live E2E workflow", () => { candidate.uses?.startsWith("actions/setup-node@"), ); const installs = allSteps.filter( - (candidate) => candidate.name === "Install trusted controller dependencies", + (candidate) => candidate.name === "Install controller dependencies", ); expect(checkouts).toHaveLength(2); @@ -276,12 +276,12 @@ describe("required live E2E workflow", () => { ).toBe(false); }); - it("cancels superseded pull request runs through the trusted controller", () => { - const workflow = readYaml(REQUIRED_LIVE_PATH); + it("cancels superseded PR runs", () => { + const workflow = readYaml(PR_GATE_PATH); const cancel = workflow.jobs["cancel-superseded"]; - const cancelStep = step(cancel, "Cancel superseded required live runs"); + const cancelStep = step(cancel, "Cancel superseded E2E runs"); - expect(cancelStep.run).toContain("tools/e2e/required-live.mts --mode cancel"); + expect(cancelStep.run).toContain("tools/e2e/pr-e2e-gate.mts --mode cancel"); expect(cancelStep.run).toContain('--pr "$PR_NUMBER"'); expect(cancelStep.run).not.toContain("${{ github.event."); expect(cancelStep.env?.GITHUB_TOKEN).toBe("${{ github.token }}"); @@ -305,27 +305,27 @@ describe("required live E2E workflow", () => { expect(execution.arguments[branchFlag + 1]).toBe(headBranch); }); - it("coordinates one check lifecycle around a bounded child run", () => { - const workflow = readYaml(REQUIRED_LIVE_PATH); + it("coordinates the check around one E2E run", () => { + const workflow = readYaml(PR_GATE_PATH); const job = workflow.jobs.coordinate; - const workspace = step(job, "Create private controller workspace"); - const start = step(job, "Start required live evaluation"); - const upload = step(job, "Upload required live plan"); - const wait = step(job, "Wait for required live run"); - const download = step(job, "Download required live evidence"); - const finish = step(job, "Finish required live evaluation"); - const fallback = step(job, "Close incomplete required live check"); - const cleanup = step(job, "Remove private controller workspace"); + const workspace = step(job, "Create private workspace"); + const start = step(job, "Start evaluation"); + const upload = step(job, "Upload risk plan"); + const wait = step(job, "Wait for E2E run"); + const download = step(job, "Download evidence"); + const finish = step(job, "Verify evidence"); + const fallback = step(job, "Close incomplete check"); + const cleanup = step(job, "Remove private workspace"); expect(job.concurrency).toEqual({ group: - "required-live-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }}", + "pr-e2e-gate-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }}", "cancel-in-progress": false, }); expect(job["timeout-minutes"]).toBe(180); - expect(workspace.run).toContain('mktemp -d "${RUNNER_TEMP}/nemoclaw-required-live.XXXXXX"'); + expect(workspace.run).toContain('mktemp -d "${RUNNER_TEMP}/nemoclaw-pr-e2e-gate.XXXXXX"'); expect(workspace.run).toContain('chmod 700 "$work_dir"'); - expect(start.run).toContain("tools/e2e/required-live.mts --mode start"); + expect(start.run).toContain("tools/e2e/pr-e2e-gate.mts --mode start"); expect(start.run).toContain('--head "$HEAD_SHA"'); expect(start.run).toContain('--head-repo "$HEAD_REPOSITORY"'); expect(start.run).toContain('--head-branch "$HEAD_BRANCH"'); @@ -343,9 +343,8 @@ describe("required live E2E workflow", () => { }); expect(start.run).not.toContain("--mode initialize"); expect(upload.if).toContain("steps.workspace.outputs.work_dir != ''"); - expect(upload.with?.path).toBe( - "${{ steps.workspace.outputs.work_dir }}/required-live-plan.json", - ); + expect(upload.with?.name).toBe("pr-e2e-risk-plan-${{ github.event.workflow_run.head_sha }}"); + expect(upload.with?.path).toBe("${{ steps.workspace.outputs.work_dir }}/risk-plan.json"); expect(wait.run).toContain("timeout --signal=TERM --kill-after=30s 105m"); expect(wait.run).toContain('gh run view "$RUN_ID" --repo "$GITHUB_REPOSITORY"'); expect(wait.run).toContain("--json status,conclusion"); @@ -365,7 +364,7 @@ describe("required live E2E workflow", () => { expect(download.run).toContain('--dir "${{ steps.workspace.outputs.work_dir }}/evidence"'); expect(download["continue-on-error"]).toBe(true); expect(finish.if).toContain("always()"); - expect(finish.run).toContain("tools/e2e/required-live.mts --mode finish"); + expect(finish.run).toContain("tools/e2e/pr-e2e-gate.mts --mode finish"); expect(finish.run).toContain('--state-hash "${{ steps.start.outputs.state_hash }}"'); expect(finish.run).toContain('--check-id "${{ steps.start.outputs.check_id }}"'); expect(finish.run).toContain('--run-id "${{ steps.start.outputs.run_id }}"'); @@ -373,7 +372,7 @@ describe("required live E2E workflow", () => { expect(fallback.if).toContain("steps.start.outputs.check_id != ''"); expect(fallback.if).toContain("steps.start.outputs.finalized != 'true'"); expect(fallback.if).toContain("steps.finish.outputs.finalized != 'true'"); - expect(fallback.run).toContain("tools/e2e/required-live.mts --mode abandon"); + expect(fallback.run).toContain("tools/e2e/pr-e2e-gate.mts --mode abandon"); expect(fallback.run).toContain('--run-id "${{ steps.start.outputs.run_id }}"'); expect(cleanup.if).toContain("always() && steps.workspace.outputs.work_dir != ''"); expect(cleanup.run).toContain('rm -rf -- "${{ steps.workspace.outputs.work_dir }}"'); @@ -394,17 +393,17 @@ describe("required live E2E workflow", () => { }), ); expect(workflow["run-name"]).toContain( - "format('E2E PR #{0} required live {1}', inputs.pr_number, inputs.correlation_id)", + "format('E2E PR #{0} ({1})', inputs.pr_number, inputs.correlation_id)", ); }); - it("executes child validation against the pull request current revision", () => { + it("validates the E2E run against the PR head commit", () => { const current = runChildValidation("a".repeat(40)); const stale = runChildValidation("c".repeat(40)); expect(current.status).toBe(0); expect(stale.status).toBe(1); - expect(stale.stdout).toContain("checkout_sha must match the pull request's current commit"); + expect(stale.stdout).toContain("checkout_sha must match the PR head commit"); }); it("logs each child state once and exits after success", () => { @@ -423,7 +422,7 @@ describe("required live E2E workflow", () => { expect(result.status).toBe(1); expect(result.stdout.match(/status=in_progress/gu)).toHaveLength(1); - expect(result.stderr).toContain("::error title=Required live run did not succeed::"); + expect(result.stderr).toContain("::error title=E2E run failed::"); expect(result.stderr).toContain("completed with conclusion failure"); }); @@ -432,14 +431,14 @@ describe("required live E2E workflow", () => { expect(result.status).toBe(1); expect(result.stderr).toContain("simulated GitHub query failure"); - expect(result.stderr).toContain("::error title=Required live status query failed::"); + expect(result.stderr).toContain("::error title=Run status query failed::"); }); it("labels only the bounded wait exit as a timeout", () => { const result = runWaitStep("timeout"); expect(result.status).toBe(124); - expect(result.stderr).toContain("::error title=Required live wait timed out::"); + expect(result.stderr).toContain("::error title=E2E run timed out::"); expect(result.stderr).toContain("did not complete within 105 minutes"); }); @@ -448,7 +447,7 @@ describe("required live E2E workflow", () => { expect(result.status).toBe(1); expect(result.ghCallCount).toBe(0); - expect(result.stderr).toContain("::error title=Invalid required live run ID::"); + expect(result.stderr).toContain("::error title=Invalid run ID::"); }); it("fails closed for an unsupported child state", () => { @@ -456,6 +455,6 @@ describe("required live E2E workflow", () => { expect(result.status).toBe(1); expect(result.ghCallCount).toBe(1); - expect(result.stderr).toContain("::error title=Unexpected required live state::"); + expect(result.stderr).toContain("::error title=Unexpected run state::"); }); }); diff --git a/test/required-live.test.ts b/test/pr-e2e-gate.test.ts similarity index 87% rename from test/required-live.test.ts rename to test/pr-e2e-gate.test.ts index 6b61c24bf68..ed168200071 100644 --- a/test/required-live.test.ts +++ b/test/pr-e2e-gate.test.ts @@ -9,24 +9,24 @@ import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { buildRiskPlan, riskPlanRequiredJobIds } from "../tools/advisors/risk-plan.mts"; import { - abandonRequiredLive, + abandonPrGate, assertCorrelatedWorkflowRun, - cancelRequiredLive, - classifyRequiredLiveEvidence, - dispatchRequiredLive, + cancelPrGate, + classifyPrGateEvidence, + dispatchPrGate, expectedSignalShards, findSignalFiles, - finishRequiredLive, + finishPrGate, + type PrGateState, type PullRequest, parseControllerCommand, pullChangedFiles, - type RequiredLiveState, - startRequiredLive, - validateRequiredLiveState, + startPrGate, + validatePrGateState, validateRiskPlan, validateSignal, validateWorkflowDispatchDetails, -} from "../tools/e2e/required-live.mts"; +} from "../tools/e2e/pr-e2e-gate.mts"; import type { E2eRiskSignal } from "../tools/e2e/risk-signal.ts"; import { createGitHubFetchRouter, @@ -87,7 +87,7 @@ function pullRequest(changedFiles = 1): PullRequest { state: "open", changed_files: changedFiles, head: { - ref: "feature/required-live", + ref: "feature/pr-e2e-gate", sha: HEAD_SHA, repo: { full_name: "NVIDIA/NemoClaw" }, }, @@ -103,7 +103,7 @@ function pullRequestListItem(pull = pullRequest()): Omit = {}, @@ -165,7 +165,7 @@ function signal( }; } -function workflowRun(gate: RequiredLiveState, overrides: Record = {}) { +function workflowRun(gate: PrGateState, overrides: Record = {}) { return { id: 23, name: "E2E", @@ -175,15 +175,15 @@ function workflowRun(gate: RequiredLiveState, overrides: Record head_sha: gate.workflowSha, status: "completed", conclusion: "success", - display_title: `E2E PR #${gate.prNumber} required live ${gate.correlationId}`, + display_title: `E2E PR #${gate.prNumber} (${gate.correlationId})`, html_url: "https://github.com/NVIDIA/NemoClaw/actions/runs/23", ...overrides, }; } -describe("required live E2E controller", () => { +describe("PR E2E controller", () => { it("parses one lifecycle command set inside a private workspace", () => { - const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-required-live-")); + const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pr-e2e-gate-")); try { expect( parseControllerCommand([ @@ -194,7 +194,7 @@ describe("required live E2E controller", () => { "--head-repo", "NVIDIA/NemoClaw", "--head-branch", - "feature/required-live", + "feature/pr-e2e-gate", "--workflow-sha", WORKFLOW_SHA, "--ci-conclusion", @@ -204,8 +204,8 @@ describe("required live E2E controller", () => { ]), ).toMatchObject({ mode: "start", - planPath: path.join(workDir, "required-live-plan.json"), - statePath: path.join(workDir, "required-live-state.json"), + planPath: path.join(workDir, "risk-plan.json"), + statePath: path.join(workDir, "controller-state.json"), evidencePath: path.join(workDir, "evidence"), }); expect(parseControllerCommand(["--mode", "cancel", "--pr", "42"])).toEqual({ @@ -229,7 +229,7 @@ describe("required live E2E controller", () => { } }); - it("accepts only the current deterministic plan and bounded state", () => { + it("validates the risk plan and bounded state", () => { const plan = buildRiskPlan({ headSha: HEAD_SHA, changedFiles: ["src/lib/onboard.ts"] }); const allowed = new Set(riskPlanRequiredJobIds(plan)); const gate = state(); @@ -239,12 +239,12 @@ describe("required live E2E controller", () => { /unsupported risk-plan version/u, ); expect(() => validateRiskPlan({ ...plan, planHash: "b".repeat(64) }, allowed)).toThrow( - /deterministic hash/u, + /hash and inputs/u, ); expect(() => validateRiskPlan(plan, new Set())).toThrow(/unknown E2E job/u); - expect(validateRequiredLiveState(gate)).toEqual(gate); - expect(() => validateRequiredLiveState({ ...gate, prNumber: 0 })).toThrow(/PR number/u); - expect(() => validateRequiredLiveState({ ...gate, expectedShards: {} })).toThrow(/shard jobs/u); + expect(validatePrGateState(gate)).toEqual(gate); + expect(() => validatePrGateState({ ...gate, prNumber: 0 })).toThrow(/PR number/u); + expect(() => validatePrGateState({ ...gate, expectedShards: {} })).toThrow(/shard jobs/u); }); it("paginates canonical pull request files and includes both names for renames", async () => { @@ -280,7 +280,7 @@ describe("required live E2E controller", () => { const gate = state(); const complete = gate.expectedJobs.map((job) => signal(gate, job)); const classify = (signals: E2eRiskSignal[], workflowConclusion: string | null = "success") => - classifyRequiredLiveEvidence({ + classifyPrGateEvidence({ workflowConclusion, expectedJobs: gate.expectedJobs, expectedShards: gate.expectedShards, @@ -289,17 +289,17 @@ describe("required live E2E controller", () => { expect(classify(complete).conclusion).toBe("success"); expect(classify([], "cancelled").conclusion).toBe("failure"); - expect(classify(complete.slice(0, 1)).title).toMatch(/missing evidence/u); - expect(classify([...complete, complete[0]!]).title).toMatch(/duplicate evidence/u); + expect(classify(complete.slice(0, 1)).title).toBe("Evidence is missing"); + expect(classify([...complete, complete[0]!]).title).toBe("Duplicate evidence"); expect( classify([signal(gate, "onboard-repair", "default", { skipped: 1 }), complete[1]!]).title, - ).toMatch(/incomplete evidence/u); + ).toBe("Evidence is incomplete"); expect( classify([ signal(gate, "onboard-repair", "default", { failed: 1, runReason: "failed" }), complete[1]!, ]).title, - ).toMatch(/test failures/u); + ).toBe("Tests failed"); }); it("binds every signal to the revision, plan, correlation, job, and shard", () => { @@ -354,7 +354,7 @@ describe("required live E2E controller", () => { ); await expect( - dispatchRequiredLive({ + dispatchPrGate({ repository: "NVIDIA/NemoClaw", token: "token", jobs, @@ -391,7 +391,7 @@ describe("required live E2E controller", () => { ).toThrow(/mismatched workflow dispatch URLs/u); }); - it("refuses dispatch when main moved past the trusted workflow revision", async () => { + it("refuses dispatch after main advances", async () => { const fetchMock = vi.spyOn(globalThis, "fetch").mockResolvedValue( githubResponse({ ref: "refs/heads/main", @@ -400,7 +400,7 @@ describe("required live E2E controller", () => { ); await expect( - dispatchRequiredLive({ + dispatchPrGate({ repository: "NVIDIA/NemoClaw", token: "token", jobs: ["onboard-repair"], @@ -410,7 +410,7 @@ describe("required live E2E controller", () => { planHash: "c".repeat(64), correlationId: CORRELATION_ID, }), - ).rejects.toThrow(/no longer the current main revision/u); + ).rejects.toThrow(/main no longer points/u); expect(fetchMock).toHaveBeenCalledOnce(); }); @@ -431,15 +431,15 @@ describe("required live E2E controller", () => { ).toThrow(/display_title/u); }); - it("runs the dispatch-to-evidence lifecycle and completes one successful check", async () => { - const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-required-live-lifecycle-")); + it("completes the check when all evidence passes", async () => { + const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pr-e2e-gate-lifecycle-")); const outputPath = path.join(workDir, "github-output"); fs.writeFileSync(outputPath, "", { mode: 0o600 }); vi.stubEnv("GITHUB_TOKEN", "token"); vi.stubEnv("GITHUB_REPOSITORY", "NVIDIA/NemoClaw"); vi.stubEnv("GITHUB_OUTPUT", outputPath); const requests: RecordedGitHubRequest[] = []; - let gate: RequiredLiveState | undefined; + let gate: PrGateState | undefined; vi.spyOn(globalThis, "fetch").mockImplementation( createGitHubFetchRouter( [ @@ -494,8 +494,8 @@ describe("required live E2E controller", () => { try { const command = startCommand(workDir); - await startRequiredLive(command); - gate = validateRequiredLiveState(JSON.parse(fs.readFileSync(command.statePath, "utf8"))); + await startPrGate(command); + gate = validatePrGateState(JSON.parse(fs.readFileSync(command.statePath, "utf8"))); for (const job of gate.expectedJobs) { for (const shard of gate.expectedShards[job]!) { const directory = path.join(command.evidencePath, `${job}-${shard}`); @@ -513,7 +513,7 @@ describe("required live E2E controller", () => { .split("\n") .map((line) => line.split("=", 2)), ); - await finishRequiredLive({ + await finishPrGate({ statePath: command.statePath, stateHash: outputs.state_hash!, evidencePath: command.evidencePath, @@ -524,6 +524,18 @@ describe("required live E2E controller", () => { expect(gate.expectedJobs).toEqual(BROAD_JOBS); expect(requests.filter((request) => request.url.includes("/pulls?"))).toHaveLength(2); expect(requests.filter((request) => request.url.endsWith("/pulls/42"))).toHaveLength(2); + const checkCreation = requests.find( + (request) => request.url.endsWith("/check-runs") && request.method === "POST", + ); + expect(checkCreation?.body).toMatchObject({ + name: "E2E / PR Gate", + head_sha: HEAD_SHA, + status: "in_progress", + output: { + title: "Evaluating PR commit", + summary: "Validating the PR and selecting E2E jobs.", + }, + }); const dispatch = requests.find((request) => request.url.endsWith("/dispatches")); expect(dispatch?.body).toMatchObject({ inputs: { @@ -541,14 +553,14 @@ describe("required live E2E controller", () => { expect(checkUpdates[0]?.body).toMatchObject({ status: "in_progress", output: { - title: "Running 13 required live E2E jobs", + title: "Running 13 E2E jobs", summary: expect.stringContaining("upgrade-stale-sandbox"), }, }); expect(checkUpdates[1]?.body).toMatchObject({ status: "completed", conclusion: "success", - output: { title: "Required live E2E passed" }, + output: { title: "All selected jobs passed" }, }); expect(fs.readFileSync(outputPath, "utf8")).toContain("finalized=true"); } finally { @@ -557,7 +569,7 @@ describe("required live E2E controller", () => { }); it("fails without dispatch when the pull request changes during planning", async () => { - const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-required-live-race-")); + const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pr-e2e-gate-race-")); const outputPath = path.join(workDir, "github-output"); fs.writeFileSync(outputPath, "", { mode: 0o600 }); vi.stubEnv("GITHUB_TOKEN", "token"); @@ -607,8 +619,8 @@ describe("required live E2E controller", () => { ); try { - await expect(startRequiredLive(startCommand(workDir))).rejects.toThrow( - /changed while required live E2E was being prepared/u, + await expect(startPrGate(startCommand(workDir))).rejects.toThrow( + /PR changed during preparation/u, ); expect(requests.some((request) => request.url.endsWith("/dispatches"))).toBe(false); expect(requests.some((request) => request.url.endsWith("/git/ref/heads/main"))).toBe(false); @@ -623,7 +635,7 @@ describe("required live E2E controller", () => { }); it("cancels the child and closes the check when startup fails after dispatch", async () => { - const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-required-live-start-")); + const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pr-e2e-gate-start-")); const outputPath = path.join(workDir, "github-output"); fs.writeFileSync(outputPath, "", { mode: 0o600 }); vi.stubEnv("GITHUB_TOKEN", "token"); @@ -686,15 +698,20 @@ describe("required live E2E controller", () => { ); try { - await expect(startRequiredLive(startCommand(workDir))).rejects.toThrow( - /simulated update failure/u, - ); + await expect(startPrGate(startCommand(workDir))).rejects.toThrow(/simulated update failure/u); expect(requests.some((request) => request.url.endsWith("/actions/runs/23/cancel"))).toBe( true, ); const checkUpdates = requests.filter((request) => request.url.endsWith("/check-runs/17")); expect(checkUpdates).toHaveLength(2); - expect(checkUpdates[1]?.body).toMatchObject({ status: "completed", conclusion: "failure" }); + expect(checkUpdates[1]?.body).toMatchObject({ + status: "completed", + conclusion: "failure", + output: { + title: "Run could not start", + summary: expect.stringContaining("The controller could not complete the check."), + }, + }); expect(fs.readFileSync(outputPath, "utf8")).toContain("finalized=true"); } finally { fs.rmSync(workDir, { recursive: true, force: true }); @@ -702,12 +719,26 @@ describe("required live E2E controller", () => { }); it.each([ - { label: "missing evidence", status: "completed", expectCancellation: false }, - { label: "an unfinished child", status: "in_progress", expectCancellation: true }, - ])("closes the check as failure for $label", async ({ status, expectCancellation }) => { - const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-required-live-finish-")); + { + label: "missing evidence", + status: "completed", + expectCancellation: false, + expectedTitle: "Evidence is missing", + }, + { + label: "an unfinished child", + status: "in_progress", + expectCancellation: true, + expectedTitle: "E2E run did not succeed", + }, + ])("closes the check as failure for $label", async ({ + status, + expectCancellation, + expectedTitle, + }) => { + const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pr-e2e-gate-finish-")); const outputPath = path.join(workDir, "github-output"); - const statePath = path.join(workDir, "required-live-state.json"); + const statePath = path.join(workDir, "controller-state.json"); const evidencePath = path.join(workDir, "evidence"); const gate = state(); const serializedState = `${JSON.stringify(gate, null, 2)}\n`; @@ -740,7 +771,7 @@ describe("required live E2E controller", () => { try { await expect( - finishRequiredLive({ + finishPrGate({ statePath, stateHash: sha256(serializedState), evidencePath, @@ -752,7 +783,11 @@ describe("required live E2E controller", () => { expectCancellation, ); const completion = requests.find((request) => request.url.endsWith("/check-runs/17")); - expect(completion?.body).toMatchObject({ status: "completed", conclusion: "failure" }); + expect(completion?.body).toMatchObject({ + status: "completed", + conclusion: "failure", + output: { title: expectedTitle }, + }); expect(fs.readFileSync(outputPath, "utf8")).toContain("finalized=true"); } finally { fs.rmSync(workDir, { recursive: true, force: true }); @@ -773,6 +808,7 @@ describe("required live E2E controller", () => { workflowRun(gate, { status: "in_progress" }), workflowRun(gate, { id: 24, status: "completed" }), workflowRun(gate, { id: 25, status: "queued", display_title: "E2E manual" }), + workflowRun({ ...gate, prNumber: 420 }, { id: 26, status: "queued" }), ], }), ), @@ -783,14 +819,17 @@ describe("required live E2E controller", () => { ]), ); - await expect(cancelRequiredLive(42)).resolves.toBe(1); + await expect(cancelPrGate(42)).resolves.toBe(1); expect( fetchMock.mock.calls.filter(([input]) => String(input).endsWith("/cancel")), ).toHaveLength(1); + expect(fetchMock.mock.calls.some(([input]) => String(input).endsWith("/26/cancel"))).toBe( + false, + ); }); it("cancels a known child and closes an abandoned check as failure", async () => { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-required-live-abandon-")); + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pr-e2e-gate-abandon-")); const outputPath = path.join(directory, "github-output"); fs.writeFileSync(outputPath, "", { mode: 0o600 }); vi.stubEnv("GITHUB_TOKEN", "token"); @@ -814,12 +853,19 @@ describe("required live E2E controller", () => { ); try { - await abandonRequiredLive(17, 23); + await abandonPrGate(17, 23); expect(requests.map((request) => request.url)).toEqual([ "https://api.github.com/repos/NVIDIA/NemoClaw/actions/runs/23/cancel", "https://api.github.com/repos/NVIDIA/NemoClaw/check-runs/17", ]); - expect(requests[1]?.body).toMatchObject({ status: "completed", conclusion: "failure" }); + expect(requests[1]?.body).toMatchObject({ + status: "completed", + conclusion: "failure", + output: { + title: "Controller stopped early", + summary: "The controller stopped before it could complete the check.", + }, + }); expect(fs.readFileSync(outputPath, "utf8")).toContain("finalized=true"); } finally { fs.rmSync(directory, { recursive: true, force: true }); @@ -827,7 +873,7 @@ describe("required live E2E controller", () => { }); it("bounds recursive signal discovery and rejects symlinks", () => { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-required-live-evidence-")); + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pr-e2e-gate-evidence-")); try { const first = path.join(directory, "first"); fs.mkdirSync(first); diff --git a/tools/e2e-advisor/README.md b/tools/e2e-advisor/README.md index dfadf98c1f2..fbc72f494da 100644 --- a/tools/e2e-advisor/README.md +++ b/tools/e2e-advisor/README.md @@ -38,57 +38,18 @@ the trusted timing signal. `.github/workflows/e2e.yaml`, but the advisor job does not trigger those commands automatically. -## Required live PR check - -The model-independent `.github/workflows/required-live-e2e.yaml` workflow owns the -required `E2E / Required Live` check for same-repository pull requests after -`CI / Pull Request` completes. -It does not consume model output or advisor artifacts. -Instead, `tools/e2e/required-live.mts` resolves the open pull request for the -triggering revision, reads its complete changed-file list, and builds a new plan -from the checked-in risk policy. -The controller dispatches every job in `requiredJobs` through `e2e.yaml`. -If no runtime risk family matches, it reports success without dispatching live E2E. - -The controller verifies the pull request identity again immediately before -dispatch. -It also records the trusted controller revision, requires that revision to -still be the current `main` revision immediately before dispatch, and accepts -only a child workflow run created from that same revision. -The child workflow validates that the pull request is still open, belongs to -the base repository, and still points to the requested checkout SHA before E2E -preparation or secret-bearing jobs can run. -It also requires selective jobs with no `targets` input, a valid plan hash, -and a valid correlation ID. -The controller uses GitHub's returned workflow run ID as the sole child-run -selector for waiting, evidence download, and completion. - -The Vitest reporter records the observed checkout SHA and pass, failure, skip, -pending, and unhandled-error counts for each selected job and matrix shard. -The checked workflow boundary requires every job named by the deterministic -policy to expose its matching job identity, attach the reporter to every -Vitest invocation, and always upload its evidence artifact. -The controller accepts only signals bound to the expected SHA, plan hash, -correlation ID, job, and shard. -It also records a SHA-256 digest of its private dispatch state and verifies that -digest before parsing downloaded evidence. - -The check has a binary result. -It succeeds only when the correlated workflow succeeds and every expected job -shard produces one complete, unskipped pass. -Workflow failures, failed tests, missing or duplicate signals, skipped or -pending tests, interrupted runs, and controller or evidence-validation errors -all fail the check. -Evidence download has its own 10-minute limit within the coordinator's -180-minute job budget, and exceeding that limit fails the check. -Pull request synchronization, reopening, or closure cancels active child runs -for that pull request, and the E2E workflow cancels a superseded child run when -a new revision is dispatched. - -E2E Advisor remains advisory. -It uses the same deterministic policy as a recommendation floor and may add -adjacent coverage, but its model output and availability never determine the -required check. +## PR E2E check + +`.github/workflows/pr-e2e-gate.yaml` owns `E2E / PR Gate` after +`CI / Pull Request` completes. It uses the same checked-in risk policy as E2E +Advisor, but rebuilds the plan from GitHub's changed-file list and never +consumes advisor output. It dispatches every selected `requiredJobs` entry and +verifies the resulting E2E evidence. See +[NemoClaw E2E CI](../../test/e2e/README.md) for the full lifecycle. + +E2E Advisor remains advisory. It uses the risk policy as a recommendation +floor and may add adjacent coverage, but its model output and availability do +not determine the PR E2E check. ## Required secret @@ -116,9 +77,9 @@ dispatch commands; it does not trigger E2E workflows automatically. - `risk-plan.json` — deterministic risk families, invariants, required jobs, changed files, and the plan digest for the pull request revision. Both E2E Advisor projections consume this required-job floor. - The required live controller independently rebuilds the plan from GitHub's - pull request file list and dispatches every selected job, so this advisor - artifact is not an input to the required check. + The PR E2E controller independently rebuilds the plan from GitHub's pull + request file list and dispatches every selected job, so this advisor + artifact is not an input to the PR E2E check. - `e2e-advisor-raw-output.txt` — raw advisor transcript and diagnostics. - `e2e-advisor-result.json` — parsed advisor response or execution metadata. - `e2e-advisor-session.html` — exported advisor session transcript. @@ -149,6 +110,3 @@ secret. Run `npm install` first so the Pi SDK dependency is available. `tools/e2e-advisor/schema.json` defines the normalized coverage recommendation shape. `tools/e2e-advisor/targets-schema.json` defines the normalized target recommendation shape used by the `targets` and `jobs` dispatch commands. - -The required live check verifies complete E2E evidence for the same pull -request revision without making model availability part of merge authority. diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index 3eb3d16d466..c44451c0a93 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -15,7 +15,7 @@ const META_JOBS = new Set(["report-to-pr", "scorecard"]); const FULL_SHA_ACTION = /^[^\s@]+@[0-9a-f]{40}$/u; const GITHUB_SCRIPT_NODE24_ACTION = "actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3"; -const REQUIRED_LIVE_REPORTER = "test/e2e/risk-signal-reporter.ts"; +const PR_GATE_REPORTER = "test/e2e/risk-signal-reporter.ts"; const E2E_ARTIFACT_ACTION = "NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@"; const ISSUE_API_REFERENCE = /\bgithub\.rest\.issues\b/u; const ISSUE_MUTATION_BEYOND_COMMENT = @@ -110,7 +110,7 @@ function requireNode24GithubScript(errors: string[], step: WorkflowStep, owner: } } -function validateRequiredLiveDispatch(errors: string[], workflow: OperationsWorkflow): void { +function validatePrGateDispatch(errors: string[], workflow: OperationsWorkflow): void { const inputs = workflow.on?.workflow_dispatch?.inputs ?? {}; for (const name of ["jobs", "pr_number", "checkout_sha", "plan_hash", "correlation_id"]) { const input = inputs[name]; @@ -126,36 +126,34 @@ function validateRequiredLiveDispatch(errors: string[], workflow: OperationsWork }; for (const [name, value] of Object.entries(expectedEnvironment)) { if (workflow.env?.[name] !== value) { - errors.push(`E2E workflow must bind ${name} to required-live metadata`); + errors.push(`E2E workflow must bind ${name} to controller metadata`); } } const runName = String(workflow["run-name"] ?? ""); for (const fragment of ["inputs.checkout_sha", "inputs.pr_number", "inputs.correlation_id"]) { - if (!runName.includes(fragment)) errors.push(`required-live run name must include ${fragment}`); + if (!runName.includes(fragment)) errors.push(`PR E2E run name must include ${fragment}`); } const concurrencyGroup = String(workflow.concurrency?.group ?? ""); if ( !concurrencyGroup.includes("inputs.checkout_sha") || !concurrencyGroup.includes("inputs.pr_number") ) { - errors.push("required-live concurrency must be scoped to its pull request"); + errors.push("PR E2E concurrency must be scoped to its pull request"); } if (workflow.concurrency?.["cancel-in-progress"] !== "${{ inputs.checkout_sha != '' }}") { - errors.push("required-live concurrency must cancel obsolete pull request runs"); + errors.push("PR E2E concurrency must cancel obsolete runs"); } const matrixJob = workflow.jobs["generate-matrix"] ?? {}; const steps = matrixJob.steps ?? []; - const validationIndex = steps.findIndex( - (step) => step.name === "Validate required-live dispatch", - ); + const validationIndex = steps.findIndex((step) => step.name === "Validate controller dispatch"); const prepareIndex = steps.findIndex((step) => step.name === "Prepare E2E workspace"); const validation = validationIndex >= 0 ? steps[validationIndex] : {}; if (validation.if !== "${{ inputs.checkout_sha != '' }}") { - errors.push("required-live validation must be activated only by checkout_sha"); + errors.push("Controller validation must be activated only by checkout_sha"); } if (validationIndex < 0 || prepareIndex < 0 || validationIndex >= prepareIndex) { - errors.push("required-live validation must run before workspace preparation"); + errors.push("Controller validation must run before workspace preparation"); } const expectedStepEnvironment = { CHECKOUT_SHA: "${{ inputs.checkout_sha }}", @@ -167,7 +165,7 @@ function validateRequiredLiveDispatch(errors: string[], workflow: OperationsWork }; for (const [name, value] of Object.entries(expectedStepEnvironment)) { if (validation.env?.[name] !== value) { - errors.push(`required-live validation must bind ${name}`); + errors.push(`Controller validation must bind ${name}`); } } const validationScript = String(validation.run ?? ""); @@ -183,7 +181,7 @@ function validateRequiredLiveDispatch(errors: string[], workflow: OperationsWork "'.head.sha'", ]) { if (!validationScript.includes(fragment)) { - errors.push(`required-live validation must retain ${fragment}`); + errors.push(`Controller validation must retain ${fragment}`); } } @@ -193,41 +191,38 @@ function validateRequiredLiveDispatch(errors: string[], workflow: OperationsWork step.uses?.startsWith("actions/checkout@") && step.with?.ref !== "${{ inputs.checkout_sha || github.sha }}" ) { - errors.push(`${jobName} checkout must use the selected immutable commit`); + errors.push(`${jobName} checkout must use the selected PR commit`); } } } } -function validateRequiredLiveEvidenceProducers( - errors: string[], - workflow: OperationsWorkflow, -): void { +function validatePrGateEvidenceProducers(errors: string[], workflow: OperationsWorkflow): void { const requiredJobs = new Set(RISK_RULES.flatMap((rule) => rule.requiredJobs)); for (const jobId of requiredJobs) { const job = workflow.jobs[jobId]; if (!job) { - errors.push(`required-live plan job is missing from E2E workflow: ${jobId}`); + errors.push(`Risk-plan job is missing from E2E workflow: ${jobId}`); continue; } if (job.env?.E2E_JOB !== "1" || job.env?.E2E_TARGET_ID !== jobId) { - errors.push(`${jobId} must expose matching required-live job identity`); + errors.push(`${jobId} must expose matching E2E job identity`); } if (typeof job.env?.E2E_ARTIFACT_DIR !== "string" || !job.env.E2E_ARTIFACT_DIR) { - errors.push(`${jobId} must expose a required-live artifact directory`); + errors.push(`${jobId} must expose an evidence artifact directory`); } const vitestSteps = (job.steps ?? []).filter((step) => String(step.run ?? "").includes("npx vitest"), ); if ( vitestSteps.length === 0 || - vitestSteps.some((step) => !String(step.run).includes(REQUIRED_LIVE_REPORTER)) + vitestSteps.some((step) => !String(step.run).includes(PR_GATE_REPORTER)) ) { - errors.push(`${jobId} must attach the required-live reporter to every Vitest invocation`); + errors.push(`${jobId} must attach the risk-signal reporter to every Vitest invocation`); } const uploads = (job.steps ?? []).filter((step) => step.uses?.startsWith(E2E_ARTIFACT_ACTION)); if (uploads.length !== 1 || uploads[0]?.if !== "always()") { - errors.push(`${jobId} must always upload one required-live evidence artifact`); + errors.push(`${jobId} must always upload one evidence artifact`); } } } @@ -546,8 +541,8 @@ export function validateE2eOperationsWorkflow( advisorPath = DEFAULT_ADVISOR_PATH, ): string[] { const errors: string[] = []; - validateRequiredLiveDispatch(errors, workflow); - validateRequiredLiveEvidenceProducers(errors, workflow); + validatePrGateDispatch(errors, workflow); + validatePrGateEvidenceProducers(errors, workflow); validateAggregation(errors, workflow); validateIssueRoutingRetirement(errors, workflow); validateScorecard(errors, workflow); diff --git a/tools/e2e/required-live.mts b/tools/e2e/pr-e2e-gate.mts similarity index 86% rename from tools/e2e/required-live.mts rename to tools/e2e/pr-e2e-gate.mts index 5993d18c61a..ef655ff9abd 100755 --- a/tools/e2e/required-live.mts +++ b/tools/e2e/pr-e2e-gate.mts @@ -24,8 +24,8 @@ import { readFreeStandingJobsInventory } from "./workflow-boundary.mts"; const E2E_WORKFLOW = "e2e.yaml"; const E2E_WORKFLOW_PATH = `.github/workflows/${E2E_WORKFLOW}`; -const CHECK_NAME = "E2E / Required Live"; -const USER_AGENT = "nemoclaw-required-live"; +const CHECK_NAME = "E2E / PR Gate"; +const USER_AGENT = "nemoclaw-pr-e2e-gate"; const SHA_PATTERN = /^[a-f0-9]{40}$/u; const HASH_PATTERN = /^[a-f0-9]{64}$/u; const REPOSITORY_PATTERN = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u; @@ -112,7 +112,7 @@ type WorkflowRunIdentity = { workflowSha: string; }; -export type RequiredLiveState = { +export type PrGateState = { version: 1; commitSha: string; workflowSha: string; @@ -123,7 +123,7 @@ export type RequiredLiveState = { expectedShards: Record; }; -export type RequiredLiveVerdict = { +export type PrGateVerdict = { conclusion: CheckConclusion; title: string; summary: string; @@ -207,8 +207,8 @@ export function privateControllerPaths(workDir: string): ControllerPaths { throw new Error("--work-dir must be an owned private absolute directory"); } return { - planPath: path.join(resolved, "required-live-plan.json"), - statePath: path.join(resolved, "required-live-state.json"), + planPath: path.join(resolved, "risk-plan.json"), + statePath: path.join(resolved, "controller-state.json"), evidencePath: path.join(resolved, "evidence"), }; } @@ -255,24 +255,24 @@ function readRegularJson(file: string, maxBytes = MAX_PLAN_BYTES): unknown { return JSON.parse(readPrivateRegularFile(file, { maxBytes })!); } -export function validateRequiredLiveState(value: unknown): RequiredLiveState { +export function validatePrGateState(value: unknown): PrGateState { if (!isObjectRecord(value) || value.version !== 1) { - throw new Error("invalid required-live state version"); + throw new Error("State version is invalid"); } if (typeof value.commitSha !== "string" || !SHA_PATTERN.test(value.commitSha)) { - throw new Error("required-live state commit SHA is invalid"); + throw new Error("State commit SHA is invalid"); } if (typeof value.workflowSha !== "string" || !SHA_PATTERN.test(value.workflowSha)) { - throw new Error("required-live state workflow SHA is invalid"); + throw new Error("State workflow SHA is invalid"); } if (typeof value.planHash !== "string" || !HASH_PATTERN.test(value.planHash)) { - throw new Error("required-live state plan hash is invalid"); + throw new Error("State plan hash is invalid"); } if (typeof value.correlationId !== "string" || !CORRELATION_PATTERN.test(value.correlationId)) { - throw new Error("required-live state correlation id is invalid"); + throw new Error("State correlation ID is invalid"); } if (!Number.isSafeInteger(value.prNumber) || (value.prNumber as number) < 1) { - throw new Error("required-live state PR number is invalid"); + throw new Error("State PR number is invalid"); } if ( !Array.isArray(value.expectedJobs) || @@ -280,14 +280,14 @@ export function validateRequiredLiveState(value: unknown): RequiredLiveState { !value.expectedJobs.every((job) => typeof job === "string" && JOB_PATTERN.test(job)) || new Set(value.expectedJobs).size !== value.expectedJobs.length ) { - throw new Error("required-live state expected jobs are invalid"); + throw new Error("State jobs are invalid"); } if (!isObjectRecord(value.expectedShards)) { - throw new Error("required-live state shards are invalid"); + throw new Error("State shards are invalid"); } const shardJobs = Object.keys(value.expectedShards).sort(); if (JSON.stringify(shardJobs) !== JSON.stringify([...value.expectedJobs].sort())) { - throw new Error("required-live state shard jobs do not match expected jobs"); + throw new Error("State shard jobs do not match expected jobs"); } for (const job of value.expectedJobs) { const shards = value.expectedShards[job]; @@ -297,10 +297,10 @@ export function validateRequiredLiveState(value: unknown): RequiredLiveState { new Set(shards).size !== shards.length || !shards.every((shard) => typeof shard === "string" && SHARD_PATTERN.test(shard)) ) { - throw new Error(`required-live state shards are invalid for ${job}`); + throw new Error(`State shards are invalid for ${job}`); } } - return value as RequiredLiveState; + return value as PrGateState; } export function validateRiskPlan(value: unknown, allowedJobs: ReadonlySet): RiskPlan { @@ -321,7 +321,7 @@ export function validateRiskPlan(value: unknown, allowedJobs: ReadonlySet, ): E2eRiskSignal { @@ -368,17 +368,17 @@ export function validateSignal( return signal; } -export function classifyRequiredLiveEvidence(options: { +export function classifyPrGateEvidence(options: { workflowConclusion: string | null; expectedJobs: readonly string[]; expectedShards: Readonly>; signals: readonly E2eRiskSignal[]; -}): RequiredLiveVerdict { +}): PrGateVerdict { if (options.workflowConclusion !== "success") { return { conclusion: "failure", - title: "Required live E2E did not complete successfully", - summary: `The correlated E2E workflow concluded ${options.workflowConclusion ?? "without a result"}.`, + title: "E2E run did not succeed", + summary: `The run concluded ${options.workflowConclusion ?? "without a result"}.`, }; } const expectedEvidence = options.expectedJobs.flatMap((job) => @@ -390,8 +390,8 @@ export function classifyRequiredLiveEvidence(options: { ) { return { conclusion: "failure", - title: "Required live E2E lacks an evidence policy", - summary: "At least one selected job has no trusted shard policy.", + title: "Evidence policy is incomplete", + summary: "At least one selected job has no configured shard policy.", }; } const byJobShard = new Map(); @@ -400,7 +400,7 @@ export function classifyRequiredLiveEvidence(options: { if (byJobShard.has(key)) { return { conclusion: "failure", - title: "Required live E2E produced duplicate evidence", + title: "Duplicate evidence", summary: `More than one signal was uploaded for ${key}.`, }; } @@ -410,8 +410,8 @@ export function classifyRequiredLiveEvidence(options: { if (missing.length > 0) { return { conclusion: "failure", - title: "Required live E2E is missing evidence", - summary: `Missing bound signals for: ${missing.join(", ")}.`, + title: "Evidence is missing", + summary: `Missing signals: ${missing.join(", ")}.`, }; } const failed = expectedEvidence.filter((key) => { @@ -421,7 +421,7 @@ export function classifyRequiredLiveEvidence(options: { if (failed.length > 0) { return { conclusion: "failure", - title: "Required live E2E reported test failures", + title: "Tests failed", summary: `Failing signals: ${failed.join(", ")}.`, }; } @@ -434,14 +434,14 @@ export function classifyRequiredLiveEvidence(options: { if (partial.length > 0) { return { conclusion: "failure", - title: "Required live E2E produced incomplete evidence", + title: "Evidence is incomplete", summary: `Incomplete or skipped signals: ${partial.join(", ")}.`, }; } return { conclusion: "success", - title: "Required live E2E passed", - summary: "Every expected job shard produced a complete, unskipped pass.", + title: "All selected jobs passed", + summary: "Every expected job shard passed with no skips or pending tests.", }; } @@ -461,7 +461,7 @@ function appendOutput(name: string, value: string): void { try { if (!fs.fstatSync(descriptor).isFile()) throw new Error("GITHUB_OUTPUT must be a regular file"); // lgtm[js/network-data-to-file] Values are reduced to a strict single-line allowlist above, - // and the trusted runner-owned output file is opened without following symlinks. + // and the runner-owned output file is opened without following symlinks. // lgtm[js/http-to-file-access] fs.writeFileSync(descriptor, `${name}=${value}\n`, "utf8"); } finally { @@ -495,7 +495,7 @@ async function createCheck( async function completeCheck( context: { repository: string; checkRunId: number }, token: string, - verdict: RequiredLiveVerdict, + verdict: PrGateVerdict, detailsUrl?: string, ): Promise { await githubApi(`repos/${context.repository}/check-runs/${context.checkRunId}`, token, { @@ -523,8 +523,8 @@ async function updateRunningCheck( status: "in_progress", details_url: childRunUrl, output: { - title: `Running ${options.jobs.length} required live E2E ${options.jobs.length === 1 ? "job" : "jobs"}`, - summary: `Plan ${options.planHash} selected: ${options.jobs.join(", ")}.`, + title: `Running ${options.jobs.length} E2E ${options.jobs.length === 1 ? "job" : "jobs"}`, + summary: `Risk plan ${options.planHash} selected: ${options.jobs.join(", ")}.`, }, }, userAgent: USER_AGENT, @@ -556,15 +556,13 @@ async function completeFailureAfterControllerError( { conclusion: "failure", title, - summary: `The required-live controller could not produce trustworthy evidence.\n\nController error: \`${reason}\``, + summary: `The controller could not complete the check.\n\nController error: \`${reason}\``, }, options.detailsUrl, ); return true; } catch (error) { - console.error( - `Failed to close required-live check after controller error: ${controllerErrorMessage(error)}`, - ); + console.error(`Failed to close check after controller error: ${controllerErrorMessage(error)}`); return false; } } @@ -719,7 +717,7 @@ function assertPullUnchanged(before: PullRequest, after: PullRequest): void { JSON.stringify({ ...pullIdentity(before), changedFiles: before.changed_files }) !== JSON.stringify({ ...pullIdentity(after), changedFiles: after.changed_files }) ) { - throw new Error("Pull request changed while required live E2E was being prepared"); + throw new Error("PR changed during preparation"); } } @@ -807,7 +805,7 @@ export function assertCorrelatedWorkflowRun( requireEqual("html_url", childRunUrl, child.html_url); requireEqual( "display_title", - `E2E PR #${identity.prNumber} required live ${identity.correlationId}`, + `E2E PR #${identity.prNumber} (${identity.correlationId})`, child.display_title, ); requireEqual("head_sha", identity.workflowSha, child.head_sha); @@ -818,12 +816,12 @@ export function assertCorrelatedWorkflowRun( } if (mismatches.length > 0) { throw new Error( - `Correlated E2E workflow identity mismatch: ${mismatches.join("; ")}; observed run_name=${diagnosticValue(child.name)} workflow_id=${diagnosticValue(child.workflow_id)}`, + `E2E run identity mismatch: ${mismatches.join("; ")}; observed run_name=${diagnosticValue(child.name)} workflow_id=${diagnosticValue(child.workflow_id)}`, ); } } -export async function dispatchRequiredLive(options: { +export async function dispatchPrGate(options: { repository: string; token: string; jobs: readonly string[]; @@ -846,7 +844,7 @@ export async function dispatchRequiredLive(options: { !HASH_PATTERN.test(options.planHash) || !CORRELATION_PATTERN.test(options.correlationId) ) { - throw new Error("required-live workflow dispatch inputs are invalid"); + throw new Error("Controller dispatch inputs are invalid"); } const main = await githubApi( `repos/${options.repository}/git/ref/heads/main`, @@ -859,9 +857,7 @@ export async function dispatchRequiredLive(options: { main.object?.type !== "commit" || main.object.sha !== options.workflowSha ) { - throw new Error( - `Trusted workflow revision ${options.workflowSha} is no longer the current main revision`, - ); + throw new Error(`main no longer points to workflow commit ${options.workflowSha}`); } const details = await githubApi( `repos/${options.repository}/actions/workflows/${E2E_WORKFLOW}/dispatches`, @@ -897,21 +893,21 @@ async function cancelChildRun(repository: string, token: string, runId: number): } } -export async function startRequiredLive( +export async function startPrGate( command: Extract, ): Promise { const { token, repository } = tokenAndRepository(); - if (!SHA_PATTERN.test(command.headSha)) throw new Error("triggering head SHA is invalid"); - if (!SHA_PATTERN.test(command.workflowSha)) throw new Error("trusted workflow SHA is invalid"); - assertRepository(command.headRepository, "triggering head repository"); + if (!SHA_PATTERN.test(command.headSha)) throw new Error("PR head SHA is invalid"); + if (!SHA_PATTERN.test(command.workflowSha)) throw new Error("workflow SHA is invalid"); + assertRepository(command.headRepository, "PR head repository"); assertBranch(command.headBranch); const checkRunId = await createCheck( repository, token, command.headSha, - "Required live E2E is evaluating this revision", - "The controller is validating the pull request and building its deterministic live-test plan.", + "Evaluating PR commit", + "Validating the PR and selecting E2E jobs.", ); appendOutput("check_id", String(checkRunId)); @@ -921,8 +917,8 @@ export async function startRequiredLive( if (command.ciConclusion !== "success") { await completeCheck({ repository, checkRunId }, token, { conclusion: "failure", - title: "Pull request CI did not pass", - summary: `CI / Pull Request concluded ${command.ciConclusion || "without a result"}; live E2E was not dispatched.`, + title: "PR CI did not pass", + summary: `CI / Pull Request concluded ${command.ciConclusion || "without a result"}; no run was dispatched.`, }); appendOutput("dispatched", "false"); appendOutput("finalized", "true"); @@ -938,7 +934,7 @@ export async function startRequiredLive( headBranch: command.headBranch, }); if (command.headRepository !== repository || pull.head.repo?.full_name !== repository) { - throw new Error("Required live E2E can run only for branches in the base repository"); + throw new Error("PR branch must be in the base repository"); } const changedFiles = await pullChangedFiles(repository, pull, token); @@ -960,24 +956,22 @@ export async function startRequiredLive( if (jobs.length === 0) { await completeCheck({ repository, checkRunId }, token, { conclusion: "success", - title: "No required live E2E selected", - summary: "The deterministic plan matched no live runtime regression family.", + title: "No E2E jobs selected", + summary: "No changed files matched an E2E risk rule.", }); appendOutput("dispatched", "false"); appendOutput("finalized", "true"); finalized = true; - console.log( - `Required live E2E completed without dispatch: pr=${pull.number} plan=${plan.planHash}`, - ); + console.log(`No run dispatched: pr=${pull.number} plan=${plan.planHash}`); return; } const expectedShards = expectedSignalShards(jobs); const correlationId = randomUUID(); if (!CORRELATION_PATTERN.test(correlationId)) { - throw new Error("generated correlation id is invalid"); + throw new Error("generated correlation ID is invalid"); } - childRunId = await dispatchRequiredLive({ + childRunId = await dispatchPrGate({ repository, token, jobs, @@ -988,7 +982,7 @@ export async function startRequiredLive( correlationId, }); appendOutput("run_id", String(childRunId)); - const state: RequiredLiveState = { + const state: PrGateState = { version: 1, commitSha: command.headSha, workflowSha: command.workflowSha, @@ -1008,7 +1002,7 @@ export async function startRequiredLive( appendOutput("state_hash", sha256(serializedState)); appendOutput("dispatched", "true"); console.log( - `Required live E2E dispatched: pr=${pull.number} run=${childRunId} plan=${plan.planHash} jobs=${jobs.join(",")} url=https://github.com/${repository}/actions/runs/${childRunId}`, + `Run dispatched: pr=${pull.number} run=${childRunId} plan=${plan.planHash} jobs=${jobs.join(",")} url=https://github.com/${repository}/actions/runs/${childRunId}`, ); } catch (error) { let reportedError = error; @@ -1025,7 +1019,7 @@ export async function startRequiredLive( const closed = await completeFailureAfterControllerError( { repository, checkRunId }, token, - "Required live E2E could not start", + "Run could not start", { error: reportedError }, ); if (closed) appendOutput("finalized", "true"); @@ -1085,7 +1079,7 @@ export function findSignalFiles( return files.sort((left, right) => left.localeCompare(right)); } -export async function finishRequiredLive(options: { +export async function finishPrGate(options: { statePath: string; stateHash: string; evidencePath: string; @@ -1104,7 +1098,7 @@ export async function finishRequiredLive(options: { if (sha256(serializedState) !== options.stateHash) { throw new Error("controller state changed after E2E dispatch"); } - const state = validateRequiredLiveState(JSON.parse(serializedState)); + const state = validatePrGateState(JSON.parse(serializedState)); const child = await githubApi( `repos/${repository}/actions/runs/${options.childRunId}`, token, @@ -1120,7 +1114,7 @@ export async function finishRequiredLive(options: { if (child.status !== "completed") { await cancelChildRun(repository, token, options.childRunId); console.log( - `Cancelled unfinished required live E2E during finalization: run=${options.childRunId} status=${child.status} url=${childRunUrl}`, + `Cancelled unfinished run during finalization: run=${options.childRunId} status=${child.status} url=${childRunUrl}`, ); } const workflowConclusion = @@ -1136,7 +1130,7 @@ export async function finishRequiredLive(options: { maxSignalFiles: expectedSignalCount + 1, }).map((file) => validateSignal(readRegularJson(file), state)) : []; - const verdict = classifyRequiredLiveEvidence({ + const verdict = classifyPrGateEvidence({ workflowConclusion, expectedJobs: state.expectedJobs, expectedShards: state.expectedShards, @@ -1146,7 +1140,7 @@ export async function finishRequiredLive(options: { appendOutput("finalized", "true"); finalized = true; console.log( - `Required live E2E completed: run=${options.childRunId} conclusion=${verdict.conclusion} title=${verdict.title} url=${childRunUrl}`, + `Run completed: run=${options.childRunId} conclusion=${verdict.conclusion} title=${verdict.title} url=${childRunUrl}`, ); if (verdict.conclusion === "failure") throw new Error(verdict.title); } catch (error) { @@ -1154,7 +1148,7 @@ export async function finishRequiredLive(options: { const closed = await completeFailureAfterControllerError( context, token, - "Required live E2E evidence could not be verified", + "Evidence could not be verified", { error, detailsUrl: childRunUrl }, ); if (closed) appendOutput("finalized", "true"); @@ -1163,7 +1157,7 @@ export async function finishRequiredLive(options: { } } -export async function abandonRequiredLive(checkRunId: number, childRunId?: number): Promise { +export async function abandonPrGate(checkRunId: number, childRunId?: number): Promise { const { token, repository } = tokenAndRepository(); let cancellationError: unknown; if (childRunId) { @@ -1178,17 +1172,17 @@ export async function abandonRequiredLive(checkRunId: number, childRunId?: numbe : ""; await completeCheck({ repository, checkRunId }, token, { conclusion: "failure", - title: "Required live E2E controller stopped early", - summary: `The controller stopped before it could produce complete evidence.${cancellationSummary}`, + title: "Controller stopped early", + summary: `The controller stopped before it could complete the check.${cancellationSummary}`, }); appendOutput("finalized", "true"); if (cancellationError) throw cancellationError; } -export async function cancelRequiredLive(prNumber: number): Promise { +export async function cancelPrGate(prNumber: number): Promise { const { token, repository } = tokenAndRepository(); if (!Number.isSafeInteger(prNumber) || prNumber < 1) throw new Error("PR number is invalid"); - const titlePrefix = `E2E PR #${prNumber} required live `; + const titlePrefix = `E2E PR #${prNumber} (`; const active: WorkflowRun[] = []; for (let page = 1; page <= MAX_ACTIVE_RUN_PAGES; page += 1) { const response = await githubApi( @@ -1206,20 +1200,20 @@ export async function cancelRequiredLive(prNumber: number): Promise { ); if (response.workflow_runs.length < 100) break; if (page === MAX_ACTIVE_RUN_PAGES) { - throw new Error("Required-live run listing exceeded its page limit"); + throw new Error("Run listing exceeded its page limit"); } } for (const run of active) { if (!Number.isSafeInteger(run.id) || run.id < 1) { - throw new Error("GitHub returned an invalid active workflow run id"); + throw new Error("GitHub returned an invalid active run ID"); } await cancelChildRun(repository, token, run.id); console.log( - `Cancelled superseded required live E2E: pr=${prNumber} run=${run.id} url=https://github.com/${repository}/actions/runs/${run.id}`, + `Cancelled superseded run: pr=${prNumber} run=${run.id} url=https://github.com/${repository}/actions/runs/${run.id}`, ); } if (active.length === 0) { - console.log(`No active required live E2E runs found for PR #${prNumber}`); + console.log(`No active E2E runs found for PR #${prNumber}`); } return active.length; } @@ -1229,18 +1223,18 @@ function reportControllerError(error: unknown): void { console.error(message); if (process.env.GITHUB_ACTIONS === "true") { const escaped = message.replace(/%/gu, "%25").replace(/\r/gu, "%0D").replace(/\n/gu, "%0A"); - console.error(`::error title=Required live E2E controller failed::${escaped}`); + console.error(`::error title=Controller failed::${escaped}`); } } async function main(): Promise { const command = parseControllerCommand(process.argv.slice(2)); if (command.mode === "start") { - await startRequiredLive(command); + await startPrGate(command); return; } if (command.mode === "finish") { - await finishRequiredLive({ + await finishPrGate({ statePath: command.statePath, stateHash: command.stateHash, evidencePath: command.evidencePath, @@ -1250,10 +1244,10 @@ async function main(): Promise { return; } if (command.mode === "abandon") { - await abandonRequiredLive(command.checkRunId, command.childRunId); + await abandonPrGate(command.checkRunId, command.childRunId); return; } - await cancelRequiredLive(command.prNumber); + await cancelPrGate(command.prNumber); } if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { diff --git a/tools/pr-review-advisor/README.md b/tools/pr-review-advisor/README.md index 2c7cd8ef873..69367503bd3 100644 --- a/tools/pr-review-advisor/README.md +++ b/tools/pr-review-advisor/README.md @@ -56,11 +56,10 @@ before the failure so later runs and reviewers do not lose substantive review hi The workflow is advisory and must not be configured as a required status check. It uses the deterministic plan as review context but does not run its jobs. E2E Advisor emits the corresponding plan-backed recommendations separately and likewise does not dispatch E2E. Model availability must -not become the authority for whether a pull request can merge. The separate model-independent -required live controller rebuilds the plan from GitHub's changed-file list for the current pull -request revision and dispatches every required job after `CI / Pull Request` completes. It does not -consume PR Review Advisor or E2E Advisor output. The `E2E / Required Live` check therefore remains -independent of both model advisors and does not make PR Review Advisor a merge gate. +not become the authority for whether a pull request can merge. The PR E2E controller separately +rebuilds the plan from GitHub's changed-file list and dispatches every selected job after +`CI / Pull Request` completes. `E2E / PR Gate` does not consume either advisor's output and does not +make PR Review Advisor a merge gate. Required-check status is point-in-time context, not a settled-CI gate. Earlier `PR_REVIEW_ADVISOR_WAIT_*` workflow variables were inert and have been removed; any future waiting @@ -84,7 +83,7 @@ Authors and coding agents should follow the shared [PR CI and Automated Review F - During rollout, non-default advisor lanes may see an older trusted `main` checkout that has the workflow matrix but not the matching model/configurable-comment support. The workflow treats that as trusted-main rollout skew, writes low-confidence skip artifacts in the lane-specific artifact directory, and suppresses that lane's sticky PR comment. Do not run PR-controlled advisor code to bypass this gate; remove the gate only after the trusted `main` implementation always supports the parallel advisor lane and configurable sticky markers. - The checked-in risk plan is deterministic and additive. PR Review Advisor reviews every listed invariant and required job for missing evidence. Both E2E Advisor result normalizers restore any - listed job that a model omits or downgrades. The required live controller separately dispatches + listed job that a model omits or downgrades. The PR E2E controller separately dispatches every listed job without consuming either advisor's normalized result. ## Required secret From 9202c9f10a6b31684ef5225d470eef643d7ae2a7 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Fri, 10 Jul 2026 16:35:56 -0700 Subject: [PATCH 6/7] fix(e2e): reject fork events before check creation Signed-off-by: Carlos Villela --- .github/workflows/pr-e2e-gate.yaml | 2 +- test/pr-e2e-gate-workflow.test.ts | 3 +++ test/pr-e2e-gate.test.ts | 16 ++++++++++++++++ tools/e2e-advisor/README.md | 10 +++++----- tools/e2e/pr-e2e-gate.mts | 3 +++ tools/pr-review-advisor/README.md | 8 ++++---- 6 files changed, 32 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pr-e2e-gate.yaml b/.github/workflows/pr-e2e-gate.yaml index e021c998042..baf263797c2 100644 --- a/.github/workflows/pr-e2e-gate.yaml +++ b/.github/workflows/pr-e2e-gate.yaml @@ -45,7 +45,7 @@ jobs: --pr "$PR_NUMBER" coordinate: - if: ${{ github.event_name == 'workflow_run' && github.repository == 'NVIDIA/NemoClaw' && github.event.workflow_run.event == 'pull_request' }} + if: ${{ github.event_name == 'workflow_run' && github.repository == 'NVIDIA/NemoClaw' && github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.head_repository.full_name == github.repository }} runs-on: ubuntu-latest timeout-minutes: 180 permissions: diff --git a/test/pr-e2e-gate-workflow.test.ts b/test/pr-e2e-gate-workflow.test.ts index 5c9d6961437..d742a1ad01f 100644 --- a/test/pr-e2e-gate-workflow.test.ts +++ b/test/pr-e2e-gate-workflow.test.ts @@ -236,6 +236,9 @@ describe("PR E2E gate workflow", () => { expect(cancel.permissions).toEqual({ actions: "write", contents: "read" }); expect(coordinate.if).toContain("github.event_name == 'workflow_run'"); expect(coordinate.if).toContain("github.event.workflow_run.event == 'pull_request'"); + expect(coordinate.if).toContain( + "github.event.workflow_run.head_repository.full_name == github.repository", + ); expect(coordinate.permissions).toEqual({ actions: "write", checks: "write", diff --git a/test/pr-e2e-gate.test.ts b/test/pr-e2e-gate.test.ts index ed168200071..f8758f21a25 100644 --- a/test/pr-e2e-gate.test.ts +++ b/test/pr-e2e-gate.test.ts @@ -431,6 +431,22 @@ describe("PR E2E controller", () => { ).toThrow(/display_title/u); }); + it("rejects fork branches before making API requests", async () => { + const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pr-e2e-gate-fork-")); + vi.stubEnv("GITHUB_TOKEN", "token"); + vi.stubEnv("GITHUB_REPOSITORY", "NVIDIA/NemoClaw"); + const fetchMock = vi.spyOn(globalThis, "fetch"); + + try { + await expect( + startPrGate({ ...startCommand(workDir), headRepository: "contributor/NemoClaw" }), + ).rejects.toThrow(/PR branch must be in the base repository/u); + expect(fetchMock).not.toHaveBeenCalled(); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + }); + it("completes the check when all evidence passes", async () => { const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pr-e2e-gate-lifecycle-")); const outputPath = path.join(workDir, "github-output"); diff --git a/tools/e2e-advisor/README.md b/tools/e2e-advisor/README.md index fbc72f494da..35e4e63bd71 100644 --- a/tools/e2e-advisor/README.md +++ b/tools/e2e-advisor/README.md @@ -40,11 +40,11 @@ the trusted timing signal. ## PR E2E check -`.github/workflows/pr-e2e-gate.yaml` owns `E2E / PR Gate` after -`CI / Pull Request` completes. It uses the same checked-in risk policy as E2E -Advisor, but rebuilds the plan from GitHub's changed-file list and never -consumes advisor output. It dispatches every selected `requiredJobs` entry and -verifies the resulting E2E evidence. See +`.github/workflows/pr-e2e-gate.yaml` owns `E2E / PR Gate` for PRs from this +repository after `CI / Pull Request` completes. It uses the same checked-in +risk policy as E2E Advisor, but rebuilds the plan from GitHub's changed-file +list and never consumes advisor output. It dispatches every selected +`requiredJobs` entry and verifies the resulting E2E evidence. See [NemoClaw E2E CI](../../test/e2e/README.md) for the full lifecycle. E2E Advisor remains advisory. It uses the risk policy as a recommendation diff --git a/tools/e2e/pr-e2e-gate.mts b/tools/e2e/pr-e2e-gate.mts index ef655ff9abd..8e69cd0b956 100755 --- a/tools/e2e/pr-e2e-gate.mts +++ b/tools/e2e/pr-e2e-gate.mts @@ -901,6 +901,9 @@ export async function startPrGate( if (!SHA_PATTERN.test(command.workflowSha)) throw new Error("workflow SHA is invalid"); assertRepository(command.headRepository, "PR head repository"); assertBranch(command.headBranch); + if (command.headRepository !== repository) { + throw new Error("PR branch must be in the base repository"); + } const checkRunId = await createCheck( repository, diff --git a/tools/pr-review-advisor/README.md b/tools/pr-review-advisor/README.md index 69367503bd3..aa297b240f7 100644 --- a/tools/pr-review-advisor/README.md +++ b/tools/pr-review-advisor/README.md @@ -56,10 +56,10 @@ before the failure so later runs and reviewers do not lose substantive review hi The workflow is advisory and must not be configured as a required status check. It uses the deterministic plan as review context but does not run its jobs. E2E Advisor emits the corresponding plan-backed recommendations separately and likewise does not dispatch E2E. Model availability must -not become the authority for whether a pull request can merge. The PR E2E controller separately -rebuilds the plan from GitHub's changed-file list and dispatches every selected job after -`CI / Pull Request` completes. `E2E / PR Gate` does not consume either advisor's output and does not -make PR Review Advisor a merge gate. +not become the authority for whether a pull request can merge. For PRs from this repository, the PR +E2E controller separately rebuilds the plan from GitHub's changed-file list and dispatches every +selected job after `CI / Pull Request` completes. `E2E / PR Gate` does not consume either advisor's +output and does not make PR Review Advisor a merge gate. Required-check status is point-in-time context, not a settled-CI gate. Earlier `PR_REVIEW_ADVISOR_WAIT_*` workflow variables were inert and have been removed; any future waiting From 6a940a985692169092e6eccddfa2746b8f3b6c23 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Fri, 10 Jul 2026 16:44:10 -0700 Subject: [PATCH 7/7] fix(e2e): reject fork cancellation events Signed-off-by: Carlos Villela --- .github/workflows/pr-e2e-gate.yaml | 2 +- test/pr-e2e-gate-workflow.test.ts | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-e2e-gate.yaml b/.github/workflows/pr-e2e-gate.yaml index baf263797c2..95b55475f0e 100644 --- a/.github/workflows/pr-e2e-gate.yaml +++ b/.github/workflows/pr-e2e-gate.yaml @@ -14,7 +14,7 @@ permissions: {} jobs: cancel-superseded: - if: ${{ github.event_name == 'pull_request_target' && github.repository == 'NVIDIA/NemoClaw' }} + if: ${{ github.event_name == 'pull_request_target' && github.repository == 'NVIDIA/NemoClaw' && github.event.pull_request.head.repo.full_name == github.repository }} runs-on: ubuntu-latest timeout-minutes: 10 permissions: diff --git a/test/pr-e2e-gate-workflow.test.ts b/test/pr-e2e-gate-workflow.test.ts index d742a1ad01f..3460569e798 100644 --- a/test/pr-e2e-gate-workflow.test.ts +++ b/test/pr-e2e-gate-workflow.test.ts @@ -233,6 +233,9 @@ describe("PR E2E gate workflow", () => { }); expect(workflow.permissions).toEqual({}); expect(cancel.if).toContain("github.event_name == 'pull_request_target'"); + expect(cancel.if).toContain( + "github.event.pull_request.head.repo.full_name == github.repository", + ); expect(cancel.permissions).toEqual({ actions: "write", contents: "read" }); expect(coordinate.if).toContain("github.event_name == 'workflow_run'"); expect(coordinate.if).toContain("github.event.workflow_run.event == 'pull_request'");