diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 8d0a63ab293..9b8f7f0f51c 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 name: E2E -run-name: ${{ inputs.risk_correlation != '' && format('E2E risk {0}', inputs.risk_correlation) || format('E2E {0}', github.ref_name) }} +run-name: "${{ inputs.checkout_sha != '' && format('E2E PR #{0} ({1})', inputs.pr_number, inputs.correlation_id) || format('E2E {0}', github.ref_name) }}" on: schedule: @@ -35,39 +35,34 @@ on: default: false type: boolean checkout_sha: - description: Current main commit selected by the trusted post-merge shadow controller. + description: PR head commit selected by the controller. required: false default: "" type: string - risk_plan_hash: - description: Deterministic post-merge risk-plan hash for shadow evidence correlation. + plan_hash: + description: SHA-256 of the selected E2E plan. required: false default: "" type: string - risk_correlation: - description: UUIDv4 correlation id for an exact-commit shadow risk run. + correlation_id: + description: Run correlation ID (UUIDv4). required: false default: "" type: string - risk_shadow: - description: Mark this selective run as post-merge risk-gate shadow evidence. - required: false - default: false - type: boolean permissions: contents: read + pull-requests: read concurrency: - group: e2e-${{ github.ref }}-${{ inputs.risk_shadow && github.run_id || inputs.targets || 'supported' }}-${{ inputs.risk_shadow && 'risk-shadow' || inputs.jobs || 'all-jobs' }} - cancel-in-progress: false + group: e2e-${{ github.ref }}-${{ inputs.checkout_sha != '' && format('pr-{0}', inputs.pr_number) || inputs.targets || 'supported' }}-${{ inputs.checkout_sha != '' && 'pr-gate' || inputs.jobs || 'all-jobs' }} + cancel-in-progress: ${{ inputs.checkout_sha != '' }} env: NEMOCLAW_E2E_EXPECTED_SHA: ${{ inputs.checkout_sha }} - NEMOCLAW_E2E_RISK_PLAN_HASH: ${{ inputs.risk_plan_hash }} - NEMOCLAW_E2E_RISK_CORRELATION: ${{ inputs.risk_correlation }} - NEMOCLAW_E2E_RISK_SHARD: default - NEMOCLAW_E2E_RISK_SHADOW: ${{ inputs.risk_shadow && '1' || '0' }} + NEMOCLAW_E2E_PLAN_HASH: ${{ inputs.plan_hash }} + NEMOCLAW_E2E_CORRELATION_ID: ${{ inputs.correlation_id }} + NEMOCLAW_E2E_SHARD: default jobs: generate-matrix: @@ -83,29 +78,36 @@ jobs: fetch-depth: 0 persist-credentials: false - - name: Validate exact-commit dispatch - if: ${{ inputs.checkout_sha != '' || inputs.risk_plan_hash != '' || inputs.risk_correlation != '' || inputs.risk_shadow }} + - name: Validate controller dispatch + if: ${{ inputs.checkout_sha != '' }} env: CHECKOUT_SHA: ${{ inputs.checkout_sha }} + GITHUB_TOKEN: ${{ github.token }} JOBS: ${{ inputs.jobs }} - PLAN_HASH: ${{ inputs.risk_plan_hash }} - RISK_CORRELATION: ${{ inputs.risk_correlation }} - RISK_SHADOW: ${{ inputs.risk_shadow }} + PLAN_HASH: ${{ inputs.plan_hash }} + PR_NUMBER: ${{ inputs.pr_number }} + CORRELATION_ID: ${{ inputs.correlation_id }} TARGETS: ${{ inputs.targets }} WORKFLOW_EVENT: ${{ github.event_name }} WORKFLOW_REF: ${{ github.ref }} - WORKFLOW_SHA: ${{ github.sha }} run: | set -euo pipefail - [[ "$WORKFLOW_EVENT" == "workflow_dispatch" && "$WORKFLOW_REF" == "refs/heads/main" ]] || { echo "::error::exact-commit runs require a workflow_dispatch from main"; exit 1; } - [[ "$RISK_SHADOW" == "true" ]] || { echo "::error::exact-commit inputs require risk_shadow=true"; exit 1; } + [[ "$WORKFLOW_EVENT" == "workflow_dispatch" && "$WORKFLOW_REF" == "refs/heads/main" ]] || { echo "::error::PR E2E runs must be dispatched from main"; exit 1; } [[ "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ ]] || { echo "::error::checkout_sha must be a lowercase 40-character SHA"; exit 1; } - [[ "$CHECKOUT_SHA" == "$WORKFLOW_SHA" ]] || { echo "::error::checkout_sha must equal the current main workflow commit"; exit 1; } - [[ "$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]] || { echo "::error::checked-out HEAD does not match checkout_sha"; exit 1; } - git merge-base --is-ancestor "$CHECKOUT_SHA" origin/main || { echo "::error::checkout_sha must already be reachable from main"; exit 1; } - [[ "$PLAN_HASH" =~ ^[a-f0-9]{64}$ ]] || { echo "::error::risk_plan_hash must be a lowercase SHA-256"; exit 1; } - [[ "$RISK_CORRELATION" =~ ^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$ ]] || { echo "::error::risk_correlation must be a lowercase UUIDv4"; exit 1; } - [[ -n "$JOBS" && -z "$TARGETS" ]] || { echo "::error::shadow risk runs require selective jobs and forbid targets/fan-out"; exit 1; } + [[ "$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]] || { echo "::error::checked-out commit does not match checkout_sha"; exit 1; } + [[ "$PLAN_HASH" =~ ^[a-f0-9]{64}$ ]] || { echo "::error::plan_hash must be a lowercase SHA-256"; exit 1; } + [[ "$CORRELATION_ID" =~ ^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$ ]] || { echo "::error::correlation_id must be a lowercase UUIDv4"; exit 1; } + [[ "$PR_NUMBER" =~ ^[1-9][0-9]*$ ]] || { echo "::error::pr_number must be a positive integer"; exit 1; } + [[ -n "$JOBS" && -z "$TARGETS" ]] || { echo "::error::PR E2E runs require jobs and do not accept targets"; exit 1; } + + pull_json="$(curl --fail --silent --show-error --proto '=https' \ + --header "Authorization: Bearer ${GITHUB_TOKEN}" \ + --header "Accept: application/vnd.github+json" \ + --header "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}")" + [[ "$(jq -r '.state' <<< "$pull_json")" == "open" ]] || { echo "::error::pull request must still be open"; exit 1; } + [[ "$(jq -r '.head.repo.full_name // ""' <<< "$pull_json")" == "$GITHUB_REPOSITORY" ]] || { echo "::error::pull request must originate from this repository"; exit 1; } + [[ "$(jq -r '.head.sha' <<< "$pull_json")" == "$CHECKOUT_SHA" ]] || { echo "::error::checkout_sha must match the PR head commit"; exit 1; } - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 @@ -3036,7 +3038,7 @@ jobs: # set, and NoNewPrivs remain evidence unless an opt-in expectation is set. NEMOCLAW_E2E_EXPECT_NON_ROOT_HOST: "1" NEMOCLAW_E2E_SECURITY_POSTURE: "1" - NEMOCLAW_E2E_RISK_SHARD: ${{ matrix.agent }} + NEMOCLAW_E2E_SHARD: ${{ matrix.agent }} NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS: "60" NEMOCLAW_RECREATE_SANDBOX: "1" @@ -4559,7 +4561,7 @@ jobs: NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" - NEMOCLAW_E2E_RISK_SHARD: ${{ matrix.agent }} + NEMOCLAW_E2E_SHARD: ${{ matrix.agent }} NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" NEMOCLAW_AGENT: ${{ matrix.agent }} @@ -5035,7 +5037,7 @@ jobs: channels-stop-start, spark-install, ] - if: ${{ always() && github.event_name == 'workflow_dispatch' && !inputs.risk_shadow }} + if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '' }} permissions: # The issue-comment endpoint accepts pull request write permission for PR comments. # Keep issues: write absent so this job cannot restore general issue routing. @@ -5238,7 +5240,7 @@ jobs: scorecard: runs-on: ubuntu-latest needs: *e2e-result-jobs - if: ${{ always() && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && !inputs.risk_shadow)) }} + if: ${{ always() && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '')) }} permissions: actions: read contents: read diff --git a/.github/workflows/post-merge-e2e-risk-gate-shadow.yaml b/.github/workflows/post-merge-e2e-risk-gate-shadow.yaml deleted file mode 100644 index d05a6598723..00000000000 --- a/.github/workflows/post-merge-e2e-risk-gate-shadow.yaml +++ /dev/null @@ -1,249 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -name: E2E / Post-merge Risk Gate Shadow - -on: - push: - branches: [main] - -permissions: - # Required to dispatch the separate E2E workflow; no other Actions mutation is performed. - actions: write - checks: write - contents: read - -concurrency: - group: e2e-post-merge-risk-gate-${{ github.sha }} - cancel-in-progress: false - -jobs: - shadow: - if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' }} - runs-on: ubuntu-latest - timeout-minutes: 120 - steps: - - name: Checkout trusted controller - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - ref: ${{ github.event.after }} - fetch-depth: 0 - persist-credentials: false - - - name: Setup Node - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 - with: - node-version: "22" - cache: npm - - - name: Install trusted controller dependencies - run: npm ci --ignore-scripts - - - id: workspace - name: Create private controller workspace - shell: bash - run: | - set -euo pipefail - work_dir="$(mktemp -d "${RUNNER_TEMP}/nemoclaw-e2e-risk-gate.XXXXXX")" - chmod 700 "$work_dir" - printf 'work_dir=%s\n' "$work_dir" >> "$GITHUB_OUTPUT" - - - id: start - name: Build plan and dispatch exact-commit E2E - env: - GITHUB_TOKEN: ${{ github.token }} - run: >- - node --experimental-strip-types tools/e2e-advisor/post-merge-risk-gate.mts - --mode start - --base "${{ github.event.before }}" - --commit "${{ github.event.after }}" - --work-dir "${{ steps.workspace.outputs.work_dir }}" - - - name: Upload post-merge risk plan - if: ${{ always() }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: post-merge-risk-plan-${{ github.event.after }} - path: ${{ steps.workspace.outputs.work_dir }}/post-merge-risk-plan.json - if-no-files-found: ignore - retention-days: 14 - - - name: Close shadow check after controller startup failure - if: ${{ always() && steps.start.outputs.check_id != '' && steps.start.outputs.dispatched != 'true' && steps.start.outputs.finalized != 'true' }} - env: - GITHUB_TOKEN: ${{ github.token }} - run: >- - node --experimental-strip-types tools/e2e-advisor/post-merge-risk-gate.mts - --mode abandon - --check-id "${{ steps.start.outputs.check_id }}" - - - id: wait - name: Wait for correlated E2E run - if: ${{ steps.start.outputs.dispatched == 'true' }} - continue-on-error: true - env: - GH_TOKEN: ${{ github.token }} - RUN_ID: ${{ steps.start.outputs.run_id }} - run: | - wait_status=0 - timeout --signal=TERM --kill-after=30s 105m bash -s <<'WAIT' || wait_status=$? - set -euo pipefail - - if [[ ! "$RUN_ID" =~ ^[1-9][0-9]*$ ]]; then - printf '::error title=Invalid correlated E2E run ID::The controller did not provide a positive numeric run ID.\n' >&2 - exit 1 - fi - - run_url="https://github.com/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}" - - last_state="" - while true; do - if ! state="$( - gh run view "$RUN_ID" --repo "$GITHUB_REPOSITORY" \ - --json status,conclusion \ - --jq '.status + ":" + (if (.conclusion == null or .conclusion == "") then "none" else .conclusion end)' - )"; then - printf '::error title=Correlated E2E status query failed::Unable to query child run %s. %s\n' \ - "$RUN_ID" "$run_url" >&2 - exit 1 - fi - - if [[ "$state" != "$last_state" ]]; then - case "$state" in - queued:none | in_progress:none | requested:none | waiting:none | pending:none) - printf 'Correlated E2E run %s status=%s url=%s\n' \ - "$RUN_ID" "${state%%:*}" "$run_url" - ;; - completed:success) - printf 'Correlated E2E run %s status=completed conclusion=success url=%s\n' \ - "$RUN_ID" "$run_url" - ;; - completed:failure | completed:cancelled | completed:timed_out | completed:action_required | completed:neutral | completed:skipped | completed:stale | completed:startup_failure) - printf '::error title=Correlated E2E run did not succeed::Run %s completed with conclusion %s. %s\n' \ - "$RUN_ID" "${state#*:}" "$run_url" >&2 - ;; - *) - printf '::error title=Unexpected correlated E2E state::Run %s returned an unsupported status/conclusion pair. %s\n' \ - "$RUN_ID" "$run_url" >&2 - ;; - esac - last_state="$state" - fi - - case "$state" in - queued:none | in_progress:none | requested:none | waiting:none | pending:none) - sleep 10 - ;; - completed:success) - exit 0 - ;; - *) - exit 1 - ;; - esac - done - WAIT - - if [ "$wait_status" -eq 124 ]; then - printf '::error title=Correlated E2E wait timed out::The child run did not complete within 105 minutes.\n' >&2 - fi - exit "$wait_status" - - - id: evidence - name: Download correlated E2E evidence - if: ${{ always() && steps.start.outputs.dispatched == 'true' }} - continue-on-error: true - env: - GH_TOKEN: ${{ github.token }} - RUN_ID: ${{ steps.start.outputs.run_id }} - run: >- - gh run download "$RUN_ID" --repo "$GITHUB_REPOSITORY" - --dir "${{ steps.workspace.outputs.work_dir }}/evidence" - - - id: finish - name: Complete exact-commit shadow check - if: ${{ always() && steps.start.outputs.dispatched == 'true' }} - env: - GITHUB_TOKEN: ${{ github.token }} - run: >- - node --experimental-strip-types tools/e2e-advisor/post-merge-risk-gate.mts - --mode finish - --work-dir "${{ steps.workspace.outputs.work_dir }}" - --state-hash "${{ steps.start.outputs.state_hash }}" - --check-id "${{ steps.start.outputs.check_id }}" - --run-id "${{ steps.start.outputs.run_id }}" - - - name: Close shadow check after completion failure - if: ${{ always() && steps.start.outputs.check_id != '' && steps.start.outputs.dispatched == 'true' && steps.finish.outcome == 'failure' && steps.finish.outputs.finalized != 'true' }} - env: - GITHUB_TOKEN: ${{ github.token }} - run: >- - node --experimental-strip-types tools/e2e-advisor/post-merge-risk-gate.mts - --mode abandon - --check-id "${{ steps.start.outputs.check_id }}" - - - name: Summarize shadow controller - if: ${{ always() }} - env: - GH_TOKEN: ${{ github.token }} - COMMIT_SHA: ${{ github.event.after }} - CHILD_RUN_ID: ${{ steps.start.outputs.run_id }} - WORK_DIR: ${{ steps.workspace.outputs.work_dir }} - START_OUTCOME: ${{ steps.start.outcome }} - WAIT_OUTCOME: ${{ steps.wait.outcome }} - EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }} - FINISH_OUTCOME: ${{ steps.finish.outcome }} - run: | - set -euo pipefail - - selected_jobs="unavailable" - plan_path="${WORK_DIR}/post-merge-risk-plan.json" - if [ -n "$WORK_DIR" ] && [ -f "$plan_path" ]; then - selected_jobs="$(node -e ' - const fs = require("node:fs"); - const plan = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); - process.stdout.write(plan.automaticJobs.join(",") || "none"); - ' "$plan_path" 2>/dev/null || printf unavailable)" - fi - - child_conclusion="not-dispatched" - if [ -n "$CHILD_RUN_ID" ]; then - child_conclusion="$( - gh run view "$CHILD_RUN_ID" --repo "$GITHUB_REPOSITORY" \ - --json conclusion --jq '.conclusion // "none"' 2>/dev/null || printf unavailable - )" - fi - - failure_phase="none" - if [ "$START_OUTCOME" = "failure" ]; then - failure_phase="plan-and-dispatch" - elif [ "$FINISH_OUTCOME" = "failure" ]; then - failure_phase="evidence-verification" - elif [ "$WAIT_OUTCOME" = "failure" ]; then - failure_phase="child-run-or-wait" - elif [ "$EVIDENCE_OUTCOME" = "failure" ]; then - failure_phase="evidence-download" - fi - - { - echo "## Post-merge E2E risk gate shadow" - echo - printf -- '- Controller run: [run `%s`](https://github.com/%s/actions/runs/%s)\n' \ - "$GITHUB_RUN_ID" "$GITHUB_REPOSITORY" "$GITHUB_RUN_ID" - printf -- '- Controller commit: `%s`\n' "$COMMIT_SHA" - printf -- '- Selected jobs: `%s`\n' "$selected_jobs" - printf -- '- Plan and dispatch: `%s`\n' "$START_OUTCOME" - if [ -n "$CHILD_RUN_ID" ]; then - printf -- '- Correlated E2E: [run `%s`](https://github.com/%s/actions/runs/%s)\n' \ - "$CHILD_RUN_ID" "$GITHUB_REPOSITORY" "$CHILD_RUN_ID" - fi - printf -- '- Child conclusion: `%s`\n' "$child_conclusion" - printf -- '- Child wait: `%s`\n' "$WAIT_OUTCOME" - printf -- '- Evidence download: `%s`\n' "$EVIDENCE_OUTCOME" - printf -- '- Evidence verification: `%s`\n' "$FINISH_OUTCOME" - printf -- '- Failure phase: `%s`\n' "$failure_phase" - } >> "$GITHUB_STEP_SUMMARY" - - - name: Remove private controller workspace - if: ${{ always() && steps.workspace.outputs.work_dir != '' }} - run: rm -rf -- "${{ steps.workspace.outputs.work_dir }}" diff --git a/.github/workflows/pr-e2e-gate.yaml b/.github/workflows/pr-e2e-gate.yaml new file mode 100644 index 00000000000..95b55475f0e --- /dev/null +++ b/.github/workflows/pr-e2e-gate.yaml @@ -0,0 +1,227 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: E2E / PR Gate + +on: + workflow_run: + workflows: ["CI / Pull Request"] + types: [completed] + pull_request_target: + types: [synchronize, reopened, closed] + +permissions: {} + +jobs: + cancel-superseded: + if: ${{ github.event_name == 'pull_request_target' && github.repository == 'NVIDIA/NemoClaw' && github.event.pull_request.head.repo.full_name == github.repository }} + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + actions: write + contents: read + steps: + - name: Checkout controller + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: ${{ github.workflow_sha }} + persist-credentials: false + + - name: Setup Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: "22" + + - name: Install controller dependencies + run: npm ci --ignore-scripts + + - name: Cancel superseded E2E runs + env: + GITHUB_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: >- + node --experimental-strip-types tools/e2e/pr-e2e-gate.mts + --mode cancel + --pr "$PR_NUMBER" + + coordinate: + if: ${{ github.event_name == 'workflow_run' && github.repository == 'NVIDIA/NemoClaw' && github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.head_repository.full_name == github.repository }} + runs-on: ubuntu-latest + timeout-minutes: 180 + permissions: + actions: write + checks: write + contents: read + pull-requests: read + concurrency: + group: pr-e2e-gate-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }} + # Let the previous coordinator observe E2E cancellation and close its check. + cancel-in-progress: false + steps: + - name: Checkout controller + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: ${{ github.workflow_sha }} + persist-credentials: false + + - name: Setup Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: "22" + + - name: Install controller dependencies + run: npm ci --ignore-scripts + + - id: workspace + name: Create private workspace + shell: bash + run: | + set -euo pipefail + work_dir="$(mktemp -d "${RUNNER_TEMP}/nemoclaw-pr-e2e-gate.XXXXXX")" + chmod 700 "$work_dir" + printf 'work_dir=%s\n' "$work_dir" >> "$GITHUB_OUTPUT" + + - id: start + name: Start evaluation + env: + CI_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + GITHUB_TOKEN: ${{ github.token }} + HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + HEAD_REPOSITORY: ${{ github.event.workflow_run.head_repository.full_name }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + WORK_DIR: ${{ steps.workspace.outputs.work_dir }} + run: >- + node --experimental-strip-types tools/e2e/pr-e2e-gate.mts + --mode start + --head "$HEAD_SHA" + --head-repo "$HEAD_REPOSITORY" + --head-branch "$HEAD_BRANCH" + --workflow-sha "$WORKFLOW_SHA" + --ci-conclusion "$CI_CONCLUSION" + --work-dir "$WORK_DIR" + + - name: Upload risk plan + if: ${{ always() && steps.workspace.outputs.work_dir != '' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: pr-e2e-risk-plan-${{ github.event.workflow_run.head_sha }} + path: ${{ steps.workspace.outputs.work_dir }}/risk-plan.json + if-no-files-found: ignore + retention-days: 14 + + - id: wait + name: Wait for E2E run + if: ${{ steps.start.outputs.dispatched == 'true' }} + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + RUN_ID: ${{ steps.start.outputs.run_id }} + run: | + wait_status=0 + timeout --signal=TERM --kill-after=30s 105m bash -s <<'WAIT' || wait_status=$? + set -euo pipefail + + if [[ ! "$RUN_ID" =~ ^[1-9][0-9]*$ ]]; then + printf '::error title=Invalid run ID::The controller did not provide a positive numeric run ID.\n' >&2 + exit 1 + fi + + run_url="https://github.com/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}" + last_state="" + while true; do + if ! state="$( + gh run view "$RUN_ID" --repo "$GITHUB_REPOSITORY" \ + --json status,conclusion \ + --jq '.status + ":" + (if (.conclusion == null or .conclusion == "") then "none" else .conclusion end)' + )"; then + printf '::error title=Run status query failed::Unable to query run %s. %s\n' \ + "$RUN_ID" "$run_url" >&2 + exit 1 + fi + + if [[ "$state" != "$last_state" ]]; then + case "$state" in + queued:none | in_progress:none | requested:none | waiting:none | pending:none) + printf 'Run %s status=%s url=%s\n' \ + "$RUN_ID" "${state%%:*}" "$run_url" + ;; + completed:success) + printf 'Run %s status=completed conclusion=success url=%s\n' \ + "$RUN_ID" "$run_url" + ;; + completed:failure | completed:cancelled | completed:timed_out | completed:action_required | completed:neutral | completed:skipped | completed:stale | completed:startup_failure) + printf '::error title=E2E run failed::Run %s completed with conclusion %s. %s\n' \ + "$RUN_ID" "${state#*:}" "$run_url" >&2 + ;; + *) + printf '::error title=Unexpected run state::Run %s returned an unsupported status/conclusion pair. %s\n' \ + "$RUN_ID" "$run_url" >&2 + ;; + esac + last_state="$state" + fi + + case "$state" in + queued:none | in_progress:none | requested:none | waiting:none | pending:none) + sleep 10 + ;; + completed:success) + exit 0 + ;; + *) + exit 1 + ;; + esac + done + WAIT + + if [ "$wait_status" -eq 124 ]; then + printf '::error title=E2E run timed out::The run did not complete within 105 minutes.\n' >&2 + fi + exit "$wait_status" + + - id: evidence + name: Download evidence + if: ${{ always() && steps.start.outputs.dispatched == 'true' }} + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + RUN_ID: ${{ steps.start.outputs.run_id }} + run: | + set -euo pipefail + download_status=0 + timeout --signal=TERM --kill-after=30s 10m \ + gh run download "$RUN_ID" --repo "$GITHUB_REPOSITORY" \ + --dir "${{ steps.workspace.outputs.work_dir }}/evidence" || download_status=$? + if [ "$download_status" -eq 124 ]; then + printf '::error title=Evidence download timed out::Artifact download exceeded 10 minutes.\n' >&2 + fi + exit "$download_status" + + - id: finish + name: Verify evidence + if: ${{ always() && steps.start.outputs.dispatched == 'true' }} + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + node --experimental-strip-types tools/e2e/pr-e2e-gate.mts + --mode finish + --work-dir "${{ steps.workspace.outputs.work_dir }}" + --state-hash "${{ steps.start.outputs.state_hash }}" + --check-id "${{ steps.start.outputs.check_id }}" + --run-id "${{ steps.start.outputs.run_id }}" + + - name: Close incomplete check + if: ${{ always() && steps.start.outputs.check_id != '' && steps.start.outputs.finalized != 'true' && steps.finish.outputs.finalized != 'true' }} + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + node --experimental-strip-types tools/e2e/pr-e2e-gate.mts + --mode abandon + --check-id "${{ steps.start.outputs.check_id }}" + --run-id "${{ steps.start.outputs.run_id }}" + + - name: Remove private workspace + if: ${{ always() && steps.workspace.outputs.work_dir != '' }} + run: rm -rf -- "${{ steps.workspace.outputs.work_dir }}" diff --git a/docs/about/release-notes.mdx b/docs/about/release-notes.mdx index e20e2dbfba0..8db02d3e063 100644 --- a/docs/about/release-notes.mdx +++ b/docs/about/release-notes.mdx @@ -42,7 +42,7 @@ The release also refreshes quickstarts and variant rendering so OpenClaw, Hermes Resume recovery now follows one explicit finite-state path, pending route reservations survive resume, sandbox create-failure reporting is separated from create-step handling, BuildKit progress no longer forces plain output, and null-name resume sessions are covered so canceled or malformed session state does not send users down the wrong recovery path. For more information, refer to [NemoClaw Quickstart with OpenClaw](../get-started/quickstart), [NemoClaw CLI Commands Reference](../reference/commands), [Manage Sandbox Lifecycle](../manage-sandboxes/lifecycle), and [Troubleshooting](../reference/troubleshooting). - Documentation and release validation are more deterministic. - The docs define extension taxonomy and SDK readiness gates, streamline the agent quickstarts, clarify legacy k3s sandbox resources, preserve list spacing in generated agent-variant pages, and add release-train risk planning with post-merge E2E shadow signals, queued Jetson dispatch guards, TUI idle regression coverage, and reusable live-readiness polling primitives. + The docs define extension taxonomy and SDK readiness gates, streamline the agent quickstarts, clarify legacy k3s sandbox resources, preserve list spacing in generated agent-variant pages, and add release-train risk planning with a PR E2E check, queued Jetson dispatch guards, TUI idle regression coverage, and reusable live-readiness polling primitives. For more information, refer to [Extension Taxonomy and SDK Readiness](../reference/extension-taxonomy-sdk-readiness), [NemoClaw Quickstart with OpenClaw](../get-started/quickstart), [Architecture Details](../reference/architecture), and the [NemoClaw E2E README](https://github.com/NVIDIA/NemoClaw/blob/main/test/e2e/README.md). ## v0.0.78 diff --git a/test/e2e-advisor-targets.test.ts b/test/e2e-advisor-targets.test.ts index 1d1f6c1cff3..8ad25d5a313 100644 --- a/test/e2e-advisor-targets.test.ts +++ b/test/e2e-advisor-targets.test.ts @@ -68,7 +68,7 @@ describe("E2E target advisor — prompt construction", () => { expect(turn.contextToolResults?.[1]?.content).toContain( "test/e2e/fixtures/phases/onboarding.ts", ); - expect(turn.contextToolResults?.[2]?.content).toContain('"version":1'); + expect(turn.contextToolResults?.[2]?.content).toContain('"version":2'); expect(turn.contextToolResults?.[3]?.content).toContain("+ echo ok"); expect(turn.contextToolResults?.[4]?.content).toContain("test-schema"); for (const result of turn.contextToolResults ?? []) { diff --git a/test/e2e-private-file.test.ts b/test/e2e-private-file.test.ts index fe4c54c8645..dc8d779df9b 100644 --- a/test/e2e-private-file.test.ts +++ b/test/e2e-private-file.test.ts @@ -8,10 +8,7 @@ import path from "node:path"; import { pathToFileURL } from "node:url"; import { describe, expect, it } from "vitest"; -import { - readPrivateRegularFile, - writePrivateRegularFile, -} from "../tools/e2e-advisor/private-file.ts"; +import { readPrivateRegularFile, writePrivateRegularFile } from "../tools/e2e/private-file.ts"; describe("private E2E controller files", () => { it("writes private regular files without following links or truncating hardlink targets", () => { @@ -44,7 +41,7 @@ describe("private E2E controller files", () => { const fifo = path.join(directory, "state.json"); try { execFileSync("mkfifo", [fifo]); - const moduleUrl = pathToFileURL(path.resolve("tools/e2e-advisor/private-file.ts")).href; + const moduleUrl = pathToFileURL(path.resolve("tools/e2e/private-file.ts")).href; const read = spawnSync( process.execPath, [ diff --git a/test/e2e-risk-signal-reporter.test.ts b/test/e2e-risk-signal-reporter.test.ts index b45f4f47105..61cf4e654cc 100644 --- a/test/e2e-risk-signal-reporter.test.ts +++ b/test/e2e-risk-signal-reporter.test.ts @@ -41,12 +41,12 @@ function environment(artifactDir: string): RiskSignalEnvironment { } describe("E2E risk signal reporter", () => { - it("stays disabled outside shadow runs", () => { + it("stays disabled when no expected commit is configured", () => { expect(configuredEnvironment({})).toBeNull(); }); - it("fails closed when shadow metadata is incomplete", () => { - expect(() => configuredEnvironment({ NEMOCLAW_E2E_RISK_SHADOW: "1" })).toThrow( + it("fails closed when run metadata is incomplete", () => { + expect(() => configuredEnvironment({ NEMOCLAW_E2E_EXPECTED_SHA: EXPECTED_SHA })).toThrow( /E2E_ARTIFACT_DIR/u, ); }); @@ -57,10 +57,9 @@ describe("E2E risk signal reporter", () => { E2E_TARGET_ID: "onboard-resume", GITHUB_WORKSPACE: "/workspace", NEMOCLAW_E2E_EXPECTED_SHA: EXPECTED_SHA, - NEMOCLAW_E2E_RISK_PLAN_HASH: PLAN_HASH, - NEMOCLAW_E2E_RISK_CORRELATION: CORRELATION_ID, - NEMOCLAW_E2E_RISK_SHARD: "default", - NEMOCLAW_E2E_RISK_SHADOW: "1", + NEMOCLAW_E2E_PLAN_HASH: PLAN_HASH, + NEMOCLAW_E2E_CORRELATION_ID: CORRELATION_ID, + NEMOCLAW_E2E_SHARD: "default", }; expect(configuredEnvironment(env, () => EXPECTED_SHA)?.testedSha).toBe(EXPECTED_SHA); diff --git a/test/e2e/README.md b/test/e2e/README.md index 8709ec27630..4c7530d5030 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -8,11 +8,10 @@ Direct E2E coverage runs through Vitest. Interactive TUI targets require `expect`. The unified workflow installs it before those targets run; local runners must provide it themselves. -- `.github/workflows/e2e.yaml` is the scheduled and manually - dispatchable live target workflow. -- `.github/workflows/post-merge-e2e-risk-gate-shadow.yaml` is the trusted post-merge - controller that selects and dispatches a bounded exact-commit subset after - pushes to `main`. +- `.github/workflows/e2e.yaml` is the scheduled, manually dispatchable, and + selectively dispatched live target workflow. +- `.github/workflows/pr-e2e-gate.yaml` is the PR controller for + `E2E / PR Gate`. - `.github/workflows/e2e-branch-validation.yaml` provisions Brev instances and runs focused E2E targets from source on a clean machine. - Platform workflows such as macOS, WSL, Ollama proxy, sandbox image, and @@ -56,41 +55,50 @@ artifact so baseline aggregation stays stable. Older issue references to Vitest target artifacts under `e2e-artifacts/vitest/` map to this consolidated `e2e-artifacts/live/` registry-target artifact layout. -## Post-merge risk shadow - -Every push to the `main` branch of `NVIDIA/NemoClaw` starts a model-independent -shadow controller. It builds the deterministic risk plan from the exact -`github.event.before` and `github.event.after` range after confirming that its checkout -matches the pushed commit. If the plan matches runtime regression families, -the controller dispatches at most three `automaticJobs` through `e2e.yaml`. -The workflow definition stays on `main`, while every E2E checkout uses the -merged commit supplied through `checkout_sha`. GitHub returns the -dispatched workflow's run ID directly, and the controller uses that ID as the -sole child-run selector for waiting, evidence download, and completion. - -Before E2E preparation or selected jobs can use repository secrets, -`e2e.yaml` verifies that the requested SHA equals the workflow's own current -`main` commit, confirms that checked-out `HEAD` matches it, proves reachability, -and accepts only selective job dispatch with valid plan and correlation -metadata. If `main` advances before an older controller dispatches, that child -fails closed and the controller records failure instead of running historical -code with current secrets. The shadow-only Vitest -reporter then writes a `risk-signal.json` for each selected job and matrix -shard. Each signal binds the observed checkout SHA, expected SHA, plan hash, +## PR E2E check + +When `CI / Pull Request` completes for a PR from this repository, +`.github/workflows/pr-e2e-gate.yaml` creates `E2E / PR Gate` for the PR head +commit. The controller reads all changed files and builds the deterministic +risk plan. If a runtime risk family matches, it dispatches every selected +`requiredJobs` entry through `e2e.yaml`; otherwise the check passes without an +E2E run. + +Before dispatch, the controller verifies that the PR is unchanged and that +`main` still points to its workflow commit. It accepts only an E2E run using +that commit. Each selected job checks out `checkout_sha`. Before preparation or +secret-bearing jobs can run, `e2e.yaml` verifies that the PR remains open, +belongs to `NVIDIA/NemoClaw`, and still has that head commit. The dispatch +includes selected jobs and valid plan and correlation metadata, but not +`targets`. The controller uses GitHub's returned run ID for waiting, evidence +download, and completion. + +The Vitest reporter writes one `risk-signal.json` for each selected job and +matrix shard. +The checked workflow boundary requires every policy-selected job to expose its +matching job identity, attach the reporter to every Vitest invocation, and +always upload its evidence artifact. +Each signal binds the observed checkout SHA, expected SHA, plan hash, correlation ID, and pass, failure, skip, pending, and unhandled-error counts. -The controller retains `post-merge-risk-plan-` for 14 days, while each +The controller retains `pr-e2e-risk-plan-` for 14 days, while each signal travels in the selected job's existing E2E artifact. - -The controller reports `E2E / Post-merge Risk Gate (shadow)` on the merged -commit. It reports success only when every expected shard produces a complete, -unskipped pass and the three-job cap did not omit required jobs. Selected E2E -workflow or test failures for the merged commit report failure. Missing, partial, skipped, -ambiguous, or manual-expansion evidence reports neutral. A plan with no matched -runtime risk reports success without dispatching live E2E. This shadow check runs -after merge and is not a required PR check. It disables PR comments and the -scheduled/manual scorecard, including scorecard Slack reporting. -Controller or evidence-verification errors close an already-created check as -neutral so incomplete evidence cannot appear successful. +Its private dispatch state is protected by a SHA-256 digest that is verified +before downloaded evidence is classified. + +When the plan selects jobs, the check passes only when the E2E run succeeds and +every expected job shard uploads one complete passing signal with no skips or +pending tests. Every other dispatched outcome fails. +The coordinator has a 180-minute job budget and gives evidence download its +own 10-minute limit, so a stalled download fails instead of consuming the +remaining coordination time. +These dispatches suppress PR comments and the scheduled or manual +scorecard, including scorecard Slack reporting. + +Synchronizing, reopening, or closing the PR cancels its active E2E runs. A new +dispatch also cancels the previous run, while the previous controller remains +available to close its check as failed. +The controller does not read PR Review Advisor or E2E Advisor output, so model +availability and recommendations are not part of merge authority. ## Onboard performance budget diff --git a/test/e2e/docs/README.md b/test/e2e/docs/README.md index 27eaceda958..f4f155f0334 100644 --- a/test/e2e/docs/README.md +++ b/test/e2e/docs/README.md @@ -106,16 +106,17 @@ test/e2e/ ## CI Entry Points - `tools/advisors/risk-plan.mts` is the small deterministic selection policy - shared by PR Review Advisor, E2E Advisor, and the model-independent post-merge - shadow controller. It maps changed runtime surfaces to invariant families and + shared by PR Review Advisor, E2E Advisor, and the PR E2E controller. It maps + changed runtime surfaces to invariant families and canonical `e2e.yaml` jobs; it is not a second test runner or migration-status - ledger. + ledger. The advisors use it as recommendation context, while the controller + applies it independently without model output. -- `.github/workflows/post-merge-e2e-risk-gate-shadow.yaml` runs only for trusted pushes to - `main`. `tools/e2e-advisor/post-merge-risk-gate.mts` builds a plan from the exact - before/after SHAs, dispatches at most three automatic jobs, and validates - `risk-signal.json` evidence for every expected job and matrix shard. The - resulting check is post-merge shadow evidence, not a required PR gate. +- `.github/workflows/pr-e2e-gate.yaml` owns `E2E / PR Gate` for PRs from this + repository after `CI / Pull Request` completes. The controller builds the + risk plan from GitHub's complete file list, dispatches every selected job, + and verifies each expected `risk-signal.json`. See + [NemoClaw E2E CI](../README.md) for the full lifecycle. - `.github/workflows/e2e.yaml` runs selected or all supported live E2E targets and uploads an explicit artifact allowlist with @@ -128,12 +129,10 @@ test/e2e/ These per-target timing summaries are artifact evidence only. The Slack and GitHub scorecard timing comparison remains scoped to the dedicated `cloud-onboard` artifact. - Exact-commit shadow dispatches require the requested checkout to equal the - workflow's current `main` commit and verify its reachability before - preparation. The controller uses GitHub's returned workflow-dispatch run ID - as the sole child-run selector for waiting, evidence download, and - completion, attaches `test/e2e/risk-signal-reporter.ts` to live Vitest - invocations, and suppresses PR reporting and scorecards. + PR E2E dispatches validate the PR head commit and controller metadata before + preparation, attach `test/e2e/risk-signal-reporter.ts` to live Vitest + invocations, and suppress PR reporting and scorecards. The workflow boundary + requires every selected job shard to upload its evidence artifact. - `.github/workflows/e2e-branch-validation.yaml`, `macos-e2e.yaml`, `wsl-e2e.yaml`, `ollama-proxy-e2e.yaml`, and `regression-e2e.yaml` call focused E2E targets directly for their E2E coverage. diff --git a/test/e2e/risk-signal-reporter.ts b/test/e2e/risk-signal-reporter.ts index 138ffeec14b..d500d250c84 100644 --- a/test/e2e/risk-signal-reporter.ts +++ b/test/e2e/risk-signal-reporter.ts @@ -7,11 +7,8 @@ import path from "node:path"; import type { TestModule } from "vitest/node"; import type { Reporter, TestRunEndReason } from "vitest/reporters"; -import { - readPrivateRegularFile, - writePrivateRegularFile, -} from "../../tools/e2e-advisor/private-file.ts"; -import type { E2eRiskSignal } from "../../tools/e2e-advisor/risk-signal.ts"; +import { readPrivateRegularFile, writePrivateRegularFile } from "../../tools/e2e/private-file.ts"; +import type { E2eRiskSignal } from "../../tools/e2e/risk-signal.ts"; export const RISK_SIGNAL_FILE = "risk-signal.json"; @@ -44,14 +41,14 @@ export function configuredEnvironment( env: NodeJS.ProcessEnv, resolveHead: (workspace: string) => string = checkedOutSha, ): RiskSignalEnvironment | null { - if (env.NEMOCLAW_E2E_RISK_SHADOW !== "1") return null; + if (!env.NEMOCLAW_E2E_EXPECTED_SHA) return null; const values = { artifactDir: env.E2E_ARTIFACT_DIR ?? "", jobId: env.E2E_TARGET_ID ?? "", - shardId: env.NEMOCLAW_E2E_RISK_SHARD ?? "", - expectedSha: env.NEMOCLAW_E2E_EXPECTED_SHA ?? "", - planHash: env.NEMOCLAW_E2E_RISK_PLAN_HASH ?? "", - correlationId: env.NEMOCLAW_E2E_RISK_CORRELATION ?? "", + shardId: env.NEMOCLAW_E2E_SHARD ?? "", + expectedSha: env.NEMOCLAW_E2E_EXPECTED_SHA, + planHash: env.NEMOCLAW_E2E_PLAN_HASH ?? "", + correlationId: env.NEMOCLAW_E2E_CORRELATION_ID ?? "", }; if (!values.artifactDir) throw new Error("risk signal requires E2E_ARTIFACT_DIR"); if (!JOB_PATTERN.test(values.jobId)) throw new Error("risk signal requires a safe E2E_TARGET_ID"); diff --git a/test/e2e/support/e2e-operations-workflow-boundary.test.ts b/test/e2e/support/e2e-operations-workflow-boundary.test.ts index ec616eb535b..44f68f62cc9 100644 --- a/test/e2e/support/e2e-operations-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-operations-workflow-boundary.test.ts @@ -57,7 +57,7 @@ describe("E2E operations workflow boundary", () => { ); }); - it("keeps PR reporting and scorecards disabled for E2E risk shadow runs", () => { + it("keeps PR reporting and scorecards disabled for PR E2E runs", () => { const workflow = readE2eOperationsWorkflow(); workflow.jobs["report-to-pr"].if = "${{ always() && github.event_name == 'workflow_dispatch' }}"; @@ -72,6 +72,53 @@ describe("E2E operations workflow boundary", () => { ); }); + it("rejects controller protocol and PR validation drift", () => { + const workflow = readE2eOperationsWorkflow(); + delete workflow.on?.workflow_dispatch?.inputs?.plan_hash; + workflow.env!.NEMOCLAW_E2E_PLAN_HASH = "${{ inputs.checkout_sha }}"; + workflow.concurrency!["cancel-in-progress"] = false; + const validation = workflow.jobs["generate-matrix"].steps!.find( + (step) => step.name === "Validate controller dispatch", + )!; + validation.if = "${{ inputs.plan_hash != '' }}"; + validation.run = "echo unchecked"; + const checkout = workflow.jobs["generate-matrix"].steps!.find((step) => + step.uses?.startsWith("actions/checkout@"), + )!; + checkout.with!.ref = "${{ github.sha }}"; + + expect(validateE2eOperationsWorkflow(workflow)).toEqual( + expect.arrayContaining([ + "workflow_dispatch plan_hash must be an optional string with an empty default", + "E2E workflow must bind NEMOCLAW_E2E_PLAN_HASH to controller metadata", + "PR E2E concurrency must cancel obsolete runs", + "Controller validation must be activated only by checkout_sha", + 'Controller validation must retain "$PR_NUMBER" =~ ^[1-9][0-9]*$', + "generate-matrix checkout must use the selected PR commit", + ]), + ); + }); + + it("keeps every planned job wired to bound evidence", () => { + const workflow = readE2eOperationsWorkflow(); + const job = workflow.jobs["cloud-onboard"]; + job.env!.E2E_TARGET_ID = "different-job"; + const run = job.steps!.find((step) => String(step.run ?? "").includes("npx vitest"))!; + run.run = run.run!.replace("test/e2e/risk-signal-reporter.ts", "default"); + const upload = job.steps!.find((step) => + step.uses?.startsWith("NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@"), + )!; + upload.if = "success()"; + + expect(validateE2eOperationsWorkflow(workflow)).toEqual( + expect.arrayContaining([ + "cloud-onboard must expose matching E2E job identity", + "cloud-onboard must attach the risk-signal reporter to every Vitest invocation", + "cloud-onboard must always upload one evidence artifact", + ]), + ); + }); + it("rejects restoration of scheduled issue routing or broad issue-write access", () => { const workflow = readE2eOperationsWorkflow(); workflow.permissions = "write-all"; diff --git a/test/post-merge-e2e-risk-gate-workflow.test.ts b/test/post-merge-e2e-risk-gate-workflow.test.ts deleted file mode 100644 index 28f8bbd80c4..00000000000 --- a/test/post-merge-e2e-risk-gate-workflow.test.ts +++ /dev/null @@ -1,353 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { spawnSync } from "node:child_process"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { describe, expect, it } from "vitest"; -import { RISK_RULES } from "../tools/advisors/risk-plan.mts"; -import { - readYaml, - type Workflow, - type WorkflowJob, - type WorkflowStep, -} from "./helpers/e2e-workflow-contract.ts"; - -const SHADOW_PATH = ".github/workflows/post-merge-e2e-risk-gate-shadow.yaml"; -const E2E_PATH = ".github/workflows/e2e.yaml"; - -type TriggeredWorkflow = Workflow & { - on?: Record; - permissions?: Record; - concurrency?: { group: string; "cancel-in-progress": boolean }; -}; - -function step(job: WorkflowJob, name: string): WorkflowStep { - const match = job.steps?.find((candidate) => candidate.name === name); - expect(match, `missing workflow step ${name}`).toBeDefined(); - return match!; -} - -function collectStrings(value: unknown): string[] { - return typeof value === "string" - ? [value] - : Array.isArray(value) - ? value.flatMap(collectStrings) - : value && typeof value === "object" - ? Object.values(value).flatMap(collectStrings) - : []; -} - -function runWaitStep( - scenario: "success" | "failure" | "query-failure" | "timeout" | "unsupported", - options: { runId?: string } = {}, -) { - const workflow = readYaml(SHADOW_PATH); - const wait = step(workflow.jobs.shadow, "Wait for correlated E2E run"); - const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-shadow-wait-")); - const binDir = path.join(tempDir, "bin"); - const callCountPath = path.join(tempDir, "gh-call-count"); - fs.mkdirSync(binDir); - fs.writeFileSync(callCountPath, "0\n"); - fs.writeFileSync( - path.join(binDir, "gh"), - `#!/usr/bin/env bash -set -euo pipefail -count="$(cat "$FAKE_GH_CALL_COUNT")" -count=$((count + 1)) -printf '%s\n' "$count" > "$FAKE_GH_CALL_COUNT" -case "$FAKE_GH_SCENARIO:$count" in - success:1 | success:2 | failure:1) printf 'in_progress:none\n' ;; - success:*) printf 'completed:success\n' ;; - failure:*) printf 'completed:failure\n' ;; - query-failure:*) printf 'simulated GitHub query failure\n' >&2; exit 1 ;; - unsupported:*) printf 'completed:unknown\n' ;; - *) exit 2 ;; -esac -`, - { mode: 0o755 }, - ); - fs.writeFileSync(path.join(binDir, "sleep"), "#!/usr/bin/env bash\nexit 0\n", { mode: 0o755 }); - fs.writeFileSync( - path.join(binDir, "timeout"), - `#!/usr/bin/env bash -set -euo pipefail -if [ "$FAKE_GH_SCENARIO" = "timeout" ]; then - exit 124 -fi -shift 3 -exec "$@" -`, - { mode: 0o755 }, - ); - - try { - const result = spawnSync("bash", ["-e", "-o", "pipefail", "-c", wait.run!], { - encoding: "utf8", - env: { - ...process.env, - FAKE_GH_CALL_COUNT: callCountPath, - FAKE_GH_SCENARIO: scenario, - GITHUB_REPOSITORY: "NVIDIA/NemoClaw", - PATH: `${binDir}:${process.env.PATH ?? ""}`, - RUN_ID: options.runId ?? "29110351531", - }, - timeout: 5_000, - }); - return { - ...result, - ghCallCount: Number(fs.readFileSync(callCountPath, "utf8").trim()), - }; - } finally { - fs.rmSync(tempDir, { recursive: true, force: true }); - } -} - -describe("post-merge E2E risk gate shadow workflow", () => { - it("uses a trusted main-push controller with minimal write permissions", () => { - const workflow = readYaml(SHADOW_PATH); - const job = workflow.jobs.shadow; - - expect(workflow.on).toEqual({ - push: { branches: ["main"] }, - }); - expect(workflow.permissions).toEqual({ - actions: "write", - checks: "write", - contents: "read", - }); - expect(job.if).toContain("github.repository == 'NVIDIA/NemoClaw'"); - expect(job.if).toContain("github.ref == 'refs/heads/main'"); - expect(collectStrings(workflow).some((value) => value.includes("${{ secrets."))).toBe(false); - }); - - it("builds the plan from the exact trusted push and bounds child-run waiting", () => { - const workflow = readYaml(SHADOW_PATH); - const job = workflow.jobs.shadow; - const checkout = step(job, "Checkout trusted controller"); - const workspace = step(job, "Create private controller workspace"); - const start = step(job, "Build plan and dispatch exact-commit E2E"); - const startupFallback = step(job, "Close shadow check after controller startup failure"); - const wait = step(job, "Wait for correlated E2E run"); - const download = step(job, "Download correlated E2E evidence"); - const finish = step(job, "Complete exact-commit shadow check"); - const completionFallback = step(job, "Close shadow check after completion failure"); - const summary = step(job, "Summarize shadow controller"); - const cleanup = step(job, "Remove private controller workspace"); - - expect(checkout.with).toMatchObject({ - ref: "${{ github.event.after }}", - "fetch-depth": 0, - "persist-credentials": false, - }); - expect(workspace.run).toContain('mktemp -d "${RUNNER_TEMP}/nemoclaw-e2e-risk-gate.XXXXXX"'); - expect(workspace.run).toContain('chmod 700 "$work_dir"'); - expect(start.run).toContain("post-merge-risk-gate.mts --mode start"); - expect(start.run).toContain('--base "${{ github.event.before }}"'); - expect(start.run).toContain('--commit "${{ github.event.after }}"'); - expect(start.run).toContain('--work-dir "${{ steps.workspace.outputs.work_dir }}"'); - expect(start["continue-on-error"]).not.toBe(true); - expect(startupFallback.if).toContain("always()"); - expect(startupFallback.if).toContain("steps.start.outputs.check_id != ''"); - expect(startupFallback.if).toContain("steps.start.outputs.dispatched != 'true'"); - expect(startupFallback.if).toContain("steps.start.outputs.finalized != 'true'"); - expect(startupFallback.run).toContain("post-merge-risk-gate.mts --mode abandon"); - expect(wait.run).toContain("timeout --signal=TERM --kill-after=30s 105m"); - expect(wait.run).toContain('gh run view "$RUN_ID" --repo "$GITHUB_REPOSITORY"'); - expect(wait.run).toContain("--json status,conclusion"); - expect(wait.run).toContain('if [[ "$state" != "$last_state" ]]'); - expect(wait.run).toContain('case "$state" in'); - expect(wait.run).toContain("completed:success"); - expect(wait.run).toContain("completed:failure"); - expect(wait.run).toContain("sleep 10"); - expect(wait.run).toContain('if [ "$wait_status" -eq 124 ]'); - expect(wait.run).toContain('exit "$wait_status"'); - expect(wait.run).not.toContain("gh run watch"); - expect(wait.run).not.toContain("--json jobs"); - expect(wait.run).not.toContain("2>/dev/null"); - expect(wait["continue-on-error"]).toBe(true); - expect(wait.env?.RUN_ID).toBe("${{ steps.start.outputs.run_id }}"); - expect(download.run).toContain('--dir "${{ steps.workspace.outputs.work_dir }}/evidence"'); - expect(download["continue-on-error"]).toBe(true); - expect(download.env?.RUN_ID).toBe("${{ steps.start.outputs.run_id }}"); - expect(finish.id).toBe("finish"); - expect(finish.if).toContain("always()"); - expect(finish["continue-on-error"]).not.toBe(true); - expect(finish.run).toContain("post-merge-risk-gate.mts --mode finish"); - expect(finish.run).toContain('--work-dir "${{ steps.workspace.outputs.work_dir }}"'); - expect(finish.run).toContain('--state-hash "${{ steps.start.outputs.state_hash }}"'); - expect(finish.run).toContain('--check-id "${{ steps.start.outputs.check_id }}"'); - expect(finish.run).toContain('--run-id "${{ steps.start.outputs.run_id }}"'); - expect(completionFallback.if).toContain("always()"); - expect(completionFallback.if).toContain("steps.finish.outcome == 'failure'"); - expect(completionFallback.if).toContain("steps.finish.outputs.finalized != 'true'"); - expect(completionFallback.run).toContain("post-merge-risk-gate.mts --mode abandon"); - expect(job.steps?.some((candidate) => candidate.name?.startsWith("Propagate "))).toBe(false); - expect(summary.if).toContain("always()"); - expect(summary.run).toContain("GITHUB_STEP_SUMMARY"); - expect(summary.run).toContain("Controller run"); - expect(summary.run).toContain("Selected jobs"); - expect(summary.run).toContain("Correlated E2E"); - expect(summary.run).toContain("Child conclusion"); - expect(summary.run).toContain("Failure phase"); - expect(cleanup.if).toContain("always() && steps.workspace.outputs.work_dir != ''"); - expect(cleanup.run).toContain('rm -rf -- "${{ steps.workspace.outputs.work_dir }}"'); - expect(collectStrings(workflow).some((value) => value.includes("/tmp/"))).toBe(false); - }); - - it("logs each child-run state once and exits after success", () => { - const result = runWaitStep("success"); - - expect(result.status).toBe(0); - expect(result.stderr).toBe(""); - expect(result.stdout.trim().split(/\r?\n/u)).toEqual([ - expect.stringContaining("status=in_progress"), - expect.stringContaining("status=completed conclusion=success"), - ]); - expect(result.stdout).not.toContain("JOBS"); - }); - - it("surfaces a terminal child-run failure", () => { - const result = runWaitStep("failure"); - - expect(result.status).toBe(1); - expect(result.stdout.match(/status=in_progress/gu)).toHaveLength(1); - expect(result.stderr).toContain("::error title=Correlated E2E run did not succeed::"); - expect(result.stderr).toContain("completed with conclusion failure"); - }); - - it("preserves GitHub CLI errors when status queries fail", () => { - const result = runWaitStep("query-failure"); - - expect(result.status).toBe(1); - expect(result.stderr).toContain("simulated GitHub query failure"); - expect(result.stderr).toContain("::error title=Correlated E2E status query failed::"); - }); - - it("labels only the bounded wait exit as a timeout", () => { - const result = runWaitStep("timeout"); - - expect(result.status).toBe(124); - expect(result.stderr).toContain("::error title=Correlated E2E wait timed out::"); - expect(result.stderr).toContain("did not complete within 105 minutes"); - }); - - it("rejects an invalid child-run ID before querying GitHub", () => { - const result = runWaitStep("success", { runId: "invalid" }); - - expect(result.status).toBe(1); - expect(result.ghCallCount).toBe(0); - expect(result.stderr).toContain("::error title=Invalid correlated E2E run ID::"); - }); - - it("fails closed for an unsupported child-run state", () => { - const result = runWaitStep("unsupported"); - - expect(result.status).toBe(1); - expect(result.ghCallCount).toBe(1); - expect(result.stderr).toContain("::error title=Unexpected correlated E2E state::"); - }); - - it("binds every E2E checkout and test signal to the merged commit", () => { - const workflow = readYaml< - Workflow & { - env?: Record; - "run-name"?: string; - concurrency?: { group: string; "cancel-in-progress": boolean }; - } - >(E2E_PATH); - const allSteps = Object.values(workflow.jobs).flatMap((job) => job.steps ?? []); - const checkouts = allSteps.filter((candidate) => - candidate.uses?.startsWith("actions/checkout@"), - ); - const testCommands = allSteps - .map((candidate) => candidate.run ?? "") - .filter((run) => run.includes("npx vitest run --project e2e-live")); - - expect(workflow["run-name"]).toContain("inputs.risk_correlation"); - expect(workflow.concurrency?.group).not.toContain("inputs.risk_correlation"); - expect(workflow.concurrency?.group).toContain("inputs.risk_shadow && github.run_id"); - expect(workflow.env).toMatchObject({ - NEMOCLAW_E2E_EXPECTED_SHA: "${{ inputs.checkout_sha }}", - NEMOCLAW_E2E_RISK_PLAN_HASH: "${{ inputs.risk_plan_hash }}", - NEMOCLAW_E2E_RISK_CORRELATION: "${{ inputs.risk_correlation }}", - NEMOCLAW_E2E_RISK_SHARD: "default", - }); - expect(checkouts.length).toBeGreaterThan(50); - expect( - checkouts.every( - (checkout) => checkout.with?.ref === "${{ inputs.checkout_sha || github.sha }}", - ), - ).toBe(true); - expect(testCommands.length).toBeGreaterThan(50); - expect( - testCommands.every((run) => run.includes("--reporter=test/e2e/risk-signal-reporter.ts")), - ).toBe(true); - expect(workflow.jobs["cloud-onboard"].env?.NEMOCLAW_PUBLIC_INSTALL_REF).toBe( - "${{ inputs.checkout_sha || github.sha }}", - ); - }); - - it("keeps every deterministic risk job signal-bearing and artifact-backed", () => { - const workflow = readYaml(E2E_PATH); - const requiredJobs = [...new Set(RISK_RULES.flatMap((rule) => rule.requiredJobs))]; - - for (const jobId of requiredJobs) { - const job = workflow.jobs[jobId]; - expect(job, `missing risk-plan job ${jobId}`).toBeDefined(); - expect( - Array.isArray(job.needs) ? job.needs : [job.needs], - `${jobId} must wait for exact-commit validation`, - ).toContain("generate-matrix"); - expect(job.env?.E2E_TARGET_ID, `${jobId} must identify its risk signal`).toBe(jobId); - const liveRuns = (job.steps ?? []) - .map((candidate) => candidate.run ?? "") - .filter((run) => run.includes("--project e2e-live")); - expect(liveRuns.length, `${jobId} must execute a live Vitest target`).toBeGreaterThan(0); - expect( - liveRuns.every((run) => run.includes("--reporter=test/e2e/risk-signal-reporter.ts")), - `${jobId} must write risk evidence for every live Vitest invocation`, - ).toBe(true); - const upload = (job.steps ?? []).find((candidate) => - candidate.uses?.includes("/.github/actions/upload-e2e-artifacts@"), - ); - expect(upload, `${jobId} must upload its risk signal`).toBeDefined(); - expect(upload?.if).toBe("always()"); - } - - expect(workflow.jobs["security-posture"].env?.NEMOCLAW_E2E_RISK_SHARD).toBe( - "${{ matrix.agent }}", - ); - expect(workflow.jobs["channels-stop-start"].env?.NEMOCLAW_E2E_RISK_SHARD).toBe( - "${{ matrix.agent }}", - ); - }); - - it("validates shadow inputs before preparing or executing the selected workspace", () => { - const workflow = readYaml(E2E_PATH); - const steps = workflow.jobs["generate-matrix"].steps ?? []; - const validateIndex = steps.findIndex( - (candidate) => candidate.name === "Validate exact-commit dispatch", - ); - const prepareIndex = steps.findIndex((candidate) => candidate.name === "Prepare E2E workspace"); - const validate = steps[validateIndex]; - - expect(validateIndex).toBeGreaterThan(0); - expect(validateIndex).toBeLessThan(prepareIndex); - expect(validate?.if).toContain("inputs.checkout_sha != ''"); - expect(validate?.env?.WORKFLOW_SHA).toBe("${{ github.sha }}"); - expect(validate?.run).toContain('[[ "$RISK_SHADOW" == "true" ]]'); - expect(validate?.run).toContain("exact-commit inputs require risk_shadow=true"); - expect(validate?.run).toContain("checkout_sha must be a lowercase 40-character SHA"); - expect(validate?.run).toContain('[[ "$CHECKOUT_SHA" == "$WORKFLOW_SHA" ]]'); - expect(validate?.run).toContain("checkout_sha must equal the current main workflow commit"); - expect(validate?.run).toContain('"$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA"'); - expect(validate?.run).toContain('git merge-base --is-ancestor "$CHECKOUT_SHA" origin/main'); - expect(validate?.run).toContain("forbid targets/fan-out"); - expect(steps[0]?.with?.["fetch-depth"]).toBe(0); - expect(workflow.jobs["report-to-pr"].if).toContain("!inputs.risk_shadow"); - expect(workflow.jobs.scorecard.if).toContain("!inputs.risk_shadow"); - }); -}); diff --git a/test/post-merge-e2e-risk-gate.test.ts b/test/post-merge-e2e-risk-gate.test.ts deleted file mode 100644 index 900d42cc429..00000000000 --- a/test/post-merge-e2e-risk-gate.test.ts +++ /dev/null @@ -1,593 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { execFileSync, spawnSync } from "node:child_process"; -import { createHash } from "node:crypto"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { afterEach, describe, expect, it, vi } from "vitest"; -import { buildRiskPlan, RISK_RULES } from "../tools/advisors/risk-plan.mts"; -import { - assertCorrelatedWorkflowRun, - assertTrustedMainPush, - changedFilesBetween, - classifyRiskEvidence, - dispatchRiskWorkflow, - expectedRiskSignalShards, - findSignalFiles, - finishRiskGate, - parseControllerCommand, - type RiskGateState, - validateRiskGateState, - validateRiskPlan, - validateSignal, - validateWorkflowDispatchDetails, -} from "../tools/e2e-advisor/post-merge-risk-gate.mts"; -import type { E2eRiskSignal } from "../tools/e2e-advisor/risk-signal.ts"; - -const HEAD_SHA = "a".repeat(40); -const ALLOWED = new Set(["onboard-repair", "onboard-resume"]); - -afterEach(() => { - vi.restoreAllMocks(); - vi.unstubAllEnvs(); -}); - -function state(): RiskGateState { - return { - version: 1, - commitSha: HEAD_SHA, - planHash: buildRiskPlan({ headSha: HEAD_SHA, changedFiles: ["src/lib/onboard.ts"] }).planHash, - correlationId: "12345678-1234-4123-8123-123456789abc", - expectedJobs: ["onboard-repair", "onboard-resume"], - expectedShards: { - "onboard-repair": ["default"], - "onboard-resume": ["default"], - }, - requiresManualExpansion: false, - }; -} - -function sha256(value: string): string { - return createHash("sha256").update(value).digest("hex"); -} - -function signal(jobId: string, overrides: Partial = {}): E2eRiskSignal { - const gate = state(); - return { - version: 1, - jobId, - shardId: "default", - expectedSha: gate.commitSha, - testedSha: gate.commitSha, - planHash: gate.planHash, - correlationId: gate.correlationId, - passed: 1, - failed: 0, - skipped: 0, - pending: 0, - unhandledErrors: 0, - runReason: "passed", - ...overrides, - }; -} - -describe("post-merge E2E risk gate", () => { - it("accepts only the exact trusted main-push context", () => { - const trusted = { - eventName: "push", - ref: "refs/heads/main", - sha: HEAD_SHA, - commitSha: HEAD_SHA, - }; - - expect(() => assertTrustedMainPush(trusted)).not.toThrow(); - expect(() => assertTrustedMainPush({ ...trusted, eventName: "pull_request" })).toThrow( - /exact trusted main push/u, - ); - expect(() => assertTrustedMainPush({ ...trusted, ref: "refs/heads/feature" })).toThrow( - /exact trusted main push/u, - ); - expect(() => assertTrustedMainPush({ ...trusted, sha: "b".repeat(40) })).toThrow( - /exact trusted main push/u, - ); - }); - - it("requires a private controller workspace and parses the abandon check id", () => { - const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-risk-controller-")); - try { - expect( - parseControllerCommand([ - "--mode", - "start", - "--base", - "b".repeat(40), - "--commit", - HEAD_SHA, - "--work-dir", - workDir, - ]), - ).toMatchObject({ - mode: "start", - planPath: path.join(workDir, "post-merge-risk-plan.json"), - statePath: path.join(workDir, "e2e-risk-gate-state.json"), - evidencePath: path.join(workDir, "evidence"), - }); - expect(parseControllerCommand(["--mode", "abandon", "--check-id", "17"])).toEqual({ - mode: "abandon", - checkRunId: 17, - }); - expect( - parseControllerCommand([ - "--mode", - "finish", - "--work-dir", - workDir, - "--state-hash", - "b".repeat(64), - "--check-id", - "17", - "--run-id", - "23", - ]), - ).toMatchObject({ - mode: "finish", - checkRunId: 17, - childRunId: 23, - stateHash: "b".repeat(64), - }); - expect(() => - parseControllerCommand(["--mode", "abandon", "--check-id", "9007199254740992"]), - ).toThrow(/safe integer range/u); - expect(() => - parseControllerCommand([ - "--mode", - "finish", - "--work-dir", - workDir, - "--state-hash", - "unsafe", - "--check-id", - "17", - "--run-id", - "23", - ]), - ).toThrow(/state-hash must be a lowercase SHA-256 hash/u); - expect(() => parseControllerCommand(["--mode", "finish"])).toThrow(/--work-dir/u); - - fs.chmodSync(workDir, 0o755); - expect(() => parseControllerCommand(["--mode", "finish", "--work-dir", workDir])).toThrow( - /owned private absolute directory/u, - ); - } finally { - fs.chmodSync(workDir, 0o700); - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("emits a single-line escaped Actions annotation for controller errors", () => { - const missingWorkDir = path.join(os.tmpdir(), "nemoclaw-risk-missing%\nworkspace"); - const result = spawnSync( - process.execPath, - [ - "--experimental-strip-types", - "tools/e2e-advisor/post-merge-risk-gate.mts", - "--mode", - "finish", - "--work-dir", - missingWorkDir, - ], - { - cwd: process.cwd(), - encoding: "utf8", - env: { ...process.env, GITHUB_ACTIONS: "true" }, - }, - ); - - expect(result.status).toBe(1); - const annotations = result.stderr - .split(/\r?\n/gu) - .filter((line) => - line.startsWith("::error title=Post-merge E2E risk gate controller failed::"), - ); - expect(annotations).toHaveLength(1); - expect(annotations[0]).toContain("nemoclaw-risk-missing%25 workspace"); - expect(annotations[0]).not.toMatch(/[\r\t]/u); - }); - - it("derives changed files from an exact checked-out commit range", () => { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-risk-git-")); - const git = (...args: string[]) => - execFileSync("git", args, { cwd: directory, encoding: "utf8" }).trim(); - try { - git("init", "--quiet"); - git("config", "user.name", "Risk Gate Test"); - git("config", "user.email", "risk-gate@example.invalid"); - fs.writeFileSync(path.join(directory, "README.md"), "base\n"); - fs.mkdirSync(path.join(directory, "src", "lib", "credentials"), { recursive: true }); - fs.writeFileSync(path.join(directory, "src", "lib", "credentials", "token.ts"), "base\n"); - git("add", "README.md", "src/lib/credentials/token.ts"); - git("-c", "commit.gpgsign=false", "commit", "--quiet", "-m", "base"); - const base = git("rev-parse", "HEAD"); - fs.mkdirSync(path.join(directory, "docs")); - git("mv", "src/lib/credentials/token.ts", "docs/token.ts"); - fs.writeFileSync(path.join(directory, "src", "feature.ts"), "export {};\n"); - git("add", "docs/token.ts", "src/feature.ts"); - git("-c", "commit.gpgsign=false", "commit", "--quiet", "-m", "feature"); - const head = git("rev-parse", "HEAD"); - - expect(changedFilesBetween(base, head, directory)).toEqual([ - "docs/token.ts", - "src/feature.ts", - "src/lib/credentials/token.ts", - ]); - expect(() => changedFilesBetween(head, base, directory)).toThrow(/does not match/u); - } finally { - fs.rmSync(directory, { recursive: true, force: true }); - } - }); - - it("accepts only the deterministic plan for the tested commit", () => { - const plan = buildRiskPlan({ headSha: HEAD_SHA, changedFiles: ["src/lib/onboard.ts"] }); - - expect(validateRiskPlan(plan, ALLOWED)).toEqual(plan); - expect(() => validateRiskPlan({ ...plan, planHash: "b".repeat(64) }, ALLOWED)).toThrow( - /deterministic hash/u, - ); - expect(() => validateRiskPlan(plan, new Set())).toThrow(/unknown E2E job/u); - }); - - it("accepts only bounded gate state for the exact commit and evidence policy", () => { - const gate = state(); - - expect(validateRiskGateState(gate)).toEqual(gate); - expect(() => validateRiskGateState({ ...gate, commitSha: "unsafe" })).toThrow(/commit SHA/u); - expect(() => validateRiskGateState({ ...gate, expectedJobs: ["../unsafe"] })).toThrow( - /expected jobs/u, - ); - expect(() => validateRiskGateState({ ...gate, expectedShards: {} })).toThrow(/shard jobs/u); - }); - - it("uses the workflow run identity returned by the exact dispatch request", async () => { - const gate = state(); - const fetchMock = vi.spyOn(globalThis, "fetch").mockResolvedValue({ - ok: true, - text: async () => - JSON.stringify({ - workflow_run_id: 23, - run_url: "https://api.github.com/repos/NVIDIA/NemoClaw/actions/runs/23", - html_url: "https://github.com/NVIDIA/NemoClaw/actions/runs/23", - }), - } as Response); - - const runId = await dispatchRiskWorkflow({ - repository: "NVIDIA/NemoClaw", - token: "token", - jobs: ["onboard-repair"], - commitSha: gate.commitSha, - planHash: gate.planHash, - correlationId: gate.correlationId, - }); - - expect(runId).toBe(23); - expect(fetchMock).toHaveBeenCalledOnce(); - expect(String(fetchMock.mock.calls[0]?.[0])).toBe( - "https://api.github.com/repos/NVIDIA/NemoClaw/actions/workflows/e2e.yaml/dispatches", - ); - expect(fetchMock.mock.calls[0]?.[1]?.method).toBe("POST"); - expect(JSON.parse(String(fetchMock.mock.calls[0]?.[1]?.body))).toMatchObject({ - ref: "main", - return_run_details: true, - inputs: { - jobs: "onboard-repair", - checkout_sha: gate.commitSha, - risk_plan_hash: gate.planHash, - risk_correlation: gate.correlationId, - risk_shadow: "true", - }, - }); - expect(() => - validateWorkflowDispatchDetails( - { - workflow_run_id: 23, - run_url: "https://api.github.com/repos/NVIDIA/NemoClaw/actions/runs/24", - html_url: "https://github.com/NVIDIA/NemoClaw/actions/runs/23", - }, - "NVIDIA/NemoClaw", - ), - ).toThrow(/mismatched workflow dispatch URLs/u); - }); - - it("reports the exact mismatched correlated workflow identity fields", () => { - const gate = state(); - const childRunId = 23; - const child = { - id: childRunId, - name: `E2E risk ${gate.correlationId}`, - path: ".github/workflows/e2e.yaml", - workflow_id: 304268429, - event: "workflow_dispatch", - head_sha: gate.commitSha, - status: "completed", - conclusion: "success", - created_at: "2026-07-08T00:00:00.000Z", - display_title: `E2E risk ${gate.correlationId}`, - html_url: `https://github.com/NVIDIA/NemoClaw/actions/runs/${childRunId}`, - }; - const identity = { - childRunId, - correlationId: gate.correlationId, - repository: "NVIDIA/NemoClaw", - }; - const cases = [ - { override: { id: 24 }, expected: "id expected=23 actual=24" }, - { - override: { path: ".github/workflows/other.yaml" }, - expected: - 'path expected=".github/workflows/e2e.yaml" actual=".github/workflows/other.yaml"', - }, - { - override: { display_title: "E2E risk wrong" }, - expected: `display_title expected="E2E risk ${gate.correlationId}" actual="E2E risk wrong"`, - }, - { - override: { workflow_id: 0 }, - expected: 'workflow_id expected="positive safe integer" actual=0', - }, - ]; - - for (const { override, expected } of cases) { - expect(() => assertCorrelatedWorkflowRun({ ...child, ...override }, identity)).toThrow( - expected, - ); - } - }); - - it("finish reports the directly dispatched child failure when main advances", async () => { - const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-risk-finish-")); - const statePath = path.join(workDir, "e2e-risk-gate-state.json"); - const gate = state(); - const serializedState = `${JSON.stringify(gate)}\n`; - const childRunId = 23; - vi.stubEnv("GITHUB_TOKEN", "token"); - vi.stubEnv("GITHUB_REPOSITORY", "NVIDIA/NemoClaw"); - fs.writeFileSync(statePath, serializedState, { mode: 0o600 }); - const fetchMock = vi - .spyOn(globalThis, "fetch") - .mockResolvedValueOnce({ - ok: true, - text: async () => - JSON.stringify({ - id: childRunId, - name: `E2E risk ${gate.correlationId}`, - event: "workflow_dispatch", - head_sha: "b".repeat(40), - status: "completed", - conclusion: "failure", - created_at: "2026-07-08T00:00:00.000Z", - display_title: `E2E risk ${gate.correlationId}`, - path: ".github/workflows/e2e.yaml", - workflow_id: 304268429, - html_url: `https://github.com/NVIDIA/NemoClaw/actions/runs/${childRunId}`, - }), - } as Response) - .mockResolvedValueOnce({ ok: true, text: async () => "{}" } as Response); - - try { - await finishRiskGate({ - statePath, - stateHash: sha256(serializedState), - evidencePath: path.join(workDir, "evidence"), - checkRunId: 17, - childRunId, - }); - - expect(fetchMock).toHaveBeenCalledTimes(2); - expect(String(fetchMock.mock.calls[1]?.[0])).toContain("check-runs/17"); - expect(JSON.parse(String(fetchMock.mock.calls[1]?.[1]?.body))).toMatchObject({ - status: "completed", - conclusion: "failure", - output: { title: "Selected E2E workflow failed" }, - }); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("rejects changed controller state before classifying downloaded evidence", async () => { - const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-risk-state-")); - const statePath = path.join(workDir, "e2e-risk-gate-state.json"); - const outputPath = path.join(workDir, "github-output"); - const originalState = `${JSON.stringify(state())}\n`; - const changedState = `${JSON.stringify({ ...state(), requiresManualExpansion: true })}\n`; - vi.stubEnv("GITHUB_TOKEN", "token"); - vi.stubEnv("GITHUB_REPOSITORY", "NVIDIA/NemoClaw"); - vi.stubEnv("GITHUB_OUTPUT", outputPath); - fs.writeFileSync(statePath, changedState, { mode: 0o600 }); - fs.writeFileSync(outputPath, "", { mode: 0o600 }); - const fetchMock = vi - .spyOn(globalThis, "fetch") - .mockResolvedValue({ ok: true, text: async () => "{}" } as Response); - - try { - await expect( - finishRiskGate({ - statePath, - stateHash: sha256(originalState), - evidencePath: path.join(workDir, "evidence"), - checkRunId: 17, - childRunId: 23, - }), - ).rejects.toThrow(/controller state changed after E2E dispatch/u); - - expect(fetchMock).toHaveBeenCalledTimes(1); - expect(String(fetchMock.mock.calls[0]?.[0])).toContain("check-runs/17"); - expect(JSON.parse(String(fetchMock.mock.calls[0]?.[1]?.body))).toMatchObject({ - status: "completed", - conclusion: "neutral", - output: { title: "Risk-selected E2E evidence could not be verified" }, - }); - expect(JSON.parse(String(fetchMock.mock.calls[0]?.[1]?.body)).output.summary).toContain( - "controller state changed after E2E dispatch", - ); - expect(fs.readFileSync(outputPath, "utf8")).toContain("finalized=true\n"); - } finally { - fs.rmSync(workDir, { recursive: true, force: true }); - } - }); - - it("bounds downloaded risk-evidence traversal by entries, depth, and signals", () => { - const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-risk-evidence-")); - const rootLink = `${directory}-link`; - const nested = path.join(directory, "artifact", "live", "job"); - try { - fs.mkdirSync(nested, { recursive: true }); - const signalFile = path.join(nested, "risk-signal.json"); - fs.writeFileSync(signalFile, "{}\n"); - - expect(findSignalFiles(directory)).toEqual([signalFile]); - expect(() => - findSignalFiles(directory, { maxDepth: 2, maxEntries: 10, maxSignalFiles: 2 }), - ).toThrow(/depth limit/u); - expect(() => - findSignalFiles(directory, { maxDepth: 8, maxEntries: 2, maxSignalFiles: 2 }), - ).toThrow(/entry limit/u); - const second = path.join(directory, "artifact-2"); - fs.mkdirSync(second); - fs.writeFileSync(path.join(second, "risk-signal.json"), "{}\n"); - expect(() => - findSignalFiles(directory, { maxDepth: 8, maxEntries: 10, maxSignalFiles: 1 }), - ).toThrow(/signal-file limit/u); - expect(() => - findSignalFiles(directory, { maxDepth: 8, maxEntries: 10, maxSignalFiles: 0 }), - ).toThrow(/limits are invalid/u); - fs.symlinkSync(directory, rootLink, "dir"); - expect(() => findSignalFiles(rootLink)).toThrow(/root must be a directory, not a symlink/u); - } finally { - fs.rmSync(rootLink, { force: true }); - fs.rmSync(directory, { recursive: true, force: true }); - } - }); - - it("accepts only signals bound to the expected job, SHA, plan, and correlation", () => { - const gate = state(); - - expect(validateSignal(signal("onboard-resume"), gate).jobId).toBe("onboard-resume"); - expect(() => - validateSignal(signal("onboard-resume", { expectedSha: "b".repeat(40) }), gate), - ).toThrow(/SHA mismatch/u); - expect(() => validateSignal(signal("other"), gate)).toThrow(/unexpected/u); - expect(() => validateSignal(signal("onboard-resume", { shardId: "unexpected" }), gate)).toThrow( - /shard/u, - ); - }); - - it("derives expected evidence shards from the trusted E2E workflow", () => { - const jobIds = [...new Set(RISK_RULES.flatMap((rule) => rule.requiredJobs))]; - const shards = expectedRiskSignalShards(jobIds); - - expect(Object.keys(shards).sort()).toEqual(jobIds.sort()); - expect(shards["onboard-resume"]).toEqual(["default"]); - expect(shards["security-posture"]).toEqual(["openclaw", "hermes"]); - expect(shards["channels-stop-start"]).toEqual(["openclaw", "hermes"]); - }); - - it("reports success only for complete unskipped evidence", () => { - const verdict = classifyRiskEvidence({ - workflowConclusion: "success", - expectedJobs: ["onboard-repair", "onboard-resume"], - expectedShards: state().expectedShards, - signals: [signal("onboard-repair"), signal("onboard-resume")], - requiresManualExpansion: false, - }); - - expect(verdict.conclusion).toBe("success"); - expect(verdict.summary).not.toContain("onboard-repair"); - }); - - it.each([ - { - label: "missing signal", - signals: [signal("onboard-repair")], - manual: false, - }, - { - label: "skipped test", - signals: [signal("onboard-repair"), signal("onboard-resume", { skipped: 1 })], - manual: false, - }, - { - label: "manual expansion", - signals: [signal("onboard-repair"), signal("onboard-resume")], - manual: true, - }, - { - label: "duplicate signal", - signals: [signal("onboard-repair"), signal("onboard-repair"), signal("onboard-resume")], - manual: false, - }, - ])("reports neutral for $label", ({ signals, manual }) => { - const verdict = classifyRiskEvidence({ - workflowConclusion: "success", - expectedJobs: ["onboard-repair", "onboard-resume"], - expectedShards: state().expectedShards, - signals, - requiresManualExpansion: manual, - }); - - expect(verdict.conclusion).toBe("neutral"); - expect(verdict.summary).not.toContain("onboard-repair"); - }); - - it("reports a product workflow failure as failure", () => { - const verdict = classifyRiskEvidence({ - workflowConclusion: "failure", - expectedJobs: ["onboard-repair"], - expectedShards: { "onboard-repair": ["default"] }, - signals: [], - requiresManualExpansion: false, - }); - - expect(verdict.conclusion).toBe("failure"); - expect(verdict.summary).not.toContain("onboard-repair"); - }); - - it("reports failed test evidence as failure even when the workflow is green", () => { - const verdict = classifyRiskEvidence({ - workflowConclusion: "success", - expectedJobs: ["onboard-repair"], - expectedShards: { "onboard-repair": ["default"] }, - signals: [signal("onboard-repair", { failed: 1, runReason: "failed" })], - requiresManualExpansion: false, - }); - - expect(verdict.conclusion).toBe("failure"); - expect(verdict.summary).not.toContain("onboard-repair"); - }); - - it("requires every expected matrix shard to pass", () => { - const complete = classifyRiskEvidence({ - workflowConclusion: "success", - expectedJobs: ["security-posture"], - expectedShards: { "security-posture": ["openclaw", "hermes"] }, - signals: [ - signal("security-posture", { shardId: "openclaw" }), - signal("security-posture", { shardId: "hermes" }), - ], - requiresManualExpansion: false, - }); - const missingShard = classifyRiskEvidence({ - workflowConclusion: "success", - expectedJobs: ["security-posture"], - expectedShards: { "security-posture": ["openclaw", "hermes"] }, - signals: [signal("security-posture", { shardId: "openclaw" })], - requiresManualExpansion: false, - }); - - expect(complete.conclusion).toBe("success"); - expect(missingShard.conclusion).toBe("neutral"); - expect(missingShard.summary).not.toContain("security-posture"); - }); -}); diff --git a/test/pr-e2e-gate-workflow.test.ts b/test/pr-e2e-gate-workflow.test.ts new file mode 100644 index 00000000000..3460569e798 --- /dev/null +++ b/test/pr-e2e-gate-workflow.test.ts @@ -0,0 +1,466 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; +import { + readYaml, + type Workflow, + type WorkflowJob, + type WorkflowStep, +} from "./helpers/e2e-workflow-contract.ts"; + +const PR_GATE_PATH = ".github/workflows/pr-e2e-gate.yaml"; +const E2E_PATH = ".github/workflows/e2e.yaml"; + +type CoordinatorJob = WorkflowJob & { + concurrency?: { group: string; "cancel-in-progress": boolean }; +}; + +type TriggeredWorkflow = Omit & { + name: string; + on: { + workflow_run: { workflows: string[]; types: string[] }; + pull_request_target: { types: string[] }; + }; + permissions: Record; + jobs: Record; +}; + +type DispatchWorkflow = Workflow & { + "run-name": string; + on: { + workflow_dispatch: { + inputs: Record; + }; + }; +}; + +function step(job: WorkflowJob, name: string): WorkflowStep { + const match = job.steps?.find((candidate) => candidate.name === name); + expect(match, `missing workflow step ${name}`).toBeDefined(); + return match!; +} + +function collectStrings(value: unknown): string[] { + return typeof value === "string" + ? [value] + : Array.isArray(value) + ? value.flatMap(collectStrings) + : value && typeof value === "object" + ? Object.values(value).flatMap(collectStrings) + : []; +} + +function runWaitStep( + scenario: "success" | "failure" | "query-failure" | "timeout" | "unsupported", + options: { runId?: string } = {}, +) { + const workflow = readYaml(PR_GATE_PATH); + const wait = step(workflow.jobs.coordinate, "Wait for E2E run"); + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pr-e2e-gate-wait-")); + const binDir = path.join(tempDir, "bin"); + const callCountPath = path.join(tempDir, "gh-call-count"); + fs.mkdirSync(binDir); + fs.writeFileSync(callCountPath, "0\n"); + fs.writeFileSync( + path.join(binDir, "gh"), + `#!/usr/bin/env bash +set -euo pipefail +count="$(cat "$FAKE_GH_CALL_COUNT")" +count=$((count + 1)) +printf '%s\n' "$count" > "$FAKE_GH_CALL_COUNT" +case "$FAKE_GH_SCENARIO:$count" in + success:1 | success:2 | failure:1) printf 'in_progress:none\n' ;; + success:*) printf 'completed:success\n' ;; + failure:*) printf 'completed:failure\n' ;; + query-failure:*) printf 'simulated GitHub query failure\n' >&2; exit 1 ;; + unsupported:*) printf 'completed:unknown\n' ;; + *) exit 2 ;; +esac +`, + { mode: 0o755 }, + ); + fs.writeFileSync(path.join(binDir, "sleep"), "#!/usr/bin/env bash\nexit 0\n", { mode: 0o755 }); + fs.writeFileSync( + path.join(binDir, "timeout"), + `#!/usr/bin/env bash +set -euo pipefail +if [ "$FAKE_GH_SCENARIO" = "timeout" ]; then + exit 124 +fi +shift 3 +exec "$@" +`, + { mode: 0o755 }, + ); + + try { + const result = spawnSync("bash", ["-e", "-o", "pipefail", "-c", wait.run!], { + encoding: "utf8", + env: { + ...process.env, + FAKE_GH_CALL_COUNT: callCountPath, + FAKE_GH_SCENARIO: scenario, + GITHUB_REPOSITORY: "NVIDIA/NemoClaw", + PATH: `${binDir}:${process.env.PATH ?? ""}`, + RUN_ID: options.runId ?? "29110351531", + }, + timeout: 5_000, + }); + return { + ...result, + ghCallCount: Number(fs.readFileSync(callCountPath, "utf8").trim()), + }; + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +} + +function runStartStep(headBranch: string) { + const workflow = readYaml(PR_GATE_PATH); + const start = step(workflow.jobs.coordinate, "Start evaluation"); + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pr-e2e-gate-start-step-")); + const binDir = path.join(tempDir, "bin"); + const argumentsPath = path.join(tempDir, "node-arguments"); + fs.mkdirSync(binDir); + fs.writeFileSync( + path.join(binDir, "node"), + '#!/usr/bin/env bash\nset -euo pipefail\nprintf \'%s\\0\' "$@" > "$FAKE_NODE_ARGUMENTS"\n', + { mode: 0o755 }, + ); + + try { + const result = spawnSync("bash", ["-e", "-o", "pipefail", "-c", start.run!], { + encoding: "utf8", + env: { + ...process.env, + CI_CONCLUSION: "success", + FAKE_NODE_ARGUMENTS: argumentsPath, + GITHUB_TOKEN: "token", + HEAD_BRANCH: headBranch, + HEAD_REPOSITORY: "NVIDIA/NemoClaw", + HEAD_SHA: "a".repeat(40), + PATH: `${binDir}:${process.env.PATH ?? ""}`, + WORKFLOW_SHA: "d".repeat(40), + WORK_DIR: tempDir, + }, + timeout: 5_000, + }); + return { + arguments: fs.readFileSync(argumentsPath, "utf8").split("\0").slice(0, -1), + result, + }; + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +} + +function runChildValidation(currentPullSha: string) { + const workflow = readYaml(E2E_PATH); + const validation = step(workflow.jobs["generate-matrix"], "Validate controller dispatch"); + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pr-e2e-gate-child-")); + const binDir = path.join(tempDir, "bin"); + fs.mkdirSync(binDir); + fs.writeFileSync( + path.join(binDir, "git"), + "#!/usr/bin/env bash\nset -euo pipefail\nprintf '%s\\n' \"$FAKE_CHECKOUT_SHA\"\n", + { mode: 0o755 }, + ); + fs.writeFileSync( + path.join(binDir, "curl"), + "#!/usr/bin/env bash\nset -euo pipefail\nprintf '{}\\n'\n", + { mode: 0o755 }, + ); + fs.writeFileSync( + path.join(binDir, "jq"), + `#!/usr/bin/env bash +set -euo pipefail +case "\${2:-}" in + .state) printf 'open\\n' ;; + .head.repo.full_name*) printf 'NVIDIA/NemoClaw\\n' ;; + .head.sha) printf '%s\\n' "$FAKE_PR_SHA" ;; + *) exit 2 ;; +esac +`, + { mode: 0o755 }, + ); + + try { + return spawnSync("bash", ["-e", "-o", "pipefail", "-c", validation.run!], { + encoding: "utf8", + env: { + ...process.env, + CHECKOUT_SHA: "a".repeat(40), + CORRELATION_ID: "12345678-1234-4123-8123-123456789abc", + FAKE_CHECKOUT_SHA: "a".repeat(40), + FAKE_PR_SHA: currentPullSha, + GITHUB_REPOSITORY: "NVIDIA/NemoClaw", + GITHUB_TOKEN: "token", + JOBS: "onboard-repair", + PATH: `${binDir}:${process.env.PATH ?? ""}`, + PLAN_HASH: "b".repeat(64), + PR_NUMBER: "42", + TARGETS: "", + WORKFLOW_EVENT: "workflow_dispatch", + WORKFLOW_REF: "refs/heads/main", + }, + }); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +} + +describe("PR E2E gate workflow", () => { + it("limits triggers and job permissions", () => { + const workflow = readYaml(PR_GATE_PATH); + const cancel = workflow.jobs["cancel-superseded"]; + const coordinate = workflow.jobs.coordinate; + + expect(workflow.name).toBe("E2E / PR Gate"); + expect(workflow.on).toEqual({ + workflow_run: { + workflows: ["CI / Pull Request"], + types: ["completed"], + }, + pull_request_target: { + types: ["synchronize", "reopened", "closed"], + }, + }); + expect(workflow.permissions).toEqual({}); + expect(cancel.if).toContain("github.event_name == 'pull_request_target'"); + expect(cancel.if).toContain( + "github.event.pull_request.head.repo.full_name == github.repository", + ); + expect(cancel.permissions).toEqual({ actions: "write", contents: "read" }); + expect(coordinate.if).toContain("github.event_name == 'workflow_run'"); + expect(coordinate.if).toContain("github.event.workflow_run.event == 'pull_request'"); + expect(coordinate.if).toContain( + "github.event.workflow_run.head_repository.full_name == github.repository", + ); + expect(coordinate.permissions).toEqual({ + actions: "write", + checks: "write", + contents: "read", + "pull-requests": "read", + }); + expect(collectStrings(workflow).some((value) => value.includes("${{ secrets."))).toBe(false); + }); + + it("pins both controller checkouts and installs without lifecycle scripts or caches", () => { + const workflow = readYaml(PR_GATE_PATH); + const allSteps = Object.values(workflow.jobs).flatMap((job) => job.steps ?? []); + const checkouts = allSteps.filter((candidate) => + candidate.uses?.startsWith("actions/checkout@"), + ); + const nodeSetups = allSteps.filter((candidate) => + candidate.uses?.startsWith("actions/setup-node@"), + ); + const installs = allSteps.filter( + (candidate) => candidate.name === "Install controller dependencies", + ); + + expect(checkouts).toHaveLength(2); + expect( + checkouts.every( + (checkout) => + checkout.with?.ref === "${{ github.workflow_sha }}" && + checkout.with?.["persist-credentials"] === false, + ), + ).toBe(true); + expect(nodeSetups).toHaveLength(2); + expect(nodeSetups.every((setup) => setup.with?.["node-version"] === "22")).toBe(true); + expect(nodeSetups.every((setup) => !("cache" in (setup.with ?? {})))).toBe(true); + expect(installs).toHaveLength(2); + expect(installs.every((install) => install.run === "npm ci --ignore-scripts")).toBe(true); + expect( + allSteps.some((candidate) => candidate.uses?.startsWith("actions/download-artifact@")), + ).toBe(false); + }); + + it("cancels superseded PR runs", () => { + const workflow = readYaml(PR_GATE_PATH); + const cancel = workflow.jobs["cancel-superseded"]; + const cancelStep = step(cancel, "Cancel superseded E2E runs"); + + expect(cancelStep.run).toContain("tools/e2e/pr-e2e-gate.mts --mode cancel"); + expect(cancelStep.run).toContain('--pr "$PR_NUMBER"'); + expect(cancelStep.run).not.toContain("${{ github.event."); + expect(cancelStep.env?.GITHUB_TOKEN).toBe("${{ github.token }}"); + expect(cancelStep.env?.PR_NUMBER).toBe("${{ github.event.pull_request.number }}"); + }); + + it.each([ + ["a single quote", "feature/'quoted"], + ["a double quote", 'feature/"quoted'], + ["command substitution", "feature/$(printf injected)"], + ["a semicolon", "feature/branch;printf injected"], + ["whitespace", "feature/space name"], + ["a newline", "feature/line\nname"], + ])("passes branch text containing $label as one inert shell argument", (_label, headBranch) => { + const execution = runStartStep(headBranch); + const branchFlag = execution.arguments.indexOf("--head-branch"); + + expect(execution.result.status).toBe(0); + expect(execution.result.stderr).toBe(""); + expect(execution.arguments.filter((argument) => argument === "--head-branch")).toHaveLength(1); + expect(execution.arguments[branchFlag + 1]).toBe(headBranch); + }); + + it("coordinates the check around one E2E run", () => { + const workflow = readYaml(PR_GATE_PATH); + const job = workflow.jobs.coordinate; + const workspace = step(job, "Create private workspace"); + const start = step(job, "Start evaluation"); + const upload = step(job, "Upload risk plan"); + const wait = step(job, "Wait for E2E run"); + const download = step(job, "Download evidence"); + const finish = step(job, "Verify evidence"); + const fallback = step(job, "Close incomplete check"); + const cleanup = step(job, "Remove private workspace"); + + expect(job.concurrency).toEqual({ + group: + "pr-e2e-gate-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }}", + "cancel-in-progress": false, + }); + expect(job["timeout-minutes"]).toBe(180); + expect(workspace.run).toContain('mktemp -d "${RUNNER_TEMP}/nemoclaw-pr-e2e-gate.XXXXXX"'); + expect(workspace.run).toContain('chmod 700 "$work_dir"'); + expect(start.run).toContain("tools/e2e/pr-e2e-gate.mts --mode start"); + expect(start.run).toContain('--head "$HEAD_SHA"'); + expect(start.run).toContain('--head-repo "$HEAD_REPOSITORY"'); + expect(start.run).toContain('--head-branch "$HEAD_BRANCH"'); + expect(start.run).toContain('--workflow-sha "$WORKFLOW_SHA"'); + expect(start.run).toContain('--ci-conclusion "$CI_CONCLUSION"'); + expect(start.run).toContain('--work-dir "$WORK_DIR"'); + expect(start.run).not.toContain("${{ github.event."); + expect(start.env).toMatchObject({ + CI_CONCLUSION: "${{ github.event.workflow_run.conclusion }}", + HEAD_BRANCH: "${{ github.event.workflow_run.head_branch }}", + HEAD_REPOSITORY: "${{ github.event.workflow_run.head_repository.full_name }}", + HEAD_SHA: "${{ github.event.workflow_run.head_sha }}", + WORKFLOW_SHA: "${{ github.workflow_sha }}", + WORK_DIR: "${{ steps.workspace.outputs.work_dir }}", + }); + expect(start.run).not.toContain("--mode initialize"); + expect(upload.if).toContain("steps.workspace.outputs.work_dir != ''"); + expect(upload.with?.name).toBe("pr-e2e-risk-plan-${{ github.event.workflow_run.head_sha }}"); + expect(upload.with?.path).toBe("${{ steps.workspace.outputs.work_dir }}/risk-plan.json"); + expect(wait.run).toContain("timeout --signal=TERM --kill-after=30s 105m"); + expect(wait.run).toContain('gh run view "$RUN_ID" --repo "$GITHUB_REPOSITORY"'); + expect(wait.run).toContain("--json status,conclusion"); + expect(wait.run).toContain('if [[ "$state" != "$last_state" ]]'); + expect(wait.run).toContain("completed:success"); + expect(wait.run).toContain("completed:failure"); + expect(wait.run).toContain("sleep 10"); + expect(wait.run).toContain('if [ "$wait_status" -eq 124 ]'); + expect(wait.run).toContain('exit "$wait_status"'); + expect(wait.run).not.toContain("gh run watch"); + expect(wait.run).not.toContain("--json jobs"); + expect(wait.run).not.toContain("2>/dev/null"); + expect(wait["continue-on-error"]).toBe(true); + expect(download.if).toContain("always()"); + expect(download.run).toContain("timeout --signal=TERM --kill-after=30s 10m"); + expect(download.run).toContain('if [ "$download_status" -eq 124 ]'); + expect(download.run).toContain('--dir "${{ steps.workspace.outputs.work_dir }}/evidence"'); + expect(download["continue-on-error"]).toBe(true); + expect(finish.if).toContain("always()"); + expect(finish.run).toContain("tools/e2e/pr-e2e-gate.mts --mode finish"); + expect(finish.run).toContain('--state-hash "${{ steps.start.outputs.state_hash }}"'); + expect(finish.run).toContain('--check-id "${{ steps.start.outputs.check_id }}"'); + expect(finish.run).toContain('--run-id "${{ steps.start.outputs.run_id }}"'); + expect(fallback.if).toContain("always()"); + expect(fallback.if).toContain("steps.start.outputs.check_id != ''"); + expect(fallback.if).toContain("steps.start.outputs.finalized != 'true'"); + expect(fallback.if).toContain("steps.finish.outputs.finalized != 'true'"); + expect(fallback.run).toContain("tools/e2e/pr-e2e-gate.mts --mode abandon"); + expect(fallback.run).toContain('--run-id "${{ steps.start.outputs.run_id }}"'); + expect(cleanup.if).toContain("always() && steps.workspace.outputs.work_dir != ''"); + expect(cleanup.run).toContain('rm -rf -- "${{ steps.workspace.outputs.work_dir }}"'); + expect(collectStrings(workflow).some((value) => value.includes("/tmp/"))).toBe(false); + }); + + it("uses one child dispatch protocol and one correlated run title", () => { + const workflow = readYaml(E2E_PATH); + const inputs = workflow.on.workflow_dispatch.inputs; + + expect(inputs).toEqual( + expect.objectContaining({ + jobs: expect.any(Object), + pr_number: expect.any(Object), + checkout_sha: expect.any(Object), + plan_hash: expect.any(Object), + correlation_id: expect.any(Object), + }), + ); + expect(workflow["run-name"]).toContain( + "format('E2E PR #{0} ({1})', inputs.pr_number, inputs.correlation_id)", + ); + }); + + it("validates the E2E run against the PR head commit", () => { + const current = runChildValidation("a".repeat(40)); + const stale = runChildValidation("c".repeat(40)); + + expect(current.status).toBe(0); + expect(stale.status).toBe(1); + expect(stale.stdout).toContain("checkout_sha must match the PR head commit"); + }); + + it("logs each child state once and exits after success", () => { + const result = runWaitStep("success"); + + expect(result.status).toBe(0); + expect(result.stderr).toBe(""); + expect(result.stdout.trim().split(/\r?\n/u)).toEqual([ + expect.stringContaining("status=in_progress"), + expect.stringContaining("status=completed conclusion=success"), + ]); + }); + + it("surfaces a terminal child failure", () => { + const result = runWaitStep("failure"); + + expect(result.status).toBe(1); + expect(result.stdout.match(/status=in_progress/gu)).toHaveLength(1); + expect(result.stderr).toContain("::error title=E2E run failed::"); + expect(result.stderr).toContain("completed with conclusion failure"); + }); + + it("preserves GitHub CLI errors when status queries fail", () => { + const result = runWaitStep("query-failure"); + + expect(result.status).toBe(1); + expect(result.stderr).toContain("simulated GitHub query failure"); + expect(result.stderr).toContain("::error title=Run status query failed::"); + }); + + it("labels only the bounded wait exit as a timeout", () => { + const result = runWaitStep("timeout"); + + expect(result.status).toBe(124); + expect(result.stderr).toContain("::error title=E2E run timed out::"); + expect(result.stderr).toContain("did not complete within 105 minutes"); + }); + + it("rejects an invalid child run ID before querying GitHub", () => { + const result = runWaitStep("success", { runId: "invalid" }); + + expect(result.status).toBe(1); + expect(result.ghCallCount).toBe(0); + expect(result.stderr).toContain("::error title=Invalid run ID::"); + }); + + it("fails closed for an unsupported child state", () => { + const result = runWaitStep("unsupported"); + + expect(result.status).toBe(1); + expect(result.ghCallCount).toBe(1); + expect(result.stderr).toContain("::error title=Unexpected run state::"); + }); +}); diff --git a/test/pr-e2e-gate.test.ts b/test/pr-e2e-gate.test.ts new file mode 100644 index 00000000000..f8758f21a25 --- /dev/null +++ b/test/pr-e2e-gate.test.ts @@ -0,0 +1,917 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; +import { buildRiskPlan, riskPlanRequiredJobIds } from "../tools/advisors/risk-plan.mts"; +import { + abandonPrGate, + assertCorrelatedWorkflowRun, + cancelPrGate, + classifyPrGateEvidence, + dispatchPrGate, + expectedSignalShards, + findSignalFiles, + finishPrGate, + type PrGateState, + type PullRequest, + parseControllerCommand, + pullChangedFiles, + startPrGate, + validatePrGateState, + validateRiskPlan, + validateSignal, + validateWorkflowDispatchDetails, +} from "../tools/e2e/pr-e2e-gate.mts"; +import type { E2eRiskSignal } from "../tools/e2e/risk-signal.ts"; +import { + createGitHubFetchRouter, + githubFetchRoute, + type RecordedGitHubRequest, +} from "./support/github-fetch-router.ts"; + +const HEAD_SHA = "a".repeat(40); +const BASE_SHA = "b".repeat(40); +const WORKFLOW_SHA = "d".repeat(40); +const CORRELATION_ID = "12345678-1234-4123-8123-123456789abc"; +const BROAD_FILES = [ + "src/lib/onboard.ts", + "src/lib/actions/upgrade-sandboxes.ts", + "src/lib/actions/sandbox/agents/apply.ts", + "src/lib/messaging/applier/agent-config.ts", + "src/lib/inference/health.ts", + "install.sh", + "src/lib/credentials/provider-list.ts", +] as const; +const BROAD_JOBS = [ + "cloud-onboard", + "credential-sanitization", + "security-posture", + "channels-add-remove", + "channels-stop-start", + "full-e2e", + "hermes-e2e", + "inference-routing", + "network-policy", + "onboard-repair", + "onboard-resume", + "state-backup-restore", + "upgrade-stale-sandbox", +] as const; + +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); +}); + +function githubResponse(value?: unknown, status = 200): Response { + return { + ok: status >= 200 && status < 300, + status, + json: async () => value, + text: async () => (value === undefined ? "" : JSON.stringify(value)), + } as Response; +} + +function sha256(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} + +function pullRequest(changedFiles = 1): PullRequest { + return { + number: 42, + state: "open", + changed_files: changedFiles, + head: { + ref: "feature/pr-e2e-gate", + sha: HEAD_SHA, + repo: { full_name: "NVIDIA/NemoClaw" }, + }, + base: { + sha: BASE_SHA, + repo: { full_name: "NVIDIA/NemoClaw" }, + }, + }; +} + +function pullRequestListItem(pull = pullRequest()): Omit { + const { changed_files: _changedFiles, ...item } = pull; + return item; +} + +function state(): PrGateState { + const plan = buildRiskPlan({ headSha: HEAD_SHA, changedFiles: ["src/lib/onboard.ts"] }); + return { + version: 1, + commitSha: HEAD_SHA, + workflowSha: WORKFLOW_SHA, + planHash: plan.planHash, + correlationId: CORRELATION_ID, + prNumber: 42, + expectedJobs: ["onboard-repair", "onboard-resume"], + expectedShards: { + "onboard-repair": ["default"], + "onboard-resume": ["default"], + }, + }; +} + +function startCommand(workDir: string) { + const command = parseControllerCommand([ + "--mode", + "start", + "--head", + HEAD_SHA, + "--head-repo", + "NVIDIA/NemoClaw", + "--head-branch", + "feature/pr-e2e-gate", + "--workflow-sha", + WORKFLOW_SHA, + "--ci-conclusion", + "success", + "--work-dir", + workDir, + ]); + expect(command.mode).toBe("start"); + return command as Extract, { mode: "start" }>; +} + +function signal( + gate: PrGateState, + jobId: string, + shardId = "default", + overrides: Partial = {}, +): E2eRiskSignal { + return { + version: 1, + jobId, + shardId, + expectedSha: gate.commitSha, + testedSha: gate.commitSha, + planHash: gate.planHash, + correlationId: gate.correlationId, + passed: 1, + failed: 0, + skipped: 0, + pending: 0, + unhandledErrors: 0, + runReason: "passed", + ...overrides, + }; +} + +function workflowRun(gate: PrGateState, overrides: Record = {}) { + return { + id: 23, + name: "E2E", + path: ".github/workflows/e2e.yaml", + workflow_id: 304268429, + event: "workflow_dispatch", + head_sha: gate.workflowSha, + status: "completed", + conclusion: "success", + display_title: `E2E PR #${gate.prNumber} (${gate.correlationId})`, + html_url: "https://github.com/NVIDIA/NemoClaw/actions/runs/23", + ...overrides, + }; +} + +describe("PR E2E controller", () => { + it("parses one lifecycle command set inside a private workspace", () => { + const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pr-e2e-gate-")); + try { + expect( + parseControllerCommand([ + "--mode", + "start", + "--head", + HEAD_SHA, + "--head-repo", + "NVIDIA/NemoClaw", + "--head-branch", + "feature/pr-e2e-gate", + "--workflow-sha", + WORKFLOW_SHA, + "--ci-conclusion", + "success", + "--work-dir", + workDir, + ]), + ).toMatchObject({ + mode: "start", + planPath: path.join(workDir, "risk-plan.json"), + statePath: path.join(workDir, "controller-state.json"), + evidencePath: path.join(workDir, "evidence"), + }); + expect(parseControllerCommand(["--mode", "cancel", "--pr", "42"])).toEqual({ + mode: "cancel", + prNumber: 42, + }); + expect( + parseControllerCommand(["--mode", "abandon", "--check-id", "17", "--run-id", "23"]), + ).toEqual({ mode: "abandon", checkRunId: 17, childRunId: 23 }); + expect(() => + parseControllerCommand(["--mode", "cancel", "--pr", "9007199254740992"]), + ).toThrow(/safe integer range/u); + + fs.chmodSync(workDir, 0o755); + expect(() => parseControllerCommand(["--mode", "finish", "--work-dir", workDir])).toThrow( + /owned private absolute directory/u, + ); + } finally { + fs.chmodSync(workDir, 0o700); + fs.rmSync(workDir, { recursive: true, force: true }); + } + }); + + it("validates the risk plan and bounded state", () => { + const plan = buildRiskPlan({ headSha: HEAD_SHA, changedFiles: ["src/lib/onboard.ts"] }); + const allowed = new Set(riskPlanRequiredJobIds(plan)); + const gate = state(); + + expect(validateRiskPlan(plan, allowed)).toEqual(plan); + expect(() => validateRiskPlan({ ...plan, version: 1 }, allowed)).toThrow( + /unsupported risk-plan version/u, + ); + expect(() => validateRiskPlan({ ...plan, planHash: "b".repeat(64) }, allowed)).toThrow( + /hash and inputs/u, + ); + expect(() => validateRiskPlan(plan, new Set())).toThrow(/unknown E2E job/u); + expect(validatePrGateState(gate)).toEqual(gate); + expect(() => validatePrGateState({ ...gate, prNumber: 0 })).toThrow(/PR number/u); + expect(() => validatePrGateState({ ...gate, expectedShards: {} })).toThrow(/shard jobs/u); + }); + + it("paginates canonical pull request files and includes both names for renames", async () => { + const pageOne = Array.from({ length: 100 }, (_, index) => ({ + filename: `src/file-${index}.ts`, + ...(index === 0 ? { previous_filename: "src/old-name.ts" } : {}), + })); + const pageTwo = [{ filename: "src/file-100.ts" }]; + const fetchMock = vi.spyOn(globalThis, "fetch").mockImplementation( + createGitHubFetchRouter([ + githubFetchRoute( + ({ url }) => url.endsWith("page=1"), + () => githubResponse(pageOne), + ), + githubFetchRoute( + ({ url }) => url.endsWith("page=2"), + () => githubResponse(pageTwo), + ), + ]), + ); + + const files = await pullChangedFiles("NVIDIA/NemoClaw", pullRequest(101), "token"); + + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(files).toHaveLength(102); + expect(files.slice(0, 3)).toEqual(["src/old-name.ts", "src/file-0.ts", "src/file-1.ts"]); + await expect(pullChangedFiles("NVIDIA/NemoClaw", pullRequest(3001), "token")).rejects.toThrow( + /between 0 and 3000/u, + ); + }); + + it("fails closed for missing, duplicate, skipped, or failing evidence", () => { + const gate = state(); + const complete = gate.expectedJobs.map((job) => signal(gate, job)); + const classify = (signals: E2eRiskSignal[], workflowConclusion: string | null = "success") => + classifyPrGateEvidence({ + workflowConclusion, + expectedJobs: gate.expectedJobs, + expectedShards: gate.expectedShards, + signals, + }); + + expect(classify(complete).conclusion).toBe("success"); + expect(classify([], "cancelled").conclusion).toBe("failure"); + expect(classify(complete.slice(0, 1)).title).toBe("Evidence is missing"); + expect(classify([...complete, complete[0]!]).title).toBe("Duplicate evidence"); + expect( + classify([signal(gate, "onboard-repair", "default", { skipped: 1 }), complete[1]!]).title, + ).toBe("Evidence is incomplete"); + expect( + classify([ + signal(gate, "onboard-repair", "default", { failed: 1, runReason: "failed" }), + complete[1]!, + ]).title, + ).toBe("Tests failed"); + }); + + it("binds every signal to the revision, plan, correlation, job, and shard", () => { + const gate = state(); + const valid = signal(gate, "onboard-repair"); + + expect(validateSignal(valid, gate)).toEqual(valid); + expect(() => validateSignal({ ...valid, testedSha: BASE_SHA }, gate)).toThrow(/tested SHA/u); + expect(() => validateSignal({ ...valid, planHash: "c".repeat(64) }, gate)).toThrow( + /plan hash/u, + ); + expect(() => + validateSignal({ ...valid, correlationId: CORRELATION_ID.replace(/.$/u, "d") }, gate), + ).toThrow(/correlation/u); + expect(() => validateSignal({ ...valid, jobId: "other" }, gate)).toThrow(/unexpected/u); + }); + + it("derives shard policy from the checked-in workflow", () => { + expect(expectedSignalShards(["onboard-repair", "onboard-resume"])).toEqual({ + "onboard-repair": ["default"], + "onboard-resume": ["default"], + }); + const broadPlan = buildRiskPlan({ headSha: HEAD_SHA, changedFiles: BROAD_FILES }); + const broadShards = expectedSignalShards(riskPlanRequiredJobIds(broadPlan)); + expect(Object.keys(broadShards)).toHaveLength(13); + expect(Object.values(broadShards).flat()).toHaveLength(15); + expect(() => expectedSignalShards(["not-a-workflow-job"])).toThrow(/does not define/u); + }); + + it("dispatches every selected job through the five-field child protocol", async () => { + const jobs = ["onboard-repair", "onboard-resume", "full-e2e", "hermes-e2e"]; + const fetchMock = vi.spyOn(globalThis, "fetch").mockImplementation( + createGitHubFetchRouter([ + githubFetchRoute( + ({ url }) => url.endsWith("/git/ref/heads/main"), + () => + githubResponse({ + ref: "refs/heads/main", + object: { type: "commit", sha: WORKFLOW_SHA }, + }), + ), + githubFetchRoute( + ({ url }) => url.endsWith("/actions/workflows/e2e.yaml/dispatches"), + () => + githubResponse({ + workflow_run_id: 23, + run_url: "https://api.github.com/repos/NVIDIA/NemoClaw/actions/runs/23", + html_url: "https://github.com/NVIDIA/NemoClaw/actions/runs/23", + }), + ), + ]), + ); + + await expect( + dispatchPrGate({ + repository: "NVIDIA/NemoClaw", + token: "token", + jobs, + prNumber: 42, + commitSha: HEAD_SHA, + workflowSha: WORKFLOW_SHA, + planHash: "c".repeat(64), + correlationId: CORRELATION_ID, + }), + ).resolves.toBe(23); + expect(String(fetchMock.mock.calls[0]?.[0])).toContain("git/ref/heads/main"); + const request = fetchMock.mock.calls[1]!; + expect(String(request[0])).toContain("actions/workflows/e2e.yaml/dispatches"); + expect(JSON.parse(String(request[1]?.body))).toEqual({ + ref: "main", + inputs: { + jobs: jobs.join(","), + pr_number: "42", + checkout_sha: HEAD_SHA, + plan_hash: "c".repeat(64), + correlation_id: CORRELATION_ID, + }, + return_run_details: true, + }); + expect(() => + validateWorkflowDispatchDetails( + { + workflow_run_id: 23, + run_url: "https://api.github.com/repos/NVIDIA/NemoClaw/actions/runs/24", + html_url: "https://github.com/NVIDIA/NemoClaw/actions/runs/23", + }, + "NVIDIA/NemoClaw", + ), + ).toThrow(/mismatched workflow dispatch URLs/u); + }); + + it("refuses dispatch after main advances", async () => { + const fetchMock = vi.spyOn(globalThis, "fetch").mockResolvedValue( + githubResponse({ + ref: "refs/heads/main", + object: { type: "commit", sha: BASE_SHA }, + }), + ); + + await expect( + dispatchPrGate({ + repository: "NVIDIA/NemoClaw", + token: "token", + jobs: ["onboard-repair"], + prNumber: 42, + commitSha: HEAD_SHA, + workflowSha: WORKFLOW_SHA, + planHash: "c".repeat(64), + correlationId: CORRELATION_ID, + }), + ).rejects.toThrow(/main no longer points/u); + expect(fetchMock).toHaveBeenCalledOnce(); + }); + + it("uses one child title for dispatch correlation and verification", () => { + const gate = state(); + const child = workflowRun(gate); + const identity = { + childRunId: 23, + correlationId: gate.correlationId, + prNumber: gate.prNumber, + repository: "NVIDIA/NemoClaw", + workflowSha: gate.workflowSha, + }; + + expect(() => assertCorrelatedWorkflowRun(child, identity)).not.toThrow(); + expect(() => + assertCorrelatedWorkflowRun({ ...child, display_title: "E2E unrelated" }, identity), + ).toThrow(/display_title/u); + }); + + it("rejects fork branches before making API requests", async () => { + const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pr-e2e-gate-fork-")); + vi.stubEnv("GITHUB_TOKEN", "token"); + vi.stubEnv("GITHUB_REPOSITORY", "NVIDIA/NemoClaw"); + const fetchMock = vi.spyOn(globalThis, "fetch"); + + try { + await expect( + startPrGate({ ...startCommand(workDir), headRepository: "contributor/NemoClaw" }), + ).rejects.toThrow(/PR branch must be in the base repository/u); + expect(fetchMock).not.toHaveBeenCalled(); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + }); + + it("completes the check when all evidence passes", async () => { + const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pr-e2e-gate-lifecycle-")); + const outputPath = path.join(workDir, "github-output"); + fs.writeFileSync(outputPath, "", { mode: 0o600 }); + vi.stubEnv("GITHUB_TOKEN", "token"); + vi.stubEnv("GITHUB_REPOSITORY", "NVIDIA/NemoClaw"); + vi.stubEnv("GITHUB_OUTPUT", outputPath); + const requests: RecordedGitHubRequest[] = []; + let gate: PrGateState | undefined; + vi.spyOn(globalThis, "fetch").mockImplementation( + createGitHubFetchRouter( + [ + githubFetchRoute( + ({ url, method }) => url.endsWith("/check-runs") && method === "POST", + () => githubResponse({ id: 17 }), + ), + githubFetchRoute( + ({ url }) => url.includes("/pulls?state=open&head="), + () => githubResponse([pullRequestListItem(pullRequest(BROAD_FILES.length))]), + ), + githubFetchRoute( + ({ url }) => url.includes("/pulls/42/files?"), + () => githubResponse(BROAD_FILES.map((filename) => ({ filename }))), + ), + githubFetchRoute( + ({ url }) => url.endsWith("/pulls/42"), + () => githubResponse(pullRequest(BROAD_FILES.length)), + ), + githubFetchRoute( + ({ url }) => url.endsWith("/git/ref/heads/main"), + () => + githubResponse({ + ref: "refs/heads/main", + object: { type: "commit", sha: WORKFLOW_SHA }, + }), + ), + githubFetchRoute( + ({ url }) => url.endsWith("/actions/workflows/e2e.yaml/dispatches"), + () => + githubResponse({ + workflow_run_id: 23, + run_url: "https://api.github.com/repos/NVIDIA/NemoClaw/actions/runs/23", + html_url: "https://github.com/NVIDIA/NemoClaw/actions/runs/23", + }), + ), + githubFetchRoute( + ({ url, method }) => url.endsWith("/actions/runs/23") && method === "GET", + () => { + expect(gate).toBeDefined(); + return githubResponse(workflowRun(gate!)); + }, + ), + githubFetchRoute( + ({ url, method }) => url.endsWith("/check-runs/17") && method === "PATCH", + () => githubResponse({}), + ), + ], + requests, + ), + ); + + try { + const command = startCommand(workDir); + await startPrGate(command); + gate = validatePrGateState(JSON.parse(fs.readFileSync(command.statePath, "utf8"))); + for (const job of gate.expectedJobs) { + for (const shard of gate.expectedShards[job]!) { + const directory = path.join(command.evidencePath, `${job}-${shard}`); + fs.mkdirSync(directory, { recursive: true }); + fs.writeFileSync( + path.join(directory, "risk-signal.json"), + `${JSON.stringify(signal(gate, job, shard))}\n`, + ); + } + } + const outputs = Object.fromEntries( + fs + .readFileSync(outputPath, "utf8") + .trim() + .split("\n") + .map((line) => line.split("=", 2)), + ); + await finishPrGate({ + statePath: command.statePath, + stateHash: outputs.state_hash!, + evidencePath: command.evidencePath, + checkRunId: Number(outputs.check_id), + childRunId: Number(outputs.run_id), + }); + + expect(gate.expectedJobs).toEqual(BROAD_JOBS); + expect(requests.filter((request) => request.url.includes("/pulls?"))).toHaveLength(2); + expect(requests.filter((request) => request.url.endsWith("/pulls/42"))).toHaveLength(2); + const checkCreation = requests.find( + (request) => request.url.endsWith("/check-runs") && request.method === "POST", + ); + expect(checkCreation?.body).toMatchObject({ + name: "E2E / PR Gate", + head_sha: HEAD_SHA, + status: "in_progress", + output: { + title: "Evaluating PR commit", + summary: "Validating the PR and selecting E2E jobs.", + }, + }); + const dispatch = requests.find((request) => request.url.endsWith("/dispatches")); + expect(dispatch?.body).toMatchObject({ + inputs: { + jobs: BROAD_JOBS.join(","), + pr_number: "42", + checkout_sha: HEAD_SHA, + plan_hash: gate.planHash, + correlation_id: gate.correlationId, + }, + }); + const checkUpdates = requests.filter( + (request) => request.url.endsWith("/check-runs/17") && request.method === "PATCH", + ); + expect(checkUpdates).toHaveLength(2); + expect(checkUpdates[0]?.body).toMatchObject({ + status: "in_progress", + output: { + title: "Running 13 E2E jobs", + summary: expect.stringContaining("upgrade-stale-sandbox"), + }, + }); + expect(checkUpdates[1]?.body).toMatchObject({ + status: "completed", + conclusion: "success", + output: { title: "All selected jobs passed" }, + }); + expect(fs.readFileSync(outputPath, "utf8")).toContain("finalized=true"); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + }); + + it("fails without dispatch when the pull request changes during planning", async () => { + const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pr-e2e-gate-race-")); + const outputPath = path.join(workDir, "github-output"); + fs.writeFileSync(outputPath, "", { mode: 0o600 }); + vi.stubEnv("GITHUB_TOKEN", "token"); + vi.stubEnv("GITHUB_REPOSITORY", "NVIDIA/NemoClaw"); + vi.stubEnv("GITHUB_OUTPUT", outputPath); + const requests: RecordedGitHubRequest[] = []; + let listCalls = 0; + let detailCalls = 0; + const updatedPull = { + ...pullRequest(), + base: { ...pullRequest().base, sha: "c".repeat(40) }, + }; + vi.spyOn(globalThis, "fetch").mockImplementation( + createGitHubFetchRouter( + [ + githubFetchRoute( + ({ url, method }) => url.endsWith("/check-runs") && method === "POST", + () => githubResponse({ id: 17 }), + ), + githubFetchRoute( + ({ url }) => url.includes("/pulls?state=open&head="), + () => { + listCalls += 1; + return githubResponse([ + pullRequestListItem(listCalls === 1 ? pullRequest() : updatedPull), + ]); + }, + ), + githubFetchRoute( + ({ url }) => url.includes("/pulls/42/files?"), + () => githubResponse([{ filename: "src/lib/onboard.ts" }]), + ), + githubFetchRoute( + ({ url }) => url.endsWith("/pulls/42"), + () => { + detailCalls += 1; + return githubResponse(detailCalls === 1 ? pullRequest() : updatedPull); + }, + ), + githubFetchRoute( + ({ url, method }) => url.endsWith("/check-runs/17") && method === "PATCH", + () => githubResponse({}), + ), + ], + requests, + ), + ); + + try { + await expect(startPrGate(startCommand(workDir))).rejects.toThrow( + /PR changed during preparation/u, + ); + expect(requests.some((request) => request.url.endsWith("/dispatches"))).toBe(false); + expect(requests.some((request) => request.url.endsWith("/git/ref/heads/main"))).toBe(false); + const finalUpdate = requests.find( + (request) => request.url.endsWith("/check-runs/17") && request.method === "PATCH", + ); + expect(finalUpdate?.body).toMatchObject({ status: "completed", conclusion: "failure" }); + expect(fs.readFileSync(outputPath, "utf8")).toContain("finalized=true"); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + }); + + it("cancels the child and closes the check when startup fails after dispatch", async () => { + const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pr-e2e-gate-start-")); + const outputPath = path.join(workDir, "github-output"); + fs.writeFileSync(outputPath, "", { mode: 0o600 }); + vi.stubEnv("GITHUB_TOKEN", "token"); + vi.stubEnv("GITHUB_REPOSITORY", "NVIDIA/NemoClaw"); + vi.stubEnv("GITHUB_OUTPUT", outputPath); + const requests: RecordedGitHubRequest[] = []; + let checkPatches = 0; + vi.spyOn(globalThis, "fetch").mockImplementation( + createGitHubFetchRouter( + [ + githubFetchRoute( + ({ url, method }) => url.endsWith("/check-runs") && method === "POST", + () => githubResponse({ id: 17 }), + ), + githubFetchRoute( + ({ url }) => url.includes("/pulls?state=open&head="), + () => githubResponse([pullRequestListItem()]), + ), + githubFetchRoute( + ({ url }) => url.includes("/pulls/42/files?"), + () => githubResponse([{ filename: "src/lib/onboard.ts" }]), + ), + githubFetchRoute( + ({ url }) => url.endsWith("/pulls/42"), + () => githubResponse(pullRequest()), + ), + githubFetchRoute( + ({ url }) => url.endsWith("/git/ref/heads/main"), + () => + githubResponse({ + ref: "refs/heads/main", + object: { type: "commit", sha: WORKFLOW_SHA }, + }), + ), + githubFetchRoute( + ({ url }) => url.endsWith("/actions/workflows/e2e.yaml/dispatches"), + () => + githubResponse({ + workflow_run_id: 23, + run_url: "https://api.github.com/repos/NVIDIA/NemoClaw/actions/runs/23", + html_url: "https://github.com/NVIDIA/NemoClaw/actions/runs/23", + }), + ), + githubFetchRoute( + ({ url, method }) => url.endsWith("/actions/runs/23/cancel") && method === "POST", + () => githubResponse(undefined, 202), + ), + githubFetchRoute( + ({ url, method }) => url.endsWith("/check-runs/17") && method === "PATCH", + () => { + checkPatches += 1; + return checkPatches === 1 + ? githubResponse({ message: "simulated update failure" }, 500) + : githubResponse({}); + }, + ), + ], + requests, + ), + ); + + try { + await expect(startPrGate(startCommand(workDir))).rejects.toThrow(/simulated update failure/u); + expect(requests.some((request) => request.url.endsWith("/actions/runs/23/cancel"))).toBe( + true, + ); + const checkUpdates = requests.filter((request) => request.url.endsWith("/check-runs/17")); + expect(checkUpdates).toHaveLength(2); + expect(checkUpdates[1]?.body).toMatchObject({ + status: "completed", + conclusion: "failure", + output: { + title: "Run could not start", + summary: expect.stringContaining("The controller could not complete the check."), + }, + }); + expect(fs.readFileSync(outputPath, "utf8")).toContain("finalized=true"); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + }); + + it.each([ + { + label: "missing evidence", + status: "completed", + expectCancellation: false, + expectedTitle: "Evidence is missing", + }, + { + label: "an unfinished child", + status: "in_progress", + expectCancellation: true, + expectedTitle: "E2E run did not succeed", + }, + ])("closes the check as failure for $label", async ({ + status, + expectCancellation, + expectedTitle, + }) => { + const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pr-e2e-gate-finish-")); + const outputPath = path.join(workDir, "github-output"); + const statePath = path.join(workDir, "controller-state.json"); + const evidencePath = path.join(workDir, "evidence"); + const gate = state(); + const serializedState = `${JSON.stringify(gate, null, 2)}\n`; + fs.writeFileSync(outputPath, "", { mode: 0o600 }); + fs.writeFileSync(statePath, serializedState, { mode: 0o600 }); + fs.mkdirSync(evidencePath); + vi.stubEnv("GITHUB_TOKEN", "token"); + vi.stubEnv("GITHUB_REPOSITORY", "NVIDIA/NemoClaw"); + vi.stubEnv("GITHUB_OUTPUT", outputPath); + const requests: RecordedGitHubRequest[] = []; + vi.spyOn(globalThis, "fetch").mockImplementation( + createGitHubFetchRouter( + [ + githubFetchRoute( + ({ url, method }) => url.endsWith("/actions/runs/23") && method === "GET", + () => githubResponse(workflowRun(gate, { status, conclusion: "success" })), + ), + githubFetchRoute( + ({ url, method }) => url.endsWith("/actions/runs/23/cancel") && method === "POST", + () => githubResponse(undefined, 202), + ), + githubFetchRoute( + ({ url, method }) => url.endsWith("/check-runs/17") && method === "PATCH", + () => githubResponse({}), + ), + ], + requests, + ), + ); + + try { + await expect( + finishPrGate({ + statePath, + stateHash: sha256(serializedState), + evidencePath, + checkRunId: 17, + childRunId: 23, + }), + ).rejects.toThrow(); + expect(requests.some((request) => request.url.endsWith("/actions/runs/23/cancel"))).toBe( + expectCancellation, + ); + const completion = requests.find((request) => request.url.endsWith("/check-runs/17")); + expect(completion?.body).toMatchObject({ + status: "completed", + conclusion: "failure", + output: { title: expectedTitle }, + }); + expect(fs.readFileSync(outputPath, "utf8")).toContain("finalized=true"); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + }); + + it("cancels only active child runs for the pull request", async () => { + vi.stubEnv("GITHUB_TOKEN", "token"); + vi.stubEnv("GITHUB_REPOSITORY", "NVIDIA/NemoClaw"); + const gate = state(); + const fetchMock = vi.spyOn(globalThis, "fetch").mockImplementation( + createGitHubFetchRouter([ + githubFetchRoute( + ({ url }) => url.includes("/actions/workflows/e2e.yaml/runs?"), + () => + githubResponse({ + workflow_runs: [ + workflowRun(gate, { status: "in_progress" }), + workflowRun(gate, { id: 24, status: "completed" }), + workflowRun(gate, { id: 25, status: "queued", display_title: "E2E manual" }), + workflowRun({ ...gate, prNumber: 420 }, { id: 26, status: "queued" }), + ], + }), + ), + githubFetchRoute( + ({ url, method }) => url.endsWith("/actions/runs/23/cancel") && method === "POST", + () => githubResponse(undefined, 202), + ), + ]), + ); + + await expect(cancelPrGate(42)).resolves.toBe(1); + expect( + fetchMock.mock.calls.filter(([input]) => String(input).endsWith("/cancel")), + ).toHaveLength(1); + expect(fetchMock.mock.calls.some(([input]) => String(input).endsWith("/26/cancel"))).toBe( + false, + ); + }); + + it("cancels a known child and closes an abandoned check as failure", async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pr-e2e-gate-abandon-")); + const outputPath = path.join(directory, "github-output"); + fs.writeFileSync(outputPath, "", { mode: 0o600 }); + vi.stubEnv("GITHUB_TOKEN", "token"); + vi.stubEnv("GITHUB_REPOSITORY", "NVIDIA/NemoClaw"); + vi.stubEnv("GITHUB_OUTPUT", outputPath); + const requests: RecordedGitHubRequest[] = []; + vi.spyOn(globalThis, "fetch").mockImplementation( + createGitHubFetchRouter( + [ + githubFetchRoute( + ({ url, method }) => url.endsWith("/actions/runs/23/cancel") && method === "POST", + () => githubResponse(undefined, 202), + ), + githubFetchRoute( + ({ url, method }) => url.endsWith("/check-runs/17") && method === "PATCH", + () => githubResponse(undefined), + ), + ], + requests, + ), + ); + + try { + await abandonPrGate(17, 23); + expect(requests.map((request) => request.url)).toEqual([ + "https://api.github.com/repos/NVIDIA/NemoClaw/actions/runs/23/cancel", + "https://api.github.com/repos/NVIDIA/NemoClaw/check-runs/17", + ]); + expect(requests[1]?.body).toMatchObject({ + status: "completed", + conclusion: "failure", + output: { + title: "Controller stopped early", + summary: "The controller stopped before it could complete the check.", + }, + }); + expect(fs.readFileSync(outputPath, "utf8")).toContain("finalized=true"); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }); + + it("bounds recursive signal discovery and rejects symlinks", () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-pr-e2e-gate-evidence-")); + try { + const first = path.join(directory, "first"); + fs.mkdirSync(first); + fs.writeFileSync(path.join(first, "risk-signal.json"), "{}\n"); + expect(findSignalFiles(directory, { maxDepth: 2, maxEntries: 3, maxSignalFiles: 1 })).toEqual( + [path.join(first, "risk-signal.json")], + ); + + const second = path.join(directory, "second"); + fs.mkdirSync(second); + fs.writeFileSync(path.join(second, "risk-signal.json"), "{}\n"); + expect(() => + findSignalFiles(directory, { maxDepth: 2, maxEntries: 8, maxSignalFiles: 1 }), + ).toThrow(/signal-file limit/u); + + fs.rmSync(second, { recursive: true }); + fs.symlinkSync(first, path.join(directory, "linked")); + expect(() => + findSignalFiles(directory, { maxDepth: 2, maxEntries: 8, maxSignalFiles: 2 }), + ).toThrow(/symlinks/u); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }); +}); diff --git a/test/pr-risk-plan.test.ts b/test/pr-risk-plan.test.ts index d238e15560c..cd47c8573d3 100644 --- a/test/pr-risk-plan.test.ts +++ b/test/pr-risk-plan.test.ts @@ -18,6 +18,7 @@ describe("deterministic PR risk plan", () => { const second = plan("src/lib/onboard.ts", "src/lib/state/registry.ts"); expect(first).toEqual(second); + expect(first.version).toBe(2); expect(first.headSha).toBe(HEAD_SHA); expect(first.planHash).toMatch(/^[a-f0-9]{64}$/u); expect(first.changedFiles).toEqual(["src/lib/onboard.ts", "src/lib/state/registry.ts"]); @@ -29,7 +30,6 @@ describe("deterministic PR risk plan", () => { expect(result.tier).toBe(0); expect(result.families).toEqual([]); expect(result.requiredJobs).toEqual([]); - expect(result.requiresManualExpansion).toBe(false); }); it("keeps the canonical cloud-onboard live test in the platform floor (#6446)", () => { @@ -145,19 +145,32 @@ describe("deterministic PR risk plan", () => { expect(riskPlanRequiredJobIds(result)).toEqual(expect.arrayContaining(jobs)); }); - it("caps automatic execution without dropping required evidence", () => { + it("keeps every required job selected for broad runtime changes (#6446)", () => { const result = plan( "src/lib/onboard.ts", + "src/lib/actions/upgrade-sandboxes.ts", + "src/lib/actions/sandbox/agents/apply.ts", "src/lib/messaging/applier/agent-config.ts", "src/lib/inference/health.ts", + "install.sh", + "src/lib/credentials/provider-list.ts", ); - expect(result.requiredJobs.length).toBeGreaterThan(result.maxAutomaticJobs); - expect(result.automaticJobs).toHaveLength(result.maxAutomaticJobs); - expect(result.requiresManualExpansion).toBe(true); - expect(result.requiredJobs.map((job) => job.id)).toEqual( - expect.arrayContaining(result.automaticJobs), - ); + expect(riskPlanRequiredJobIds(result)).toEqual([ + "cloud-onboard", + "credential-sanitization", + "security-posture", + "channels-add-remove", + "channels-stop-start", + "full-e2e", + "hermes-e2e", + "inference-routing", + "network-policy", + "onboard-repair", + "onboard-resume", + "state-backup-restore", + "upgrade-stale-sandbox", + ]); }); it("raises PR review test depth for a matched runtime risk", () => { diff --git a/test/support/github-fetch-router.ts b/test/support/github-fetch-router.ts new file mode 100644 index 00000000000..3c8995404f5 --- /dev/null +++ b/test/support/github-fetch-router.ts @@ -0,0 +1,49 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export interface RecordedGitHubRequest { + body?: unknown; + method: string; + url: string; +} + +export interface GitHubFetchRoute { + matches: (request: RecordedGitHubRequest) => boolean; + respond: (request: RecordedGitHubRequest) => Promise | Response; +} + +async function readRequestBody(input: Parameters[0], init?: RequestInit) { + if (init?.body !== undefined) return String(init.body); + if (input instanceof Request) return input.clone().text(); + return ""; +} + +export function githubFetchRoute( + matches: GitHubFetchRoute["matches"], + respond: GitHubFetchRoute["respond"], +): GitHubFetchRoute { + return { matches, respond }; +} + +export function createGitHubFetchRouter( + routes: readonly GitHubFetchRoute[], + requests?: RecordedGitHubRequest[], +): typeof fetch { + return (async (input, init) => { + const requestInput = input instanceof Request ? input : undefined; + const serializedBody = await readRequestBody(input, init); + const request: RecordedGitHubRequest = { + url: requestInput?.url ?? String(input), + method: init?.method ?? requestInput?.method ?? "GET", + body: serializedBody === "" ? undefined : JSON.parse(serializedBody), + }; + requests?.push(request); + const matchingRoutes = routes.filter((candidate) => candidate.matches(request)); + if (matchingRoutes.length !== 1) { + throw new Error( + `Expected one route for ${request.method} ${request.url}, matched ${matchingRoutes.length}`, + ); + } + return matchingRoutes[0]!.respond(request); + }) as typeof fetch; +} diff --git a/tools/advisors/risk-plan.mts b/tools/advisors/risk-plan.mts index d41f5f7550f..0921bd98b90 100644 --- a/tools/advisors/risk-plan.mts +++ b/tools/advisors/risk-plan.mts @@ -3,8 +3,7 @@ import { createHash } from "node:crypto"; -export const RISK_PLAN_VERSION = 1 as const; -export const DEFAULT_MAX_AUTOMATIC_JOBS = 3; +export const RISK_PLAN_VERSION = 2 as const; export type RiskTier = 0 | 1 | 2 | 3; export type RiskFamilyId = @@ -41,9 +40,6 @@ export type RiskPlan = { tier: RiskTier; families: RiskPlanFamily[]; requiredJobs: RiskPlanJob[]; - automaticJobs: string[]; - maxAutomaticJobs: number; - requiresManualExpansion: boolean; }; type RiskRule = Omit & { @@ -223,7 +219,6 @@ function planDigest(value: Omit): string { export function buildRiskPlan(options: { headSha: string; changedFiles: readonly string[]; - maxAutomaticJobs?: number; }): RiskPlan { const changedFiles = stableUnique(options.changedFiles); const runtimeFiles = changedFiles.filter(isRuntimeRelevant); @@ -263,8 +258,6 @@ export function buildRiskPlan(options: { const requiredJobs = [...jobs.values()].sort( (left, right) => right.tier - left.tier || left.id.localeCompare(right.id), ); - const maxAutomaticJobs = options.maxAutomaticJobs ?? DEFAULT_MAX_AUTOMATIC_JOBS; - const automaticJobs = requiredJobs.slice(0, maxAutomaticJobs).map((job) => job.id); const tier = families.reduce( (highest, family) => Math.max(highest, family.tier) as RiskTier, 0, @@ -276,9 +269,6 @@ export function buildRiskPlan(options: { tier, families, requiredJobs, - automaticJobs, - maxAutomaticJobs, - requiresManualExpansion: requiredJobs.length > maxAutomaticJobs, }; return { ...withoutHash, planHash: planDigest(withoutHash) }; diff --git a/tools/e2e-advisor/README.md b/tools/e2e-advisor/README.md index 11b6991f91f..35e4e63bd71 100644 --- a/tools/e2e-advisor/README.md +++ b/tools/e2e-advisor/README.md @@ -38,42 +38,18 @@ the trusted timing signal. `.github/workflows/e2e.yaml`, but the advisor job does not trigger those commands automatically. -## Post-merge shadow controller - -The model-independent `.github/workflows/post-merge-e2e-risk-gate-shadow.yaml` workflow -uses the same checked-in risk-plan policy after a commit lands on `main`. It -does not consume model output or the advisor artifact. Instead, -`tools/e2e-advisor/post-merge-risk-gate.mts` confirms the trusted controller checkout, -builds a new plan from the exact `github.event.before` and `github.event.after` range, -and dispatches at most three `automaticJobs` to `e2e.yaml` against the merged -commit. The controller opts into GitHub's workflow-dispatch run details and -uses the returned run ID as the sole child-run selector, so a lookalike run -cannot win a polling race for the same correlation ID. - -The child workflow validates that the exact checkout SHA equals the workflow's -own current `main` commit, verifies its reachability, and checks selective-job -inputs, plan hash, and correlation ID before E2E preparation or secret-bearing -jobs can run. If `main` advances before an older controller dispatches, that -child fails closed and the controller reports failure rather than executing the -older commit with current secrets. The shadow-only Vitest reporter -records the observed checkout SHA and pass, failure, skip, pending, and -unhandled-error counts for each job and matrix shard. The controller accepts -only signals bound to the expected SHA, plan hash, correlation ID, job, and -shard. - -The start step records a SHA-256 digest of its private controller state in a -GitHub step output. After child artifacts are downloaded, the finish step reads -that state once and verifies the digest before parsing it or classifying any -evidence, so downloaded files cannot change the dispatch state used for the -check result. - -The controller writes a check on the merged commit without posting a PR -comment or running the scheduled/manual scorecard. Complete unskipped evidence -reports success, selected E2E workflow or test failures for the merged commit report -failure, and incomplete, ambiguous, skipped, or cap-limited evidence that -requires manual expansion reports neutral. If no runtime risk family matches, -it reports success without dispatching E2E. This is post-merge shadow evidence, -not a required pre-merge check. +## PR E2E check + +`.github/workflows/pr-e2e-gate.yaml` owns `E2E / PR Gate` for PRs from this +repository after `CI / Pull Request` completes. It uses the same checked-in +risk policy as E2E Advisor, but rebuilds the plan from GitHub's changed-file +list and never consumes advisor output. It dispatches every selected +`requiredJobs` entry and verifies the resulting E2E evidence. See +[NemoClaw E2E CI](../../test/e2e/README.md) for the full lifecycle. + +E2E Advisor remains advisory. It uses the risk policy as a recommendation +floor and may add adjacent coverage, but its model output and availability do +not determine the PR E2E check. ## Required secret @@ -98,11 +74,12 @@ dispatch commands; it does not trigger E2E workflows automatically. ## Artifacts - `e2e-advisor-prompt.md` — task prompt sent to the advisor. Diff, changed files, metadata, and schema are exposed through deterministic turn-scoped context tools and captured in the session transcript. -- `risk-plan.json` — deterministic risk families, invariants, and required jobs for the PR - head commit and changed-file set, plus a capped `automaticJobs` subset, - manual-expansion state, and the plan digest. Both E2E advisor projections consume the - required-job floor, while the separate post-merge controller rebuilds the plan and - dispatches the automatic subset. +- `risk-plan.json` — deterministic risk families, invariants, required jobs, + changed files, and the plan digest for the pull request revision. + Both E2E Advisor projections consume this required-job floor. + The PR E2E controller independently rebuilds the plan from GitHub's pull + request file list and dispatches every selected job, so this advisor + artifact is not an input to the PR E2E check. - `e2e-advisor-raw-output.txt` — raw advisor transcript and diagnostics. - `e2e-advisor-result.json` — parsed advisor response or execution metadata. - `e2e-advisor-session.html` — exported advisor session transcript. @@ -133,7 +110,3 @@ secret. Run `npm install` first so the Pi SDK dependency is available. `tools/e2e-advisor/schema.json` defines the normalized coverage recommendation shape. `tools/e2e-advisor/targets-schema.json` defines the normalized target recommendation shape used by the `targets` and `jobs` dispatch commands. - -The post-merge shadow check does not establish pre-merge enforcement. Any future required check -must verify complete E2E evidence for the same PR head commit without making model availability part -of the merge authority. diff --git a/tools/e2e-advisor/post-merge-risk-gate.mts b/tools/e2e-advisor/post-merge-risk-gate.mts deleted file mode 100755 index cb75b049b3d..00000000000 --- a/tools/e2e-advisor/post-merge-risk-gate.mts +++ /dev/null @@ -1,965 +0,0 @@ -#!/usr/bin/env node - -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { execFileSync } from "node:child_process"; -import { createHash, randomUUID } from "node:crypto"; -import fs from "node:fs"; -import path from "node:path"; -import { pathToFileURL } from "node:url"; - -import YAML from "yaml"; - -import { githubApi } from "../advisors/github.mts"; -import { parseArgs } from "../advisors/io.mts"; -import { buildRiskPlan, type RiskPlan } from "../advisors/risk-plan.mts"; -import { readFreeStandingJobsInventory } from "../e2e/workflow-boundary.mts"; -import { readPrivateRegularFile, writePrivateRegularFile } from "./private-file.ts"; -import type { E2eRiskSignal } from "./risk-signal.ts"; - -const E2E_WORKFLOW = "e2e.yaml"; -const E2E_WORKFLOW_PATH = `.github/workflows/${E2E_WORKFLOW}`; -const CHECK_NAME = "E2E / Post-merge Risk Gate (shadow)"; -const SHA_PATTERN = /^[a-f0-9]{40}$/u; -const HASH_PATTERN = /^[a-f0-9]{64}$/u; -const JOB_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]*$/u; -const SHARD_PATTERN = /^(?:default|[A-Za-z0-9][A-Za-z0-9_-]*)$/u; -const CORRELATION_PATTERN = - /^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/u; -const MAX_PLAN_BYTES = 1024 * 1024; -const MAX_CONTROLLER_ERROR_CHARS = 512; -const DEFAULT_EVIDENCE_LIMITS = { - maxDepth: 8, - maxEntries: 4096, - maxSignalFiles: 12, -} as const; - -type ControllerPaths = { - planPath: string; - statePath: string; - evidencePath: string; -}; - -export type ControllerCommand = - | ({ mode: "start"; baseSha: string; commitSha: string } & ControllerPaths) - | ({ - mode: "finish"; - checkRunId: number; - childRunId: number; - stateHash: string; - } & ControllerPaths) - | { mode: "abandon"; checkRunId: number }; - -type CheckConclusion = "success" | "failure" | "neutral"; - -type WorkflowRun = { - id: number; - name: string; - path: string; - workflow_id: number; - event: string; - head_sha: string; - status: string; - conclusion: string | null; - created_at: string; - display_title: string; - html_url: string; -}; - -type CheckRun = { id: number }; - -type WorkflowDispatchDetails = { - workflow_run_id: number; - run_url: string; - html_url: string; -}; - -type WorkflowRunIdentity = { - childRunId: number; - correlationId: string; - repository: string; -}; - -export type RiskGateState = { - version: 1; - commitSha: string; - planHash: string; - correlationId: string; - expectedJobs: string[]; - expectedShards: Record; - requiresManualExpansion: boolean; -}; - -export type RiskEvidenceVerdict = { - conclusion: CheckConclusion; - title: string; - summary: string; -}; - -export function assertTrustedMainPush(options: { - eventName: string | undefined; - ref: string | undefined; - sha: string | undefined; - commitSha: string; -}): void { - if ( - options.eventName !== "push" || - options.ref !== "refs/heads/main" || - options.sha !== options.commitSha - ) { - throw new Error("post-merge risk dispatch requires the exact trusted main push context"); - } -} - -function isObjectRecord(value: unknown): value is Record { - return !!value && typeof value === "object" && !Array.isArray(value); -} - -function requiredArgument(value: string | undefined, name: string): string { - if (!value) throw new Error(`--${name} is required`); - return value; -} - -function parsePositiveId(value: string, name: string): number { - if (!/^[1-9][0-9]*$/u.test(value)) throw new Error(`${name} must be a positive integer`); - const parsed = Number(value); - if (!Number.isSafeInteger(parsed)) throw new Error(`${name} exceeds the safe integer range`); - return parsed; -} - -function parseHash(value: string | undefined, name: string): string { - const parsed = requiredArgument(value, name); - if (!HASH_PATTERN.test(parsed)) throw new Error(`--${name} must be a lowercase SHA-256 hash`); - return parsed; -} - -function sha256(value: string): string { - return createHash("sha256").update(value).digest("hex"); -} - -export function privateControllerPaths(workDir: string): ControllerPaths { - const resolved = path.resolve(workDir); - const stat = fs.lstatSync(resolved); - const currentUid = typeof process.getuid === "function" ? process.getuid() : null; - if ( - resolved !== workDir || - !stat.isDirectory() || - stat.isSymbolicLink() || - (stat.mode & 0o077) !== 0 || - (currentUid !== null && stat.uid !== currentUid) - ) { - throw new Error("--work-dir must be an owned private absolute directory"); - } - return { - planPath: path.join(resolved, "post-merge-risk-plan.json"), - statePath: path.join(resolved, "e2e-risk-gate-state.json"), - evidencePath: path.join(resolved, "evidence"), - }; -} - -export function parseControllerCommand(argv: string[]): ControllerCommand { - const args = parseArgs(argv); - if (args.mode === "start") { - return { - mode: "start", - baseSha: requiredArgument(args.base, "base"), - commitSha: requiredArgument(args.commit, "commit"), - ...privateControllerPaths(requiredArgument(args.workDir, "work-dir")), - }; - } - if (args.mode === "finish") { - return { - mode: "finish", - ...privateControllerPaths(requiredArgument(args.workDir, "work-dir")), - checkRunId: parsePositiveId(requiredArgument(args.checkId, "check-id"), "--check-id"), - childRunId: parsePositiveId(requiredArgument(args.runId, "run-id"), "--run-id"), - stateHash: parseHash(args.stateHash, "state-hash"), - }; - } - if (args.mode === "abandon") { - return { - mode: "abandon", - checkRunId: parsePositiveId(requiredArgument(args.checkId, "check-id"), "--check-id"), - }; - } - throw new Error("--mode must be start, finish, or abandon"); -} - -function readRegularJson(file: string, maxBytes = MAX_PLAN_BYTES): unknown { - return JSON.parse(readPrivateRegularFile(file, { maxBytes })!); -} - -export function validateRiskGateState(value: unknown): RiskGateState { - if (!isObjectRecord(value) || value.version !== 1) { - throw new Error("invalid risk-gate state version"); - } - if (typeof value.commitSha !== "string" || !SHA_PATTERN.test(value.commitSha)) { - throw new Error("risk-gate state commit SHA is invalid"); - } - if (typeof value.planHash !== "string" || !HASH_PATTERN.test(value.planHash)) { - throw new Error("risk-gate state plan hash is invalid"); - } - if (typeof value.correlationId !== "string" || !CORRELATION_PATTERN.test(value.correlationId)) { - throw new Error("risk-gate state correlation id is invalid"); - } - if ( - !Array.isArray(value.expectedJobs) || - value.expectedJobs.length < 1 || - value.expectedJobs.length > 3 || - !value.expectedJobs.every((job) => typeof job === "string" && JOB_PATTERN.test(job)) || - new Set(value.expectedJobs).size !== value.expectedJobs.length - ) { - throw new Error("risk-gate state expected jobs are invalid"); - } - if (!isObjectRecord(value.expectedShards)) { - throw new Error("risk-gate state shards are invalid"); - } - const shardJobs = Object.keys(value.expectedShards).sort(); - if (JSON.stringify(shardJobs) !== JSON.stringify([...value.expectedJobs].sort())) { - throw new Error("risk-gate state shard jobs do not match expected jobs"); - } - for (const job of value.expectedJobs) { - const shards = value.expectedShards[job]; - if ( - !Array.isArray(shards) || - shards.length < 1 || - new Set(shards).size !== shards.length || - !shards.every((shard) => typeof shard === "string" && SHARD_PATTERN.test(shard)) - ) { - throw new Error(`risk-gate state shards are invalid for ${job}`); - } - } - if (typeof value.requiresManualExpansion !== "boolean") { - throw new Error("risk-gate manual-expansion state is invalid"); - } - return value as RiskGateState; -} - -export function validateRiskPlan(value: unknown, allowedJobs: ReadonlySet): RiskPlan { - if (!isObjectRecord(value)) throw new Error("risk plan must be an object"); - if (value.version !== 1) throw new Error("unsupported risk-plan version"); - if (typeof value.headSha !== "string" || !SHA_PATTERN.test(value.headSha)) { - throw new Error("risk plan headSha must be a lowercase 40-character SHA"); - } - if ( - !Array.isArray(value.changedFiles) || - !value.changedFiles.every((file) => typeof file === "string") - ) { - throw new Error("risk plan changedFiles must be strings"); - } - if (value.maxAutomaticJobs !== 3) throw new Error("risk plan automatic-job cap must be 3"); - const rebuilt = buildRiskPlan({ - headSha: value.headSha, - changedFiles: value.changedFiles, - maxAutomaticJobs: value.maxAutomaticJobs, - }); - if (JSON.stringify(value) !== JSON.stringify(rebuilt)) { - throw new Error("risk plan does not match its deterministic hash and inputs"); - } - if (!HASH_PATTERN.test(rebuilt.planHash)) throw new Error("risk plan hash is invalid"); - const automatic = new Set(rebuilt.automaticJobs); - if (automatic.size !== rebuilt.automaticJobs.length) { - throw new Error("risk plan automatic jobs must be unique"); - } - for (const job of rebuilt.requiredJobs) { - if (!JOB_PATTERN.test(job.id) || !allowedJobs.has(job.id)) { - throw new Error(`risk plan names unknown E2E job: ${job.id}`); - } - } - return rebuilt; -} - -export function validateSignal( - value: unknown, - state: Pick< - RiskGateState, - "commitSha" | "planHash" | "correlationId" | "expectedJobs" | "expectedShards" - >, -): E2eRiskSignal { - if (!isObjectRecord(value) || value.version !== 1) { - throw new Error("invalid risk signal version"); - } - const signal = value as E2eRiskSignal; - if (!state.expectedJobs.includes(signal.jobId)) throw new Error("risk signal job is unexpected"); - if (!state.expectedShards[signal.jobId]?.includes(signal.shardId)) { - throw new Error("risk signal shard is unexpected"); - } - if (signal.expectedSha !== state.commitSha) throw new Error("risk signal SHA mismatch"); - if (signal.testedSha !== state.commitSha) throw new Error("risk signal tested SHA mismatch"); - if (signal.planHash !== state.planHash) throw new Error("risk signal plan hash mismatch"); - if (signal.correlationId !== state.correlationId) { - throw new Error("risk signal correlation mismatch"); - } - for (const key of ["passed", "failed", "skipped", "pending", "unhandledErrors"] as const) { - if (!Number.isSafeInteger(signal[key]) || signal[key] < 0) { - throw new Error(`risk signal ${key} must be a non-negative integer`); - } - } - if (!(["passed", "failed", "interrupted"] as const).includes(signal.runReason)) { - throw new Error("risk signal runReason is invalid"); - } - return signal; -} - -export function classifyRiskEvidence(options: { - workflowConclusion: string | null; - expectedJobs: readonly string[]; - expectedShards: Readonly>; - signals: readonly E2eRiskSignal[]; - requiresManualExpansion: boolean; -}): RiskEvidenceVerdict { - if ( - ["failure", "timed_out", "action_required", "startup_failure"].includes( - options.workflowConclusion ?? "", - ) - ) { - return { - conclusion: "failure", - title: "Selected E2E workflow failed", - summary: "The correlated workflow reported a failing terminal conclusion.", - }; - } - if (options.workflowConclusion !== "success") { - return { - conclusion: "neutral", - title: "Selected E2E workflow produced no complete signal", - summary: "The correlated workflow did not report a successful terminal conclusion.", - }; - } - const byJobShard = new Map(); - const duplicates = new Set(); - for (const signal of options.signals) { - const key = `${signal.jobId}:${signal.shardId}`; - if (byJobShard.has(key)) duplicates.add(key); - byJobShard.set(key, signal); - } - if (duplicates.size > 0) { - return { - conclusion: "neutral", - title: "Selected E2E jobs produced ambiguous evidence", - summary: "More than one signal was uploaded for at least one expected job shard.", - }; - } - const expectedEvidence = options.expectedJobs.flatMap((job) => - (options.expectedShards[job] ?? []).map((shard) => `${job}:${shard}`), - ); - const jobsWithoutShardPolicy = options.expectedJobs.filter( - (job) => (options.expectedShards[job]?.length ?? 0) === 0, - ); - if (jobsWithoutShardPolicy.length > 0) { - return { - conclusion: "neutral", - title: "Selected E2E jobs lack an evidence policy", - summary: "At least one selected job had no trusted shard policy.", - }; - } - const missing = expectedEvidence.filter((key) => !byJobShard.has(key)); - if (missing.length > 0) { - // Missing bound evidence is unverifiable, not proof that product behavior - // failed. Shadow checks become green only for complete evidence; neutral - // keeps incomplete infrastructure evidence from masquerading as a pass. - return { - conclusion: "neutral", - title: "Selected E2E jobs are missing test evidence", - summary: "At least one expected job shard did not upload a bound risk signal.", - }; - } - const failed = expectedEvidence.filter((key) => { - const signal = byJobShard.get(key)!; - return signal.failed > 0 || signal.unhandledErrors > 0 || signal.runReason === "failed"; - }); - if (failed.length > 0) { - return { - conclusion: "failure", - title: "Selected E2E jobs reported test failures", - summary: "At least one selected job shard reported a test failure or unhandled error.", - }; - } - const partial = expectedEvidence.filter((key) => { - const signal = byJobShard.get(key)!; - return ( - signal.passed < 1 || signal.skipped > 0 || signal.pending > 0 || signal.runReason !== "passed" - ); - }); - if (partial.length > 0) { - return { - conclusion: "neutral", - title: "Selected E2E jobs produced partial or skipped evidence", - summary: "At least one expected job shard did not produce a complete, unskipped pass.", - }; - } - if (options.requiresManualExpansion) { - return { - conclusion: "neutral", - title: "Automatic shadow subset passed; broader evidence is required", - summary: - "The risk plan exceeded the three-job automatic cap, so this passing subset is not complete merge evidence.", - }; - } - return { - conclusion: "success", - title: "All risk-selected E2E jobs passed", - summary: "Every expected job shard produced complete, unskipped evidence.", - }; -} - -function appendOutput(name: string, value: string): void { - const output = process.env.GITHUB_OUTPUT; - if (!output) return; - if (!/^(?:check_id|dispatched|finalized|run_id|state_hash)$/u.test(name)) { - throw new Error("invalid controller output name"); - } - const validValue = - name === "state_hash" ? HASH_PATTERN.test(value) : /^(?:true|false|[1-9][0-9]*)$/u.test(value); - if (!validValue) { - throw new Error("invalid controller output value"); - } - const descriptor = fs.openSync( - output, - fs.constants.O_WRONLY | fs.constants.O_APPEND | (fs.constants.O_NOFOLLOW ?? 0), - ); - try { - if (!fs.fstatSync(descriptor).isFile()) throw new Error("GITHUB_OUTPUT must be a regular file"); - // GitHub supplies this output file; values are restricted above to fixed - // booleans, positive decimal IDs, or a lowercase SHA-256 digest before the - // descriptor write. - // codeql[js/http-to-file-access] - fs.writeFileSync(descriptor, `${name}=${value}\n`, "utf8"); - } finally { - fs.closeSync(descriptor); - } -} - -async function createCheck( - repository: string, - token: string, - headSha: string, - title: string, - summary: string, -): Promise { - const check = await githubApi(`repos/${repository}/check-runs`, token, { - method: "POST", - body: { - name: CHECK_NAME, - head_sha: headSha, - status: "in_progress", - output: { title, summary }, - }, - userAgent: "nemoclaw-e2e-risk-gate", - }); - if (!Number.isSafeInteger(check.id) || check.id < 1) - throw new Error("GitHub returned an invalid check id"); - return check.id; -} - -async function completeCheck( - context: { repository: string; checkRunId: number }, - token: string, - verdict: RiskEvidenceVerdict, - detailsUrl?: string, -): Promise { - await githubApi(`repos/${context.repository}/check-runs/${context.checkRunId}`, token, { - method: "PATCH", - body: { - status: "completed", - conclusion: verdict.conclusion, - completed_at: new Date().toISOString(), - details_url: detailsUrl, - output: { title: verdict.title, summary: verdict.summary }, - }, - userAgent: "nemoclaw-e2e-risk-gate", - }); -} - -function controllerErrorMessage(error: unknown): string { - const message = error instanceof Error ? error.message : String(error); - const singleLine = message - .replace(/[\r\n\t]+/gu, " ") - .replace(/\s{2,}/gu, " ") - .trim(); - return singleLine.length > MAX_CONTROLLER_ERROR_CHARS - ? `${singleLine.slice(0, MAX_CONTROLLER_ERROR_CHARS - 3)}...` - : singleLine; -} - -async function completeNeutralAfterControllerError( - context: { repository: string; checkRunId: number }, - token: string, - title: string, - options: { error: unknown; detailsUrl?: string }, -): Promise { - const reason = controllerErrorMessage(options.error).replace(/`/gu, "'"); - try { - await completeCheck( - context, - token, - { - conclusion: "neutral", - title, - summary: - "The shadow controller could not produce complete, trustworthy evidence." + - `\n\nController error: \`${reason}\``, - }, - options.detailsUrl, - ); - return true; - } catch (error) { - console.error( - `failed to close shadow check after controller error: ${error instanceof Error ? error.message : String(error)}`, - ); - return false; - } -} - -export function changedFilesBetween( - baseSha: string, - commitSha: string, - workspace = process.cwd(), -): string[] { - if (!SHA_PATTERN.test(baseSha) || !SHA_PATTERN.test(commitSha)) { - throw new Error("base and tested commits must be lowercase 40-character SHAs"); - } - const checkedOutSha = execFileSync("git", ["rev-parse", "--verify", "HEAD"], { - cwd: workspace, - encoding: "utf8", - stdio: ["ignore", "pipe", "pipe"], - }).trim(); - if (checkedOutSha !== commitSha) { - throw new Error("trusted controller checkout does not match the tested commit"); - } - const output = execFileSync( - "git", - ["diff", "--no-renames", "--name-only", "-z", baseSha, commitSha], - { - cwd: workspace, - encoding: "utf8", - maxBuffer: 16 * 1024 * 1024, - stdio: ["ignore", "pipe", "pipe"], - }, - ); - const files = output.split("\0").filter(Boolean); - if (files.length > 5000) throw new Error("post-merge risk plan exceeds 5000 changed files"); - if (files.some((file) => file.startsWith("/") || file.split("/").includes(".."))) { - throw new Error("post-merge diff contains an unsafe repository path"); - } - return files; -} - -export function expectedRiskSignalShards( - jobIds: readonly string[], - workflowPath = ".github/workflows/e2e.yaml", -): Record { - const workflow = YAML.parse(fs.readFileSync(workflowPath, "utf8")) as unknown; - const jobs = isObjectRecord(workflow) && isObjectRecord(workflow.jobs) ? workflow.jobs : {}; - return Object.fromEntries( - jobIds.map((jobId) => { - const job = isObjectRecord(jobs[jobId]) ? jobs[jobId] : {}; - const strategy = isObjectRecord(job.strategy) ? job.strategy : {}; - const matrix = isObjectRecord(strategy.matrix) ? strategy.matrix : null; - let shards = ["default"]; - if (matrix) { - const keys = Object.keys(matrix); - if (keys.length === 1 && Array.isArray(matrix.agent)) { - shards = matrix.agent.filter((value): value is string => typeof value === "string"); - if (shards.length !== matrix.agent.length) { - throw new Error(`${jobId} risk matrix agent values must be strings`); - } - } else if (keys.length === 1 && Array.isArray(matrix.include)) { - shards = matrix.include.map((entry) => { - if (!isObjectRecord(entry) || typeof entry.agent !== "string") { - throw new Error(`${jobId} risk matrix include entries must name an agent`); - } - return entry.agent; - }); - } else { - throw new Error(`${jobId} uses an unsupported risk-evidence matrix`); - } - } - if ( - shards.length === 0 || - new Set(shards).size !== shards.length || - shards.some((shard) => !SHARD_PATTERN.test(shard)) - ) { - throw new Error(`${jobId} risk evidence shards must be unique safe identifiers`); - } - return [jobId, shards]; - }), - ); -} - -export function validateWorkflowDispatchDetails( - value: unknown, - repository: string, -): WorkflowDispatchDetails { - if (!isObjectRecord(value)) { - throw new Error("GitHub returned invalid workflow dispatch details"); - } - const runId = value.workflow_run_id; - if (!Number.isSafeInteger(runId) || (runId as number) < 1) { - throw new Error("GitHub returned an invalid dispatched workflow run id"); - } - const expectedApiUrl = `https://api.github.com/repos/${repository}/actions/runs/${runId}`; - const expectedHtmlUrl = `https://github.com/${repository}/actions/runs/${runId}`; - if (value.run_url !== expectedApiUrl || value.html_url !== expectedHtmlUrl) { - throw new Error("GitHub returned mismatched workflow dispatch URLs"); - } - return value as WorkflowDispatchDetails; -} - -function diagnosticValue(value: unknown): string { - const serialized = JSON.stringify(value) ?? String(value); - return serialized.length > 256 ? `${serialized.slice(0, 253)}...` : serialized; -} - -export function assertCorrelatedWorkflowRun( - child: WorkflowRun, - identity: WorkflowRunIdentity, -): void { - const childRunUrl = `https://github.com/${identity.repository}/actions/runs/${identity.childRunId}`; - const mismatches: string[] = []; - const requireEqual = (field: string, expected: unknown, actual: unknown): void => { - if (actual !== expected) { - mismatches.push( - `${field} expected=${diagnosticValue(expected)} actual=${diagnosticValue(actual)}`, - ); - } - }; - - requireEqual("id", identity.childRunId, child.id); - requireEqual("path", E2E_WORKFLOW_PATH, child.path); - requireEqual("event", "workflow_dispatch", child.event); - requireEqual("html_url", childRunUrl, child.html_url); - requireEqual("display_title", `E2E risk ${identity.correlationId}`, child.display_title); - if (!SHA_PATTERN.test(child.head_sha)) { - mismatches.push( - `head_sha expected="40 lowercase hexadecimal characters" actual=${diagnosticValue(child.head_sha)}`, - ); - } - if (!Number.isSafeInteger(child.workflow_id) || child.workflow_id < 1) { - mismatches.push( - `workflow_id expected="positive safe integer" actual=${diagnosticValue(child.workflow_id)}`, - ); - } - if (mismatches.length > 0) { - throw new Error( - `correlated E2E workflow identity mismatch: ${mismatches.join("; ")}; observed run_name=${diagnosticValue(child.name)} workflow_id=${diagnosticValue(child.workflow_id)}`, - ); - } -} - -export async function dispatchRiskWorkflow(options: { - repository: string; - token: string; - jobs: readonly string[]; - commitSha: string; - planHash: string; - correlationId: string; -}): Promise { - if ( - !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(options.repository) || - !options.token || - options.jobs.length < 1 || - options.jobs.length > 3 || - new Set(options.jobs).size !== options.jobs.length || - options.jobs.some((job) => !JOB_PATTERN.test(job)) || - !SHA_PATTERN.test(options.commitSha) || - !HASH_PATTERN.test(options.planHash) || - !CORRELATION_PATTERN.test(options.correlationId) - ) { - throw new Error("risk workflow dispatch inputs are invalid"); - } - const details = await githubApi( - `repos/${options.repository}/actions/workflows/${E2E_WORKFLOW}/dispatches`, - options.token, - { - method: "POST", - body: { - ref: "main", - inputs: { - jobs: options.jobs.join(","), - checkout_sha: options.commitSha, - risk_plan_hash: options.planHash, - risk_correlation: options.correlationId, - risk_shadow: "true", - }, - // GitHub REST 2022-11-28 otherwise returns no run identity. - return_run_details: true, - }, - userAgent: "nemoclaw-e2e-risk-gate", - }, - ); - return validateWorkflowDispatchDetails(details, options.repository).workflow_run_id; -} - -async function start(options: { - baseSha: string; - commitSha: string; - planPath: string; - statePath: string; -}): Promise { - const token = process.env.GITHUB_TOKEN ?? ""; - const repository = process.env.GITHUB_REPOSITORY ?? ""; - if (!token || !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository)) { - throw new Error("GITHUB_TOKEN and a safe GITHUB_REPOSITORY are required"); - } - assertTrustedMainPush({ - eventName: process.env.GITHUB_EVENT_NAME, - ref: process.env.GITHUB_REF, - sha: process.env.GITHUB_SHA, - commitSha: options.commitSha, - }); - - // The inventory and controller are both read from the exact trusted main - // commit. A second copied allowlist would drift without adding a trust - // boundary: compromising this inventory already means compromising main. - const allowedJobs = new Set(readFreeStandingJobsInventory().allowedJobs); - const plan = validateRiskPlan( - buildRiskPlan({ - headSha: options.commitSha, - changedFiles: changedFilesBetween(options.baseSha, options.commitSha), - }), - allowedJobs, - ); - writePrivateRegularFile(options.planPath, `${JSON.stringify(plan, null, 2)}\n`); - - const checkRunId = await createCheck( - repository, - token, - options.commitSha, - "Post-merge risk-selected E2E is being dispatched", - `Plan ${plan.planHash.slice(0, 12)} selected ${plan.automaticJobs.length} automatic job(s).`, - ); - appendOutput("check_id", String(checkRunId)); - try { - const expectedShards = expectedRiskSignalShards(plan.automaticJobs); - if (plan.requiredJobs.length === 0) { - await completeCheck({ repository, checkRunId }, token, { - conclusion: "success", - title: "No post-merge runtime E2E required", - summary: "The deterministic risk plan matched no runtime regression family.", - }); - appendOutput("dispatched", "false"); - appendOutput("finalized", "true"); - console.log( - `Completed shadow check without dispatch: plan=${plan.planHash.slice(0, 12)} selected_jobs=none`, - ); - return; - } - - const correlationId = randomUUID(); - if (!CORRELATION_PATTERN.test(correlationId)) { - throw new Error("generated correlation id is invalid"); - } - const childRunId = await dispatchRiskWorkflow({ - repository, - token, - jobs: plan.automaticJobs, - commitSha: options.commitSha, - planHash: plan.planHash, - correlationId, - }); - const state: RiskGateState = { - version: 1, - commitSha: options.commitSha, - planHash: plan.planHash, - correlationId, - expectedJobs: plan.automaticJobs, - expectedShards, - requiresManualExpansion: plan.requiresManualExpansion, - }; - const serializedState = `${JSON.stringify(state, null, 2)}\n`; - writePrivateRegularFile(options.statePath, serializedState); - appendOutput("state_hash", sha256(serializedState)); - appendOutput("run_id", String(childRunId)); - appendOutput("dispatched", "true"); - console.log( - `Dispatched risk-selected E2E run ${childRunId}: plan=${plan.planHash.slice(0, 12)} selected_jobs=${plan.automaticJobs.join(",")} url=https://github.com/${repository}/actions/runs/${childRunId}`, - ); - } catch (error) { - const finalized = await completeNeutralAfterControllerError( - { repository, checkRunId }, - token, - "Risk-selected E2E could not be dispatched", - { error }, - ); - if (finalized) appendOutput("finalized", "true"); - throw error; - } -} - -export function findSignalFiles( - root: string, - limits: { - maxDepth: number; - maxEntries: number; - maxSignalFiles: number; - } = DEFAULT_EVIDENCE_LIMITS, -): string[] { - if (!fs.existsSync(root)) return []; - if ( - !Number.isSafeInteger(limits.maxDepth) || - limits.maxDepth < 0 || - !Number.isSafeInteger(limits.maxEntries) || - limits.maxEntries < 1 || - !Number.isSafeInteger(limits.maxSignalFiles) || - limits.maxSignalFiles < 1 - ) { - throw new Error("risk evidence traversal limits are invalid"); - } - const rootStat = fs.lstatSync(root); - if (!rootStat.isDirectory() || rootStat.isSymbolicLink()) { - throw new Error("risk evidence root must be a directory, not a symlink"); - } - const files: string[] = []; - let entriesVisited = 0; - const visit = (directory: string, depth: number): void => { - const handle = fs.opendirSync(directory); - try { - let entry = handle.readSync(); - while (entry !== null) { - entriesVisited += 1; - if (entriesVisited > limits.maxEntries) { - throw new Error("risk evidence exceeds the entry limit"); - } - const full = path.join(directory, entry.name); - if (entry.isSymbolicLink()) throw new Error("risk evidence must not contain symlinks"); - if (entry.isDirectory()) { - if (depth >= limits.maxDepth) throw new Error("risk evidence exceeds the depth limit"); - visit(full, depth + 1); - } else if (entry.isFile() && entry.name === "risk-signal.json") { - files.push(full); - if (files.length > limits.maxSignalFiles) { - throw new Error("risk evidence exceeds the signal-file limit"); - } - } - entry = handle.readSync(); - } - } finally { - handle.closeSync(); - } - }; - visit(root, 0); - return files.sort((left, right) => left.localeCompare(right)); -} - -export async function finishRiskGate(options: { - statePath: string; - stateHash: string; - evidencePath: string; - checkRunId: number; - childRunId: number; -}): Promise { - const token = process.env.GITHUB_TOKEN ?? ""; - const repository = process.env.GITHUB_REPOSITORY ?? ""; - if (!token || !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository)) { - throw new Error("GITHUB_TOKEN and a safe GITHUB_REPOSITORY are required"); - } - const { checkRunId, childRunId } = options; - const childRunUrl = `https://github.com/${repository}/actions/runs/${childRunId}`; - const context = { repository, checkRunId }; - try { - if (!HASH_PATTERN.test(options.stateHash)) throw new Error("controller state hash is invalid"); - const serializedState = readPrivateRegularFile(options.statePath, { - maxBytes: MAX_PLAN_BYTES, - })!; - if (sha256(serializedState) !== options.stateHash) { - throw new Error("controller state changed after E2E dispatch"); - } - const state = validateRiskGateState(JSON.parse(serializedState)); - const child = await githubApi( - `repos/${repository}/actions/runs/${childRunId}`, - token, - { userAgent: "nemoclaw-e2e-risk-gate" }, - ); - assertCorrelatedWorkflowRun(child, { - childRunId, - correlationId: state.correlationId, - repository, - }); - console.log( - `Verified correlated E2E run ${childRunId}: conclusion=${child.conclusion ?? "none"} url=${childRunUrl}`, - ); - const signals = - child.conclusion === "success" - ? findSignalFiles(options.evidencePath).map((file) => - validateSignal(readRegularJson(file), state), - ) - : []; - const verdict = classifyRiskEvidence({ - workflowConclusion: child.conclusion, - expectedJobs: state.expectedJobs, - expectedShards: state.expectedShards, - signals, - requiresManualExpansion: state.requiresManualExpansion, - }); - await completeCheck(context, token, verdict, childRunUrl); - console.log(`Completed shadow check: conclusion=${verdict.conclusion} title=${verdict.title}`); - } catch (error) { - const finalized = await completeNeutralAfterControllerError( - context, - token, - "Risk-selected E2E evidence could not be verified", - { error, detailsUrl: childRunUrl }, - ); - if (finalized) appendOutput("finalized", "true"); - throw error; - } -} - -async function abandon(checkRunId: number): Promise { - const token = process.env.GITHUB_TOKEN ?? ""; - const repository = process.env.GITHUB_REPOSITORY ?? ""; - if (!token || !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository)) { - throw new Error("GITHUB_TOKEN and a safe GITHUB_REPOSITORY are required"); - } - await completeCheck({ repository, checkRunId }, token, { - conclusion: "neutral", - title: "Risk-selected E2E controller stopped early", - summary: - "The shadow controller stopped before it could produce complete evidence. Inspect the controller workflow for details.", - }); -} - -function reportControllerError(error: unknown): void { - const message = controllerErrorMessage(error); - console.error(message); - if (process.env.GITHUB_ACTIONS === "true") { - const escaped = message.replace(/%/gu, "%25").replace(/\r/gu, "%0D").replace(/\n/gu, "%0A"); - console.error(`::error title=Post-merge E2E risk gate controller failed::${escaped}`); - } -} - -async function main(): Promise { - const command = parseControllerCommand(process.argv.slice(2)); - if (command.mode === "start") { - await start({ - baseSha: command.baseSha, - commitSha: command.commitSha, - planPath: command.planPath, - statePath: command.statePath, - }); - return; - } - if (command.mode === "finish") { - await finishRiskGate({ - statePath: command.statePath, - stateHash: command.stateHash, - evidencePath: command.evidencePath, - checkRunId: command.checkRunId, - childRunId: command.childRunId, - }); - return; - } - if (command.mode === "abandon") { - await abandon(command.checkRunId); - return; - } -} - -if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - main().catch((error: unknown) => { - reportControllerError(error); - process.exit(1); - }); -} diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index 35dd668a698..c44451c0a93 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -6,6 +6,7 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import YAML from "yaml"; +import { RISK_RULES } from "../advisors/risk-plan.mts"; const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); const DEFAULT_WORKFLOW_PATH = join(REPO_ROOT, ".github", "workflows", "e2e.yaml"); @@ -14,6 +15,8 @@ const META_JOBS = new Set(["report-to-pr", "scorecard"]); const FULL_SHA_ACTION = /^[^\s@]+@[0-9a-f]{40}$/u; const GITHUB_SCRIPT_NODE24_ACTION = "actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3"; +const PR_GATE_REPORTER = "test/e2e/risk-signal-reporter.ts"; +const E2E_ARTIFACT_ACTION = "NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@"; const ISSUE_API_REFERENCE = /\bgithub\.rest\.issues\b/u; const ISSUE_MUTATION_BEYOND_COMMENT = /github\.rest\.issues\.(?:addAssignees|addLabels|create|deleteComment|lock|removeAssignees|removeLabel|setLabels|unlock|update|updateComment)\s*\(/u; @@ -44,8 +47,14 @@ type WorkflowJob = { }; export type OperationsWorkflow = { + concurrency?: { + "cancel-in-progress"?: unknown; + group?: unknown; + }; + env?: Record; jobs: Record; permissions?: WorkflowPermissions; + "run-name"?: unknown; on?: { workflow_dispatch?: { inputs?: Record>; @@ -101,6 +110,123 @@ function requireNode24GithubScript(errors: string[], step: WorkflowStep, owner: } } +function validatePrGateDispatch(errors: string[], workflow: OperationsWorkflow): void { + const inputs = workflow.on?.workflow_dispatch?.inputs ?? {}; + for (const name of ["jobs", "pr_number", "checkout_sha", "plan_hash", "correlation_id"]) { + const input = inputs[name]; + if (input?.type !== "string" || input.default !== "") { + errors.push(`workflow_dispatch ${name} must be an optional string with an empty default`); + } + } + const expectedEnvironment = { + NEMOCLAW_E2E_CORRELATION_ID: "${{ inputs.correlation_id }}", + NEMOCLAW_E2E_EXPECTED_SHA: "${{ inputs.checkout_sha }}", + NEMOCLAW_E2E_PLAN_HASH: "${{ inputs.plan_hash }}", + NEMOCLAW_E2E_SHARD: "default", + }; + for (const [name, value] of Object.entries(expectedEnvironment)) { + if (workflow.env?.[name] !== value) { + errors.push(`E2E workflow must bind ${name} to controller metadata`); + } + } + const runName = String(workflow["run-name"] ?? ""); + for (const fragment of ["inputs.checkout_sha", "inputs.pr_number", "inputs.correlation_id"]) { + if (!runName.includes(fragment)) errors.push(`PR E2E run name must include ${fragment}`); + } + const concurrencyGroup = String(workflow.concurrency?.group ?? ""); + if ( + !concurrencyGroup.includes("inputs.checkout_sha") || + !concurrencyGroup.includes("inputs.pr_number") + ) { + errors.push("PR E2E concurrency must be scoped to its pull request"); + } + if (workflow.concurrency?.["cancel-in-progress"] !== "${{ inputs.checkout_sha != '' }}") { + errors.push("PR E2E concurrency must cancel obsolete runs"); + } + + const matrixJob = workflow.jobs["generate-matrix"] ?? {}; + const steps = matrixJob.steps ?? []; + const validationIndex = steps.findIndex((step) => step.name === "Validate controller dispatch"); + const prepareIndex = steps.findIndex((step) => step.name === "Prepare E2E workspace"); + const validation = validationIndex >= 0 ? steps[validationIndex] : {}; + if (validation.if !== "${{ inputs.checkout_sha != '' }}") { + errors.push("Controller validation must be activated only by checkout_sha"); + } + if (validationIndex < 0 || prepareIndex < 0 || validationIndex >= prepareIndex) { + errors.push("Controller validation must run before workspace preparation"); + } + const expectedStepEnvironment = { + CHECKOUT_SHA: "${{ inputs.checkout_sha }}", + JOBS: "${{ inputs.jobs }}", + PLAN_HASH: "${{ inputs.plan_hash }}", + PR_NUMBER: "${{ inputs.pr_number }}", + CORRELATION_ID: "${{ inputs.correlation_id }}", + TARGETS: "${{ inputs.targets }}", + }; + for (const [name, value] of Object.entries(expectedStepEnvironment)) { + if (validation.env?.[name] !== value) { + errors.push(`Controller validation must bind ${name}`); + } + } + const validationScript = String(validation.run ?? ""); + for (const fragment of [ + '"$WORKFLOW_EVENT" == "workflow_dispatch"', + '"$WORKFLOW_REF" == "refs/heads/main"', + '"$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA"', + '"$PR_NUMBER" =~ ^[1-9][0-9]*$', + '[[ -n "$JOBS" && -z "$TARGETS" ]]', + "https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}", + "'.state'", + "'.head.repo.full_name // \"\"'", + "'.head.sha'", + ]) { + if (!validationScript.includes(fragment)) { + errors.push(`Controller validation must retain ${fragment}`); + } + } + + for (const [jobName, job] of Object.entries(workflow.jobs)) { + for (const step of job.steps ?? []) { + if ( + step.uses?.startsWith("actions/checkout@") && + step.with?.ref !== "${{ inputs.checkout_sha || github.sha }}" + ) { + errors.push(`${jobName} checkout must use the selected PR commit`); + } + } + } +} + +function validatePrGateEvidenceProducers(errors: string[], workflow: OperationsWorkflow): void { + const requiredJobs = new Set(RISK_RULES.flatMap((rule) => rule.requiredJobs)); + for (const jobId of requiredJobs) { + const job = workflow.jobs[jobId]; + if (!job) { + errors.push(`Risk-plan job is missing from E2E workflow: ${jobId}`); + continue; + } + if (job.env?.E2E_JOB !== "1" || job.env?.E2E_TARGET_ID !== jobId) { + errors.push(`${jobId} must expose matching E2E job identity`); + } + if (typeof job.env?.E2E_ARTIFACT_DIR !== "string" || !job.env.E2E_ARTIFACT_DIR) { + errors.push(`${jobId} must expose an evidence artifact directory`); + } + const vitestSteps = (job.steps ?? []).filter((step) => + String(step.run ?? "").includes("npx vitest"), + ); + if ( + vitestSteps.length === 0 || + vitestSteps.some((step) => !String(step.run).includes(PR_GATE_REPORTER)) + ) { + errors.push(`${jobId} must attach the risk-signal reporter to every Vitest invocation`); + } + const uploads = (job.steps ?? []).filter((step) => step.uses?.startsWith(E2E_ARTIFACT_ACTION)); + if (uploads.length !== 1 || uploads[0]?.if !== "always()") { + errors.push(`${jobId} must always upload one evidence artifact`); + } + } +} + function validateAggregation(errors: string[], workflow: OperationsWorkflow): void { const executionJobs = Object.keys(workflow.jobs).filter((name) => !META_JOBS.has(name)); const reportNeeds = needs(workflow.jobs["report-to-pr"] ?? {}); @@ -150,7 +276,7 @@ function validateIssueRoutingRetirement(errors: string[], workflow: OperationsWo } if ( job.if !== - "${{ always() && github.event_name == 'workflow_dispatch' && !inputs.risk_shadow }}" + "${{ always() && github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '' }}" ) { errors.push("report-to-pr must run only for manual workflow dispatches"); } @@ -215,7 +341,7 @@ function validateScorecard(errors: string[], workflow: OperationsWorkflow): void const permissions = permissionMap(job.permissions); if ( job.if !== - "${{ always() && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && !inputs.risk_shadow)) }}" + "${{ always() && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '')) }}" ) { errors.push("scorecard must run after scheduled and manual E2E executions"); } @@ -415,6 +541,8 @@ export function validateE2eOperationsWorkflow( advisorPath = DEFAULT_ADVISOR_PATH, ): string[] { const errors: string[] = []; + validatePrGateDispatch(errors, workflow); + validatePrGateEvidenceProducers(errors, workflow); validateAggregation(errors, workflow); validateIssueRoutingRetirement(errors, workflow); validateScorecard(errors, workflow); diff --git a/tools/e2e/pr-e2e-gate.mts b/tools/e2e/pr-e2e-gate.mts new file mode 100755 index 00000000000..8e69cd0b956 --- /dev/null +++ b/tools/e2e/pr-e2e-gate.mts @@ -0,0 +1,1261 @@ +#!/usr/bin/env node + +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createHash, randomUUID } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +import YAML from "yaml"; + +import { githubApi, githubRestPaginated } from "../advisors/github.mts"; +import { parseArgs } from "../advisors/io.mts"; +import { + buildRiskPlan, + RISK_PLAN_VERSION, + type RiskPlan, + riskPlanRequiredJobIds, +} from "../advisors/risk-plan.mts"; +import { readPrivateRegularFile, writePrivateRegularFile } from "./private-file.ts"; +import type { E2eRiskSignal } from "./risk-signal.ts"; +import { readFreeStandingJobsInventory } from "./workflow-boundary.mts"; + +const E2E_WORKFLOW = "e2e.yaml"; +const E2E_WORKFLOW_PATH = `.github/workflows/${E2E_WORKFLOW}`; +const CHECK_NAME = "E2E / PR Gate"; +const USER_AGENT = "nemoclaw-pr-e2e-gate"; +const SHA_PATTERN = /^[a-f0-9]{40}$/u; +const HASH_PATTERN = /^[a-f0-9]{64}$/u; +const REPOSITORY_PATTERN = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u; +const JOB_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]*$/u; +const SHARD_PATTERN = /^(?:default|[A-Za-z0-9][A-Za-z0-9_-]*)$/u; +const CORRELATION_PATTERN = + /^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/u; +const RUN_REASONS = new Set(["passed", "failed", "interrupted"]); +const MAX_PLAN_BYTES = 1024 * 1024; +const MAX_CONTROLLER_ERROR_CHARS = 512; +const MAX_PR_FILES = 3000; +const MAX_ACTIVE_RUN_PAGES = 10; +const EVIDENCE_LIMITS = { + maxDepth: 8, + maxEntries: 4096, +} as const; + +type ControllerPaths = { + planPath: string; + statePath: string; + evidencePath: string; +}; + +export type ControllerCommand = + | ({ + mode: "start"; + headSha: string; + headRepository: string; + headBranch: string; + workflowSha: string; + ciConclusion: string; + } & ControllerPaths) + | ({ + mode: "finish"; + checkRunId: number; + childRunId: number; + stateHash: string; + } & ControllerPaths) + | { mode: "abandon"; checkRunId: number; childRunId?: number } + | { mode: "cancel"; prNumber: number }; + +type CheckConclusion = "success" | "failure"; + +export type PullRequest = { + number: number; + state: string; + changed_files: number; + head: { ref: string; sha: string; repo: { full_name: string } | null }; + base: { sha: string; repo: { full_name: string } }; +}; + +type PullRequestListItem = Omit; + +type PullRequestFile = { filename: string; previous_filename?: string }; + +type WorkflowRun = { + id: number; + name: string; + path: string; + workflow_id: number; + event: string; + head_sha: string; + status: string; + conclusion: string | null; + display_title: string; + html_url: string; +}; + +type WorkflowRunsResponse = { workflow_runs: WorkflowRun[] }; +type CheckRun = { id: number }; +type GitReference = { ref: string; object: { type: string; sha: string } }; + +type WorkflowDispatchDetails = { + workflow_run_id: number; + run_url: string; + html_url: string; +}; + +type WorkflowRunIdentity = { + childRunId: number; + correlationId: string; + prNumber: number; + repository: string; + workflowSha: string; +}; + +export type PrGateState = { + version: 1; + commitSha: string; + workflowSha: string; + planHash: string; + correlationId: string; + prNumber: number; + expectedJobs: string[]; + expectedShards: Record; +}; + +export type PrGateVerdict = { + conclusion: CheckConclusion; + title: string; + summary: string; +}; + +function isObjectRecord(value: unknown): value is Record { + return !!value && typeof value === "object" && !Array.isArray(value); +} + +function requiredArgument(value: string | undefined, name: string): string { + if (!value) throw new Error(`--${name} is required`); + return value; +} + +function parsePositiveId(value: string, name: string): number { + if (!/^[1-9][0-9]*$/u.test(value)) throw new Error(`${name} must be a positive integer`); + const parsed = Number(value); + if (!Number.isSafeInteger(parsed)) throw new Error(`${name} exceeds the safe integer range`); + return parsed; +} + +function parseHash(value: string | undefined, name: string): string { + const parsed = requiredArgument(value, name); + if (!HASH_PATTERN.test(parsed)) throw new Error(`--${name} must be a lowercase SHA-256 hash`); + return parsed; +} + +function sha256(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} + +function assertRepository(value: string, name: string): void { + if (!REPOSITORY_PATTERN.test(value)) throw new Error(`${name} must be an owner/repository name`); +} + +function assertBranch(value: string): void { + if ( + value.length > 255 || + /[\u0000-\u001f\u007f\\]/u.test(value) || + value.startsWith("/") || + value.endsWith("/") || + value.includes("..") || + value.includes("@{") + ) { + throw new Error("head branch is invalid"); + } +} + +function assertRepositoryPath(value: string): void { + if ( + value.length === 0 || + value.length > 4096 || + value.startsWith("/") || + value.includes("\\") || + /[\u0000\r\n]/u.test(value) || + value.split("/").some((part) => part === "." || part === "..") + ) { + throw new Error("pull request files contain an unsafe repository path"); + } +} + +function tokenAndRepository(): { token: string; repository: string } { + const token = process.env.GITHUB_TOKEN ?? ""; + const repository = process.env.GITHUB_REPOSITORY ?? ""; + if (!token) throw new Error("GITHUB_TOKEN is required"); + assertRepository(repository, "GITHUB_REPOSITORY"); + return { token, repository }; +} + +export function privateControllerPaths(workDir: string): ControllerPaths { + const resolved = path.resolve(workDir); + const stat = fs.lstatSync(resolved); + const currentUid = typeof process.getuid === "function" ? process.getuid() : null; + if ( + resolved !== workDir || + !stat.isDirectory() || + stat.isSymbolicLink() || + (stat.mode & 0o077) !== 0 || + (currentUid !== null && stat.uid !== currentUid) + ) { + throw new Error("--work-dir must be an owned private absolute directory"); + } + return { + planPath: path.join(resolved, "risk-plan.json"), + statePath: path.join(resolved, "controller-state.json"), + evidencePath: path.join(resolved, "evidence"), + }; +} + +export function parseControllerCommand(argv: string[]): ControllerCommand { + const args = parseArgs(argv); + if (args.mode === "start") { + return { + mode: "start", + headSha: requiredArgument(args.head, "head"), + headRepository: requiredArgument(args.headRepo, "head-repo"), + headBranch: requiredArgument(args.headBranch, "head-branch"), + workflowSha: requiredArgument(args.workflowSha, "workflow-sha"), + ciConclusion: requiredArgument(args.ciConclusion, "ci-conclusion"), + ...privateControllerPaths(requiredArgument(args.workDir, "work-dir")), + }; + } + if (args.mode === "finish") { + return { + mode: "finish", + ...privateControllerPaths(requiredArgument(args.workDir, "work-dir")), + checkRunId: parsePositiveId(requiredArgument(args.checkId, "check-id"), "--check-id"), + childRunId: parsePositiveId(requiredArgument(args.runId, "run-id"), "--run-id"), + stateHash: parseHash(args.stateHash, "state-hash"), + }; + } + if (args.mode === "abandon") { + return { + mode: "abandon", + checkRunId: parsePositiveId(requiredArgument(args.checkId, "check-id"), "--check-id"), + childRunId: args.runId ? parsePositiveId(args.runId, "--run-id") : undefined, + }; + } + if (args.mode === "cancel") { + return { + mode: "cancel", + prNumber: parsePositiveId(requiredArgument(args.pr, "pr"), "--pr"), + }; + } + throw new Error("--mode must be start, finish, abandon, or cancel"); +} + +function readRegularJson(file: string, maxBytes = MAX_PLAN_BYTES): unknown { + return JSON.parse(readPrivateRegularFile(file, { maxBytes })!); +} + +export function validatePrGateState(value: unknown): PrGateState { + if (!isObjectRecord(value) || value.version !== 1) { + throw new Error("State version is invalid"); + } + if (typeof value.commitSha !== "string" || !SHA_PATTERN.test(value.commitSha)) { + throw new Error("State commit SHA is invalid"); + } + if (typeof value.workflowSha !== "string" || !SHA_PATTERN.test(value.workflowSha)) { + throw new Error("State workflow SHA is invalid"); + } + if (typeof value.planHash !== "string" || !HASH_PATTERN.test(value.planHash)) { + throw new Error("State plan hash is invalid"); + } + if (typeof value.correlationId !== "string" || !CORRELATION_PATTERN.test(value.correlationId)) { + throw new Error("State correlation ID is invalid"); + } + if (!Number.isSafeInteger(value.prNumber) || (value.prNumber as number) < 1) { + throw new Error("State PR number is invalid"); + } + if ( + !Array.isArray(value.expectedJobs) || + value.expectedJobs.length < 1 || + !value.expectedJobs.every((job) => typeof job === "string" && JOB_PATTERN.test(job)) || + new Set(value.expectedJobs).size !== value.expectedJobs.length + ) { + throw new Error("State jobs are invalid"); + } + if (!isObjectRecord(value.expectedShards)) { + throw new Error("State shards are invalid"); + } + const shardJobs = Object.keys(value.expectedShards).sort(); + if (JSON.stringify(shardJobs) !== JSON.stringify([...value.expectedJobs].sort())) { + throw new Error("State shard jobs do not match expected jobs"); + } + for (const job of value.expectedJobs) { + const shards = value.expectedShards[job]; + if ( + !Array.isArray(shards) || + shards.length < 1 || + new Set(shards).size !== shards.length || + !shards.every((shard) => typeof shard === "string" && SHARD_PATTERN.test(shard)) + ) { + throw new Error(`State shards are invalid for ${job}`); + } + } + return value as PrGateState; +} + +export function validateRiskPlan(value: unknown, allowedJobs: ReadonlySet): RiskPlan { + if (!isObjectRecord(value)) throw new Error("risk plan must be an object"); + if (value.version !== RISK_PLAN_VERSION) throw new Error("unsupported risk-plan version"); + if (typeof value.headSha !== "string" || !SHA_PATTERN.test(value.headSha)) { + throw new Error("risk plan headSha must be a lowercase 40-character SHA"); + } + if ( + !Array.isArray(value.changedFiles) || + !value.changedFiles.every((file) => typeof file === "string") + ) { + throw new Error("risk plan changedFiles must be strings"); + } + for (const file of value.changedFiles) assertRepositoryPath(file as string); + const rebuilt = buildRiskPlan({ + headSha: value.headSha, + changedFiles: value.changedFiles as string[], + }); + if (JSON.stringify(value) !== JSON.stringify(rebuilt)) { + throw new Error("risk plan does not match its hash and inputs"); + } + if (!HASH_PATTERN.test(rebuilt.planHash)) throw new Error("risk plan hash is invalid"); + const selectedJobs = riskPlanRequiredJobIds(rebuilt); + if (new Set(selectedJobs).size !== selectedJobs.length) { + throw new Error("risk plan required jobs must be unique"); + } + for (const job of selectedJobs) { + if (!JOB_PATTERN.test(job) || !allowedJobs.has(job)) { + throw new Error(`risk plan names unknown E2E job: ${job}`); + } + } + return rebuilt; +} + +export function validateSignal( + value: unknown, + state: Pick< + PrGateState, + "commitSha" | "planHash" | "correlationId" | "expectedJobs" | "expectedShards" + >, +): E2eRiskSignal { + if (!isObjectRecord(value) || value.version !== 1) { + throw new Error("invalid E2E signal version"); + } + const signal = value as E2eRiskSignal; + if (!state.expectedJobs.includes(signal.jobId)) throw new Error("E2E signal job is unexpected"); + if (!state.expectedShards[signal.jobId]?.includes(signal.shardId)) { + throw new Error("E2E signal shard is unexpected"); + } + if (signal.expectedSha !== state.commitSha) throw new Error("E2E signal SHA mismatch"); + if (signal.testedSha !== state.commitSha) throw new Error("E2E signal tested SHA mismatch"); + if (signal.planHash !== state.planHash) throw new Error("E2E signal plan hash mismatch"); + if (signal.correlationId !== state.correlationId) { + throw new Error("E2E signal correlation mismatch"); + } + for (const key of ["passed", "failed", "skipped", "pending", "unhandledErrors"] as const) { + if (!Number.isSafeInteger(signal[key]) || signal[key] < 0) { + throw new Error(`E2E signal ${key} must be a non-negative integer`); + } + } + if (!RUN_REASONS.has(signal.runReason)) { + throw new Error("E2E signal runReason is invalid"); + } + return signal; +} + +export function classifyPrGateEvidence(options: { + workflowConclusion: string | null; + expectedJobs: readonly string[]; + expectedShards: Readonly>; + signals: readonly E2eRiskSignal[]; +}): PrGateVerdict { + if (options.workflowConclusion !== "success") { + return { + conclusion: "failure", + title: "E2E run did not succeed", + summary: `The run concluded ${options.workflowConclusion ?? "without a result"}.`, + }; + } + const expectedEvidence = options.expectedJobs.flatMap((job) => + (options.expectedShards[job] ?? []).map((shard) => `${job}:${shard}`), + ); + if ( + options.expectedJobs.length === 0 || + options.expectedJobs.some((job) => (options.expectedShards[job]?.length ?? 0) === 0) + ) { + return { + conclusion: "failure", + title: "Evidence policy is incomplete", + summary: "At least one selected job has no configured shard policy.", + }; + } + const byJobShard = new Map(); + for (const signal of options.signals) { + const key = `${signal.jobId}:${signal.shardId}`; + if (byJobShard.has(key)) { + return { + conclusion: "failure", + title: "Duplicate evidence", + summary: `More than one signal was uploaded for ${key}.`, + }; + } + byJobShard.set(key, signal); + } + const missing = expectedEvidence.filter((key) => !byJobShard.has(key)); + if (missing.length > 0) { + return { + conclusion: "failure", + title: "Evidence is missing", + summary: `Missing signals: ${missing.join(", ")}.`, + }; + } + const failed = expectedEvidence.filter((key) => { + const signal = byJobShard.get(key)!; + return signal.failed > 0 || signal.unhandledErrors > 0 || signal.runReason === "failed"; + }); + if (failed.length > 0) { + return { + conclusion: "failure", + title: "Tests failed", + summary: `Failing signals: ${failed.join(", ")}.`, + }; + } + const partial = expectedEvidence.filter((key) => { + const signal = byJobShard.get(key)!; + return ( + signal.passed < 1 || signal.skipped > 0 || signal.pending > 0 || signal.runReason !== "passed" + ); + }); + if (partial.length > 0) { + return { + conclusion: "failure", + title: "Evidence is incomplete", + summary: `Incomplete or skipped signals: ${partial.join(", ")}.`, + }; + } + return { + conclusion: "success", + title: "All selected jobs passed", + summary: "Every expected job shard passed with no skips or pending tests.", + }; +} + +function appendOutput(name: string, value: string): void { + const output = process.env.GITHUB_OUTPUT; + if (!output) return; + if (!/^(?:check_id|dispatched|finalized|run_id|state_hash)$/u.test(name)) { + throw new Error("invalid controller output name"); + } + const validValue = + name === "state_hash" ? HASH_PATTERN.test(value) : /^(?:true|false|[1-9][0-9]*)$/u.test(value); + if (!validValue) throw new Error("invalid controller output value"); + const descriptor = fs.openSync( + output, + fs.constants.O_WRONLY | fs.constants.O_APPEND | (fs.constants.O_NOFOLLOW ?? 0), + ); + try { + if (!fs.fstatSync(descriptor).isFile()) throw new Error("GITHUB_OUTPUT must be a regular file"); + // lgtm[js/network-data-to-file] Values are reduced to a strict single-line allowlist above, + // and the runner-owned output file is opened without following symlinks. + // lgtm[js/http-to-file-access] + fs.writeFileSync(descriptor, `${name}=${value}\n`, "utf8"); + } finally { + fs.closeSync(descriptor); + } +} + +async function createCheck( + repository: string, + token: string, + headSha: string, + title: string, + summary: string, +): Promise { + const check = await githubApi(`repos/${repository}/check-runs`, token, { + method: "POST", + body: { + name: CHECK_NAME, + head_sha: headSha, + status: "in_progress", + output: { title, summary }, + }, + userAgent: USER_AGENT, + }); + if (!Number.isSafeInteger(check.id) || check.id < 1) { + throw new Error("GitHub returned an invalid check id"); + } + return check.id; +} + +async function completeCheck( + context: { repository: string; checkRunId: number }, + token: string, + verdict: PrGateVerdict, + detailsUrl?: string, +): Promise { + await githubApi(`repos/${context.repository}/check-runs/${context.checkRunId}`, token, { + method: "PATCH", + body: { + status: "completed", + conclusion: verdict.conclusion, + completed_at: new Date().toISOString(), + details_url: detailsUrl, + output: { title: verdict.title, summary: verdict.summary }, + }, + userAgent: USER_AGENT, + }); +} + +async function updateRunningCheck( + context: { repository: string; checkRunId: number }, + token: string, + options: { childRunId: number; jobs: readonly string[]; planHash: string }, +): Promise { + const childRunUrl = `https://github.com/${context.repository}/actions/runs/${options.childRunId}`; + await githubApi(`repos/${context.repository}/check-runs/${context.checkRunId}`, token, { + method: "PATCH", + body: { + status: "in_progress", + details_url: childRunUrl, + output: { + title: `Running ${options.jobs.length} E2E ${options.jobs.length === 1 ? "job" : "jobs"}`, + summary: `Risk plan ${options.planHash} selected: ${options.jobs.join(", ")}.`, + }, + }, + userAgent: USER_AGENT, + }); +} + +function controllerErrorMessage(error: unknown): string { + const message = error instanceof Error ? error.message : String(error); + const singleLine = message + .replace(/[\r\n\t]+/gu, " ") + .replace(/\s{2,}/gu, " ") + .trim(); + return singleLine.length > MAX_CONTROLLER_ERROR_CHARS + ? `${singleLine.slice(0, MAX_CONTROLLER_ERROR_CHARS - 3)}...` + : singleLine; +} + +async function completeFailureAfterControllerError( + context: { repository: string; checkRunId: number }, + token: string, + title: string, + options: { error: unknown; detailsUrl?: string }, +): Promise { + const reason = controllerErrorMessage(options.error).replace(/`/gu, "'"); + try { + await completeCheck( + context, + token, + { + conclusion: "failure", + title, + summary: `The controller could not complete the check.\n\nController error: \`${reason}\``, + }, + options.detailsUrl, + ); + return true; + } catch (error) { + console.error(`Failed to close check after controller error: ${controllerErrorMessage(error)}`); + return false; + } +} + +function validatePullRequestIdentity(value: unknown): PullRequestListItem { + if ( + !isObjectRecord(value) || + !Number.isSafeInteger(value.number) || + (value.number as number) < 1 + ) { + throw new Error("GitHub returned an invalid pull request number"); + } + if (value.state !== "open") throw new Error("GitHub returned invalid pull request state"); + if (!isObjectRecord(value.head) || !isObjectRecord(value.base)) { + throw new Error("GitHub returned invalid pull request refs"); + } + const head = value.head; + const base = value.base; + if ( + typeof head.ref !== "string" || + typeof head.sha !== "string" || + !SHA_PATTERN.test(head.sha) || + !isObjectRecord(head.repo) || + typeof head.repo.full_name !== "string" || + !REPOSITORY_PATTERN.test(head.repo.full_name) || + typeof base.sha !== "string" || + !SHA_PATTERN.test(base.sha) || + !isObjectRecord(base.repo) || + typeof base.repo.full_name !== "string" || + !REPOSITORY_PATTERN.test(base.repo.full_name) + ) { + throw new Error("GitHub returned invalid pull request identity"); + } + return value as PullRequestListItem; +} + +function validatePullRequest(value: unknown): PullRequest { + const identity = validatePullRequestIdentity(value); + if (!isObjectRecord(value) || !Number.isSafeInteger(value.changed_files)) { + throw new Error("GitHub returned an invalid pull request changed-file count"); + } + return { ...identity, changed_files: value.changed_files as number }; +} + +function pullIdentity(pull: PullRequestListItem): Record { + return { + number: pull.number, + state: pull.state, + headRef: pull.head.ref, + headSha: pull.head.sha, + headRepository: pull.head.repo?.full_name, + baseSha: pull.base.sha, + baseRepository: pull.base.repo.full_name, + }; +} + +export async function resolvePullRequest(options: { + repository: string; + token: string; + headSha: string; + headRepository: string; + headBranch: string; +}): Promise { + assertRepository(options.repository, "repository"); + assertRepository(options.headRepository, "head repository"); + if (!options.token) throw new Error("GitHub token is required"); + if (!SHA_PATTERN.test(options.headSha)) throw new Error("head SHA is invalid"); + assertBranch(options.headBranch); + const owner = options.headRepository.split("/", 1)[0]!; + const query = encodeURIComponent(`${owner}:${options.headBranch}`); + const response = await githubApi( + `repos/${options.repository}/pulls?state=open&head=${query}&per_page=100`, + options.token, + { userAgent: USER_AGENT }, + ); + if (!Array.isArray(response)) throw new Error("GitHub returned an invalid pull request list"); + const matches = response + .map(validatePullRequestIdentity) + .filter( + (pull) => + pull.head.sha === options.headSha && + pull.head.ref === options.headBranch && + pull.head.repo?.full_name === options.headRepository && + pull.base.repo.full_name === options.repository, + ); + if (matches.length !== 1) { + throw new Error( + `Expected one open pull request for the triggering revision; found ${matches.length}`, + ); + } + const detail = validatePullRequest( + await githubApi( + `repos/${options.repository}/pulls/${matches[0]!.number}`, + options.token, + { + userAgent: USER_AGENT, + }, + ), + ); + if (JSON.stringify(pullIdentity(matches[0]!)) !== JSON.stringify(pullIdentity(detail))) { + throw new Error("Pull request identity changed while its details were being resolved"); + } + return detail; +} + +export async function pullChangedFiles( + repository: string, + pull: PullRequest, + token: string, +): Promise { + assertRepository(repository, "repository"); + if (!token) throw new Error("GitHub token is required"); + if ( + !Number.isSafeInteger(pull.changed_files) || + pull.changed_files < 0 || + pull.changed_files > MAX_PR_FILES + ) { + throw new Error(`Pull request changed-file count must be between 0 and ${MAX_PR_FILES}`); + } + const files = await githubRestPaginated( + `repos/${repository}/pulls/${pull.number}/files`, + token, + MAX_PR_FILES, + ); + if (files.length !== pull.changed_files) { + throw new Error( + `Pull request file listing is incomplete: expected ${pull.changed_files}, received ${files.length}`, + ); + } + const changed: string[] = []; + const seen = new Set(); + for (const entry of files) { + if (!isObjectRecord(entry) || typeof entry.filename !== "string") { + throw new Error("GitHub returned an invalid pull request file entry"); + } + const names = [entry.previous_filename, entry.filename].filter( + (name): name is string => typeof name === "string", + ); + for (const name of names) { + assertRepositoryPath(name); + if (!seen.has(name)) { + seen.add(name); + changed.push(name); + } + } + } + return changed; +} + +function assertPullUnchanged(before: PullRequest, after: PullRequest): void { + if ( + JSON.stringify({ ...pullIdentity(before), changedFiles: before.changed_files }) !== + JSON.stringify({ ...pullIdentity(after), changedFiles: after.changed_files }) + ) { + throw new Error("PR changed during preparation"); + } +} + +export function expectedSignalShards( + jobIds: readonly string[], + workflowPath = ".github/workflows/e2e.yaml", +): Record { + const workflow = YAML.parse(fs.readFileSync(workflowPath, "utf8")) as unknown; + const jobs = isObjectRecord(workflow) && isObjectRecord(workflow.jobs) ? workflow.jobs : {}; + return Object.fromEntries( + jobIds.map((jobId) => { + if (!isObjectRecord(jobs[jobId])) throw new Error(`E2E workflow does not define ${jobId}`); + const job = jobs[jobId]; + const strategy = isObjectRecord(job.strategy) ? job.strategy : {}; + const matrix = isObjectRecord(strategy.matrix) ? strategy.matrix : null; + let shards = ["default"]; + if (matrix) { + const keys = Object.keys(matrix); + if (keys.length === 1 && Array.isArray(matrix.agent)) { + shards = matrix.agent.filter((value): value is string => typeof value === "string"); + if (shards.length !== matrix.agent.length) { + throw new Error(`${jobId} matrix agent values must be strings`); + } + } else if (keys.length === 1 && Array.isArray(matrix.include)) { + shards = matrix.include.map((entry) => { + if (!isObjectRecord(entry) || typeof entry.agent !== "string") { + throw new Error(`${jobId} matrix include entries must name an agent`); + } + return entry.agent; + }); + } else { + throw new Error(`${jobId} uses an unsupported evidence matrix`); + } + } + if ( + shards.length === 0 || + new Set(shards).size !== shards.length || + shards.some((shard) => !SHARD_PATTERN.test(shard)) + ) { + throw new Error(`${jobId} evidence shards must be unique safe identifiers`); + } + return [jobId, shards]; + }), + ); +} + +export function validateWorkflowDispatchDetails( + value: unknown, + repository: string, +): WorkflowDispatchDetails { + if (!isObjectRecord(value)) throw new Error("GitHub returned invalid workflow dispatch details"); + const runId = value.workflow_run_id; + if (!Number.isSafeInteger(runId) || (runId as number) < 1) { + throw new Error("GitHub returned an invalid dispatched workflow run id"); + } + const expectedApiUrl = `https://api.github.com/repos/${repository}/actions/runs/${runId}`; + const expectedHtmlUrl = `https://github.com/${repository}/actions/runs/${runId}`; + if (value.run_url !== expectedApiUrl || value.html_url !== expectedHtmlUrl) { + throw new Error("GitHub returned mismatched workflow dispatch URLs"); + } + return value as WorkflowDispatchDetails; +} + +function diagnosticValue(value: unknown): string { + const serialized = JSON.stringify(value) ?? String(value); + return serialized.length > 256 ? `${serialized.slice(0, 253)}...` : serialized; +} + +export function assertCorrelatedWorkflowRun( + child: WorkflowRun, + identity: WorkflowRunIdentity, +): void { + const childRunUrl = `https://github.com/${identity.repository}/actions/runs/${identity.childRunId}`; + const mismatches: string[] = []; + const requireEqual = (field: string, expected: unknown, actual: unknown): void => { + if (actual !== expected) { + mismatches.push( + `${field} expected=${diagnosticValue(expected)} actual=${diagnosticValue(actual)}`, + ); + } + }; + requireEqual("id", identity.childRunId, child.id); + requireEqual("path", E2E_WORKFLOW_PATH, child.path); + requireEqual("event", "workflow_dispatch", child.event); + requireEqual("html_url", childRunUrl, child.html_url); + requireEqual( + "display_title", + `E2E PR #${identity.prNumber} (${identity.correlationId})`, + child.display_title, + ); + requireEqual("head_sha", identity.workflowSha, child.head_sha); + if (!Number.isSafeInteger(child.workflow_id) || child.workflow_id < 1) { + mismatches.push( + `workflow_id expected="positive safe integer" actual=${diagnosticValue(child.workflow_id)}`, + ); + } + if (mismatches.length > 0) { + throw new Error( + `E2E run identity mismatch: ${mismatches.join("; ")}; observed run_name=${diagnosticValue(child.name)} workflow_id=${diagnosticValue(child.workflow_id)}`, + ); + } +} + +export async function dispatchPrGate(options: { + repository: string; + token: string; + jobs: readonly string[]; + prNumber: number; + commitSha: string; + workflowSha: string; + planHash: string; + correlationId: string; +}): Promise { + assertRepository(options.repository, "repository"); + if ( + !options.token || + options.jobs.length < 1 || + new Set(options.jobs).size !== options.jobs.length || + options.jobs.some((job) => !JOB_PATTERN.test(job)) || + !Number.isSafeInteger(options.prNumber) || + options.prNumber < 1 || + !SHA_PATTERN.test(options.commitSha) || + !SHA_PATTERN.test(options.workflowSha) || + !HASH_PATTERN.test(options.planHash) || + !CORRELATION_PATTERN.test(options.correlationId) + ) { + throw new Error("Controller dispatch inputs are invalid"); + } + const main = await githubApi( + `repos/${options.repository}/git/ref/heads/main`, + options.token, + { userAgent: USER_AGENT }, + ); + if ( + !main || + main.ref !== "refs/heads/main" || + main.object?.type !== "commit" || + main.object.sha !== options.workflowSha + ) { + throw new Error(`main no longer points to workflow commit ${options.workflowSha}`); + } + const details = await githubApi( + `repos/${options.repository}/actions/workflows/${E2E_WORKFLOW}/dispatches`, + options.token, + { + method: "POST", + body: { + ref: "main", + inputs: { + jobs: options.jobs.join(","), + pr_number: String(options.prNumber), + checkout_sha: options.commitSha, + plan_hash: options.planHash, + correlation_id: options.correlationId, + }, + return_run_details: true, + }, + userAgent: USER_AGENT, + }, + ); + return validateWorkflowDispatchDetails(details, options.repository).workflow_run_id; +} + +async function cancelChildRun(repository: string, token: string, runId: number): Promise { + try { + await githubApi(`repos/${repository}/actions/runs/${runId}/cancel`, token, { + method: "POST", + userAgent: USER_AGENT, + }); + } catch (error) { + if (/failed: 409\b/u.test(controllerErrorMessage(error))) return; + throw error; + } +} + +export async function startPrGate( + command: Extract, +): Promise { + const { token, repository } = tokenAndRepository(); + if (!SHA_PATTERN.test(command.headSha)) throw new Error("PR head SHA is invalid"); + if (!SHA_PATTERN.test(command.workflowSha)) throw new Error("workflow SHA is invalid"); + assertRepository(command.headRepository, "PR head repository"); + assertBranch(command.headBranch); + if (command.headRepository !== repository) { + throw new Error("PR branch must be in the base repository"); + } + + const checkRunId = await createCheck( + repository, + token, + command.headSha, + "Evaluating PR commit", + "Validating the PR and selecting E2E jobs.", + ); + appendOutput("check_id", String(checkRunId)); + + let finalized = false; + let childRunId: number | undefined; + try { + if (command.ciConclusion !== "success") { + await completeCheck({ repository, checkRunId }, token, { + conclusion: "failure", + title: "PR CI did not pass", + summary: `CI / Pull Request concluded ${command.ciConclusion || "without a result"}; no run was dispatched.`, + }); + appendOutput("dispatched", "false"); + appendOutput("finalized", "true"); + finalized = true; + throw new Error(`CI / Pull Request concluded ${command.ciConclusion || "without a result"}`); + } + + const pull = await resolvePullRequest({ + repository, + token, + headSha: command.headSha, + headRepository: command.headRepository, + headBranch: command.headBranch, + }); + if (command.headRepository !== repository || pull.head.repo?.full_name !== repository) { + throw new Error("PR branch must be in the base repository"); + } + + const changedFiles = await pullChangedFiles(repository, pull, token); + const allowedJobs = new Set(readFreeStandingJobsInventory().allowedJobs); + const plan = validateRiskPlan( + buildRiskPlan({ headSha: command.headSha, changedFiles }), + allowedJobs, + ); + writePrivateRegularFile(command.planPath, `${JSON.stringify(plan, null, 2)}\n`); + const jobs = riskPlanRequiredJobIds(plan); + const currentPull = await resolvePullRequest({ + repository, + token, + headSha: command.headSha, + headRepository: command.headRepository, + headBranch: command.headBranch, + }); + assertPullUnchanged(pull, currentPull); + if (jobs.length === 0) { + await completeCheck({ repository, checkRunId }, token, { + conclusion: "success", + title: "No E2E jobs selected", + summary: "No changed files matched an E2E risk rule.", + }); + appendOutput("dispatched", "false"); + appendOutput("finalized", "true"); + finalized = true; + console.log(`No run dispatched: pr=${pull.number} plan=${plan.planHash}`); + return; + } + + const expectedShards = expectedSignalShards(jobs); + const correlationId = randomUUID(); + if (!CORRELATION_PATTERN.test(correlationId)) { + throw new Error("generated correlation ID is invalid"); + } + childRunId = await dispatchPrGate({ + repository, + token, + jobs, + prNumber: pull.number, + commitSha: command.headSha, + workflowSha: command.workflowSha, + planHash: plan.planHash, + correlationId, + }); + appendOutput("run_id", String(childRunId)); + const state: PrGateState = { + version: 1, + commitSha: command.headSha, + workflowSha: command.workflowSha, + planHash: plan.planHash, + correlationId, + prNumber: pull.number, + expectedJobs: jobs, + expectedShards, + }; + const serializedState = `${JSON.stringify(state, null, 2)}\n`; + writePrivateRegularFile(command.statePath, serializedState); + await updateRunningCheck({ repository, checkRunId }, token, { + childRunId, + jobs, + planHash: plan.planHash, + }); + appendOutput("state_hash", sha256(serializedState)); + appendOutput("dispatched", "true"); + console.log( + `Run dispatched: pr=${pull.number} run=${childRunId} plan=${plan.planHash} jobs=${jobs.join(",")} url=https://github.com/${repository}/actions/runs/${childRunId}`, + ); + } catch (error) { + let reportedError = error; + if (!finalized && childRunId) { + try { + await cancelChildRun(repository, token, childRunId); + } catch (cancelError) { + reportedError = new Error( + `${controllerErrorMessage(error)}; child cancellation failed: ${controllerErrorMessage(cancelError)}`, + ); + } + } + if (!finalized) { + const closed = await completeFailureAfterControllerError( + { repository, checkRunId }, + token, + "Run could not start", + { error: reportedError }, + ); + if (closed) appendOutput("finalized", "true"); + } + throw reportedError; + } +} + +export function findSignalFiles( + root: string, + limits: { maxDepth: number; maxEntries: number; maxSignalFiles: number }, +): string[] { + if (!fs.existsSync(root)) return []; + if ( + !Number.isSafeInteger(limits.maxDepth) || + limits.maxDepth < 0 || + !Number.isSafeInteger(limits.maxEntries) || + limits.maxEntries < 1 || + !Number.isSafeInteger(limits.maxSignalFiles) || + limits.maxSignalFiles < 1 + ) { + throw new Error("E2E evidence traversal limits are invalid"); + } + const rootStat = fs.lstatSync(root); + if (!rootStat.isDirectory() || rootStat.isSymbolicLink()) { + throw new Error("E2E evidence root must be a directory, not a symlink"); + } + const files: string[] = []; + let entriesVisited = 0; + const visit = (directory: string, depth: number): void => { + const handle = fs.opendirSync(directory); + try { + let entry = handle.readSync(); + while (entry !== null) { + entriesVisited += 1; + if (entriesVisited > limits.maxEntries) { + throw new Error("E2E evidence exceeds the entry limit"); + } + const full = path.join(directory, entry.name); + if (entry.isSymbolicLink()) throw new Error("E2E evidence must not contain symlinks"); + if (entry.isDirectory()) { + if (depth >= limits.maxDepth) throw new Error("E2E evidence exceeds the depth limit"); + visit(full, depth + 1); + } else if (entry.isFile() && entry.name === "risk-signal.json") { + files.push(full); + if (files.length > limits.maxSignalFiles) { + throw new Error("E2E evidence exceeds the signal-file limit"); + } + } + entry = handle.readSync(); + } + } finally { + handle.closeSync(); + } + }; + visit(root, 0); + return files.sort((left, right) => left.localeCompare(right)); +} + +export async function finishPrGate(options: { + statePath: string; + stateHash: string; + evidencePath: string; + checkRunId: number; + childRunId: number; +}): Promise { + const { token, repository } = tokenAndRepository(); + const childRunUrl = `https://github.com/${repository}/actions/runs/${options.childRunId}`; + const context = { repository, checkRunId: options.checkRunId }; + let finalized = false; + try { + if (!HASH_PATTERN.test(options.stateHash)) throw new Error("controller state hash is invalid"); + const serializedState = readPrivateRegularFile(options.statePath, { + maxBytes: MAX_PLAN_BYTES, + })!; + if (sha256(serializedState) !== options.stateHash) { + throw new Error("controller state changed after E2E dispatch"); + } + const state = validatePrGateState(JSON.parse(serializedState)); + const child = await githubApi( + `repos/${repository}/actions/runs/${options.childRunId}`, + token, + { userAgent: USER_AGENT }, + ); + assertCorrelatedWorkflowRun(child, { + childRunId: options.childRunId, + correlationId: state.correlationId, + prNumber: state.prNumber, + repository, + workflowSha: state.workflowSha, + }); + if (child.status !== "completed") { + await cancelChildRun(repository, token, options.childRunId); + console.log( + `Cancelled unfinished run during finalization: run=${options.childRunId} status=${child.status} url=${childRunUrl}`, + ); + } + const workflowConclusion = + child.status === "completed" ? child.conclusion : `unfinished (${child.status})`; + const expectedSignalCount = Object.values(state.expectedShards).reduce( + (total, shards) => total + shards.length, + 0, + ); + const signals = + workflowConclusion === "success" + ? findSignalFiles(options.evidencePath, { + ...EVIDENCE_LIMITS, + maxSignalFiles: expectedSignalCount + 1, + }).map((file) => validateSignal(readRegularJson(file), state)) + : []; + const verdict = classifyPrGateEvidence({ + workflowConclusion, + expectedJobs: state.expectedJobs, + expectedShards: state.expectedShards, + signals, + }); + await completeCheck(context, token, verdict, childRunUrl); + appendOutput("finalized", "true"); + finalized = true; + console.log( + `Run completed: run=${options.childRunId} conclusion=${verdict.conclusion} title=${verdict.title} url=${childRunUrl}`, + ); + if (verdict.conclusion === "failure") throw new Error(verdict.title); + } catch (error) { + if (!finalized) { + const closed = await completeFailureAfterControllerError( + context, + token, + "Evidence could not be verified", + { error, detailsUrl: childRunUrl }, + ); + if (closed) appendOutput("finalized", "true"); + } + throw error; + } +} + +export async function abandonPrGate(checkRunId: number, childRunId?: number): Promise { + const { token, repository } = tokenAndRepository(); + let cancellationError: unknown; + if (childRunId) { + try { + await cancelChildRun(repository, token, childRunId); + } catch (error) { + cancellationError = error; + } + } + const cancellationSummary = cancellationError + ? ` Child cancellation also failed: ${controllerErrorMessage(cancellationError)}.` + : ""; + await completeCheck({ repository, checkRunId }, token, { + conclusion: "failure", + title: "Controller stopped early", + summary: `The controller stopped before it could complete the check.${cancellationSummary}`, + }); + appendOutput("finalized", "true"); + if (cancellationError) throw cancellationError; +} + +export async function cancelPrGate(prNumber: number): Promise { + const { token, repository } = tokenAndRepository(); + if (!Number.isSafeInteger(prNumber) || prNumber < 1) throw new Error("PR number is invalid"); + const titlePrefix = `E2E PR #${prNumber} (`; + const active: WorkflowRun[] = []; + for (let page = 1; page <= MAX_ACTIVE_RUN_PAGES; page += 1) { + const response = await githubApi( + `repos/${repository}/actions/workflows/${E2E_WORKFLOW}/runs?event=workflow_dispatch&per_page=100&page=${page}`, + token, + { userAgent: USER_AGENT }, + ); + if (!response || !Array.isArray(response.workflow_runs)) { + throw new Error("GitHub returned an invalid workflow run list"); + } + active.push( + ...response.workflow_runs.filter( + (run) => run.display_title.startsWith(titlePrefix) && run.status !== "completed", + ), + ); + if (response.workflow_runs.length < 100) break; + if (page === MAX_ACTIVE_RUN_PAGES) { + throw new Error("Run listing exceeded its page limit"); + } + } + for (const run of active) { + if (!Number.isSafeInteger(run.id) || run.id < 1) { + throw new Error("GitHub returned an invalid active run ID"); + } + await cancelChildRun(repository, token, run.id); + console.log( + `Cancelled superseded run: pr=${prNumber} run=${run.id} url=https://github.com/${repository}/actions/runs/${run.id}`, + ); + } + if (active.length === 0) { + console.log(`No active E2E runs found for PR #${prNumber}`); + } + return active.length; +} + +function reportControllerError(error: unknown): void { + const message = controllerErrorMessage(error); + console.error(message); + if (process.env.GITHUB_ACTIONS === "true") { + const escaped = message.replace(/%/gu, "%25").replace(/\r/gu, "%0D").replace(/\n/gu, "%0A"); + console.error(`::error title=Controller failed::${escaped}`); + } +} + +async function main(): Promise { + const command = parseControllerCommand(process.argv.slice(2)); + if (command.mode === "start") { + await startPrGate(command); + return; + } + if (command.mode === "finish") { + await finishPrGate({ + statePath: command.statePath, + stateHash: command.stateHash, + evidencePath: command.evidencePath, + checkRunId: command.checkRunId, + childRunId: command.childRunId, + }); + return; + } + if (command.mode === "abandon") { + await abandonPrGate(command.checkRunId, command.childRunId); + return; + } + await cancelPrGate(command.prNumber); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error: unknown) => { + reportControllerError(error); + process.exit(1); + }); +} diff --git a/tools/e2e-advisor/private-file.ts b/tools/e2e/private-file.ts similarity index 100% rename from tools/e2e-advisor/private-file.ts rename to tools/e2e/private-file.ts diff --git a/tools/e2e-advisor/risk-signal.ts b/tools/e2e/risk-signal.ts similarity index 100% rename from tools/e2e-advisor/risk-signal.ts rename to tools/e2e/risk-signal.ts diff --git a/tools/pr-review-advisor/README.md b/tools/pr-review-advisor/README.md index e4331a08c51..aa297b240f7 100644 --- a/tools/pr-review-advisor/README.md +++ b/tools/pr-review-advisor/README.md @@ -56,9 +56,10 @@ before the failure so later runs and reviewers do not lose substantive review hi The workflow is advisory and must not be configured as a required status check. It uses the deterministic plan as review context but does not run its jobs. E2E Advisor emits the corresponding plan-backed recommendations separately and likewise does not dispatch E2E. Model availability must -not become the authority for whether a pull request can merge. After a commit lands, a separate -model-independent shadow controller rebuilds the plan from the exact `main` push range and runs its -capped automatic subset. That post-merge check does not make PR Review Advisor a merge gate. +not become the authority for whether a pull request can merge. For PRs from this repository, the PR +E2E controller separately rebuilds the plan from GitHub's changed-file list and dispatches every +selected job after `CI / Pull Request` completes. `E2E / PR Gate` does not consume either advisor's +output and does not make PR Review Advisor a merge gate. Required-check status is point-in-time context, not a settled-CI gate. Earlier `PR_REVIEW_ADVISOR_WAIT_*` workflow variables were inert and have been removed; any future waiting @@ -82,7 +83,8 @@ Authors and coding agents should follow the shared [PR CI and Automated Review F - During rollout, non-default advisor lanes may see an older trusted `main` checkout that has the workflow matrix but not the matching model/configurable-comment support. The workflow treats that as trusted-main rollout skew, writes low-confidence skip artifacts in the lane-specific artifact directory, and suppresses that lane's sticky PR comment. Do not run PR-controlled advisor code to bypass this gate; remove the gate only after the trusted `main` implementation always supports the parallel advisor lane and configurable sticky markers. - The checked-in risk plan is deterministic and additive. PR Review Advisor reviews every listed invariant and required job for missing evidence. Both E2E Advisor result normalizers restore any - listed job that a model omits or downgrades. + listed job that a model omits or downgrades. The PR E2E controller separately dispatches + every listed job without consuming either advisor's normalized result. ## Required secret diff --git a/tools/pr-review-advisor/analyze.mts b/tools/pr-review-advisor/analyze.mts index 8e5350dd23d..ce48fb3ea29 100755 --- a/tools/pr-review-advisor/analyze.mts +++ b/tools/pr-review-advisor/analyze.mts @@ -2141,7 +2141,6 @@ function buildReconciliationTurnContext( tier: context.riskPlan.tier, familyIds: context.riskPlan.families.map((family) => family.id), requiredJobIds: context.riskPlan.requiredJobs.map((job) => job.id), - requiresManualExpansion: context.riskPlan.requiresManualExpansion, }, linkedIssues: (context.github?.linkedIssues ?? []).map(({ number, fetchError }) => ({ number, @@ -2173,9 +2172,6 @@ export function buildRiskPlanReviewContext(plan: RiskPlan): Record