From 215c0a63d42c2e1ddd3a481c8553e2d3462bc8ec Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Wed, 8 Jul 2026 00:06:40 -0700 Subject: [PATCH 1/9] feat(ci): add post-merge E2E risk shadow Signed-off-by: Carlos Villela --- .github/workflows/e2e.yaml | 289 +++++--- .github/workflows/main.yaml | 24 + .../post-merge-e2e-risk-gate-shadow.yaml | 102 +++ .github/workflows/pr.yaml | 27 + AGENTS.md | 4 +- CONTRIBUTING.md | 11 + test/e2e-risk-signal-reporter.test.ts | 110 +++ test/e2e/README.md | 34 + test/e2e/docs/README.md | 23 +- test/e2e/risk-signal-reporter.ts | 161 +++++ .../e2e-operations-workflow-boundary.test.ts | 15 + .../post-merge-e2e-risk-gate-workflow.test.ts | 163 +++++ test/post-merge-e2e-risk-gate.test.ts | 241 +++++++ test/pr-workflow-contract.test.ts | 10 + tools/e2e-advisor/README.md | 36 +- tools/e2e-advisor/post-merge-risk-gate.mts | 630 ++++++++++++++++++ tools/e2e-advisor/risk-signal.ts | 18 + tools/e2e/operations-workflow-boundary.mts | 7 +- tools/pr-review-advisor/README.md | 4 +- 19 files changed, 1818 insertions(+), 91 deletions(-) create mode 100644 .github/workflows/post-merge-e2e-risk-gate-shadow.yaml create mode 100644 test/e2e-risk-signal-reporter.test.ts create mode 100644 test/e2e/risk-signal-reporter.ts create mode 100644 test/post-merge-e2e-risk-gate-workflow.test.ts create mode 100644 test/post-merge-e2e-risk-gate.test.ts create mode 100755 tools/e2e-advisor/post-merge-risk-gate.mts create mode 100644 tools/e2e-advisor/risk-signal.ts diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 3b179192075..8f9fd74f5d6 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 name: E2E +run-name: ${{ inputs.risk_correlation != '' && format('E2E risk {0}', inputs.risk_correlation) || format('E2E {0}', github.ref_name) }} on: schedule: @@ -28,14 +29,41 @@ on: required: false default: false type: boolean + checkout_sha: + description: Exact commit to test while keeping the workflow definition on the dispatched ref. + required: false + default: "" + type: string + risk_plan_hash: + description: Deterministic post-merge risk-plan hash for shadow evidence correlation. + required: false + default: "" + type: string + risk_correlation: + description: UUIDv4 correlation id for an exact-commit shadow risk run. + required: false + default: "" + type: string + risk_shadow: + description: Mark this selective run as post-merge risk-gate shadow evidence. + required: false + default: false + type: boolean permissions: contents: read concurrency: - group: e2e-${{ github.ref }}-${{ inputs.targets || 'supported' }}-${{ inputs.jobs || 'all-jobs' }} + group: e2e-${{ github.ref }}-${{ inputs.risk_correlation || inputs.targets || 'supported' }}-${{ inputs.jobs || 'all-jobs' }} cancel-in-progress: false +env: + NEMOCLAW_E2E_EXPECTED_SHA: ${{ inputs.checkout_sha }} + NEMOCLAW_E2E_RISK_PLAN_HASH: ${{ inputs.risk_plan_hash }} + NEMOCLAW_E2E_RISK_CORRELATION: ${{ inputs.risk_correlation }} + NEMOCLAW_E2E_RISK_SHARD: default + NEMOCLAW_E2E_RISK_SHADOW: ${{ inputs.risk_shadow && '1' || '0' }} + jobs: generate-matrix: runs-on: ubuntu-latest @@ -46,8 +74,36 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} + fetch-depth: 0 persist-credentials: false + - name: Validate exact-commit dispatch + if: ${{ inputs.checkout_sha != '' || inputs.risk_plan_hash != '' || inputs.risk_correlation != '' || inputs.risk_shadow }} + env: + CHECKOUT_SHA: ${{ inputs.checkout_sha }} + JOBS: ${{ inputs.jobs }} + PLAN_HASH: ${{ inputs.risk_plan_hash }} + RISK_CORRELATION: ${{ inputs.risk_correlation }} + RISK_SHADOW: ${{ inputs.risk_shadow }} + TARGETS: ${{ inputs.targets }} + WORKFLOW_EVENT: ${{ github.event_name }} + WORKFLOW_REF: ${{ github.ref }} + run: | + set -euo pipefail + [[ "$WORKFLOW_EVENT" == "workflow_dispatch" && "$WORKFLOW_REF" == "refs/heads/main" ]] || { echo "::error::exact-commit runs require a workflow_dispatch from main"; exit 1; } + [[ "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ ]] || { echo "::error::checkout_sha must be a lowercase 40-character SHA"; exit 1; } + [[ "$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]] || { echo "::error::checked-out HEAD does not match checkout_sha"; exit 1; } + git merge-base --is-ancestor "$CHECKOUT_SHA" origin/main || { echo "::error::checkout_sha must already be reachable from main"; exit 1; } + if [[ "$RISK_SHADOW" == "true" ]]; then + [[ "$PLAN_HASH" =~ ^[a-f0-9]{64}$ ]] || { echo "::error::risk_plan_hash must be a lowercase SHA-256"; exit 1; } + [[ "$RISK_CORRELATION" =~ ^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$ ]] || { echo "::error::risk_correlation must be a lowercase UUIDv4"; exit 1; } + [[ -n "$JOBS" && -z "$TARGETS" ]] || { echo "::error::shadow risk runs require selective jobs and forbid targets/fan-out"; exit 1; } + elif [[ -n "$PLAN_HASH" || -n "$RISK_CORRELATION" ]]; then + echo "::error::risk correlation metadata requires risk_shadow=true" + exit 1 + fi + - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 with: @@ -260,6 +316,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false # Keep only the credential-bearing step anchored. Cleanup mappings stay @@ -354,7 +411,7 @@ jobs: TARGET_ID: ${{ matrix.id }} run: | set -euo pipefail - npx vitest run --project e2e-live test/e2e/live/registry-targets.test.ts -t "^${TARGET_ID}$" --silent=false --reporter=default + npx vitest run --project e2e-live test/e2e/live/registry-targets.test.ts -t "^${TARGET_ID}$" --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts # The sanitizer reads raw traces only after checking the workflow-owned # runner-temp path, then writes the timing-only file into upload roots. @@ -470,6 +527,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - name: Prepare E2E workspace @@ -484,7 +542,7 @@ jobs: set -euo pipefail npx vitest run --project e2e-live \ test/e2e/live/openshell-version-pin.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload OpenShell version-pin artifacts if: always() @@ -511,6 +569,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -542,7 +601,7 @@ jobs: "$OPENSHELL_GATEWAY_BIN" --version npx vitest run --project e2e-live \ test/e2e/live/openshell-gateway-auth-source-contract.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload OpenShell gateway auth contract artifacts if: always() @@ -573,6 +632,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -629,7 +689,7 @@ jobs: "$OPENSHELL_BIN" --version npx vitest run --project e2e-live \ test/e2e/live/mcp-bridge.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - id: mcp_artifact_secret_scan name: Scan MCP artifacts for fixture credentials @@ -671,6 +731,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -731,7 +792,7 @@ jobs: "$OPENSHELL_BIN" --version npx vitest run --project e2e-live \ test/e2e/live/mcp-bridge.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - id: mcp_artifact_secret_scan name: Scan MCP artifacts for fixture credentials @@ -765,6 +826,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - name: Prepare E2E workspace @@ -778,7 +840,7 @@ jobs: set -euo pipefail npx vitest run --project e2e-live \ test/e2e/live/onboard-negative-paths.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload onboard negative-paths artifacts if: always() @@ -800,6 +862,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -834,7 +897,7 @@ jobs: "$OPENSHELL_BIN" --version npx vitest run --project e2e-live \ test/e2e/live/skill-agent.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload skill-agent artifacts if: always() @@ -875,6 +938,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -889,7 +953,7 @@ jobs: set -euo pipefail npx vitest run --project e2e-live \ test/e2e/live/openclaw-skill-cli.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload OpenClaw skill CLI artifacts if: always() @@ -917,6 +981,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - name: Prepare E2E workspace @@ -929,7 +994,7 @@ jobs: set -euo pipefail npx vitest run --project e2e-live \ test/e2e/live/docs-validation.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload docs validation artifacts if: always() @@ -948,6 +1013,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - name: Prepare E2E workspace @@ -964,7 +1030,7 @@ jobs: set -euo pipefail npx vitest run --project e2e-live \ test/e2e/live/inference-routing.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload inference routing artifacts if: always() @@ -987,6 +1053,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -1005,7 +1072,7 @@ jobs: set -euo pipefail npx vitest run --project e2e-live \ test/e2e/live/cloud-inference.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload cloud inference artifacts if: always() @@ -1038,6 +1105,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -1061,7 +1129,7 @@ jobs: export OPENSHELL_BIN echo "Using OPENSHELL_BIN=$OPENSHELL_BIN" "$OPENSHELL_BIN" --version - npx vitest run --project e2e-live test/e2e/live/gpu-e2e.test.ts --silent=false --reporter=default + npx vitest run --project e2e-live test/e2e/live/gpu-e2e.test.ts --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload GPU E2E artifacts if: always() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 @@ -1094,6 +1162,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -1120,7 +1189,7 @@ jobs: export OPENSHELL_BIN echo "Using OPENSHELL_BIN=$OPENSHELL_BIN" "$OPENSHELL_BIN" --version - npx vitest run --project e2e-live test/e2e/live/agent-turn-latency.test.ts --silent=false --reporter=default + npx vitest run --project e2e-live test/e2e/live/agent-turn-latency.test.ts --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload agent turn latency artifacts if: always() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 @@ -1149,6 +1218,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -1163,7 +1233,7 @@ jobs: set -euo pipefail export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" export OPENSHELL_BIN="$(command -v openshell || true)" - npx vitest run --project e2e-live test/e2e/live/kimi-inference-compat.test.ts --silent=false --reporter=default + npx vitest run --project e2e-live test/e2e/live/kimi-inference-compat.test.ts --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload Kimi compatibility artifacts if: always() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 @@ -1215,6 +1285,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -1230,7 +1301,7 @@ jobs: set -euo pipefail export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" export OPENSHELL_BIN="$(command -v openshell || true)" - npx vitest run --project e2e-live test/e2e/live/hermes-inference-switch.test.ts --silent=false --reporter=default + npx vitest run --project e2e-live test/e2e/live/hermes-inference-switch.test.ts --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload Hermes inference switch artifacts if: always() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 @@ -1262,6 +1333,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -1290,7 +1362,7 @@ jobs: fi export OPENSHELL_BIN "$OPENSHELL_BIN" --version - npx vitest run --project e2e-live test/e2e/live/brave-search.test.ts --silent=false --reporter=default + npx vitest run --project e2e-live test/e2e/live/brave-search.test.ts --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload Brave search artifacts if: always() @@ -1316,6 +1388,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -1328,7 +1401,7 @@ jobs: - name: Run Ollama auth proxy live Vitest test run: | set -euo pipefail - npx vitest run --project e2e-live test/e2e/live/ollama-auth-proxy.test.ts --silent=false --reporter=default + npx vitest run --project e2e-live test/e2e/live/ollama-auth-proxy.test.ts --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload Ollama auth proxy artifacts if: always() @@ -1358,6 +1431,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -1385,7 +1459,7 @@ jobs: fi export OPENSHELL_BIN "$OPENSHELL_BIN" --version - npx vitest run --project e2e-live test/e2e/live/cron-preflight-inference-local.test.ts --silent=false --reporter=default + npx vitest run --project e2e-live test/e2e/live/cron-preflight-inference-local.test.ts --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload cron preflight inference.local artifacts if: always() @@ -1415,6 +1489,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -1468,7 +1543,7 @@ jobs: "$OPENSHELL_BIN" --version npx vitest run --project e2e-live \ test/e2e/live/issue-4434-tui-unreachable-inference.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: "Upload issue #4434 TUI unreachable inference artifacts" if: always() @@ -1496,6 +1571,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -1514,7 +1590,7 @@ jobs: set -euo pipefail npx vitest run --project e2e-live \ test/e2e/live/credential-sanitization.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload credential sanitization artifacts if: always() @@ -1541,6 +1617,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -1565,7 +1642,7 @@ jobs: set -euo pipefail npx vitest run --project e2e-live \ test/e2e/live/credential-migration.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload credential migration artifacts if: always() @@ -1593,6 +1670,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -1618,7 +1696,7 @@ jobs: export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" npx vitest run --project e2e-live \ test/e2e/live/sessions-agents-cli.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload sessions/agents CLI artifacts if: always() @@ -1651,6 +1729,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -1670,7 +1749,7 @@ jobs: set -euo pipefail npx vitest run --project e2e-live \ test/e2e/live/hermes-slack-e2e.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload Hermes Slack artifacts if: always() @@ -1705,6 +1784,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -1719,7 +1799,7 @@ jobs: set -euo pipefail npx vitest run --project e2e-live \ test/e2e/live/hermes-e2e.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload Hermes live Vitest artifacts if: always() @@ -1752,6 +1832,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -1764,7 +1845,7 @@ jobs: set -euo pipefail npx vitest run --project e2e-live \ test/e2e/live/hermes-gpu-startup.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload Hermes GPU startup artifacts if: always() @@ -1797,6 +1878,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -1811,7 +1893,7 @@ jobs: set -euo pipefail npx vitest run --project e2e-live \ test/e2e/live/hermes-e2e.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload Hermes dashboard live Vitest artifacts if: always() @@ -1841,6 +1923,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -1862,7 +1945,7 @@ jobs: set -euo pipefail npx vitest run --project e2e-live \ test/e2e/live/hermes-discord.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload Hermes Discord artifacts if: always() @@ -1892,6 +1975,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -1939,7 +2023,7 @@ jobs: set -euo pipefail npx vitest run --project e2e-live \ test/e2e/live/network-policy.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload network-policy artifacts if: always() @@ -1969,6 +2053,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -2009,7 +2094,7 @@ jobs: "$OPENSHELL_BIN" --version npx vitest run --project e2e-live \ test/e2e/live/common-egress-agent.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload common-egress agent artifacts if: always() @@ -2038,6 +2123,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -2057,7 +2143,7 @@ jobs: set -euo pipefail npx vitest run --project e2e-live \ test/e2e/live/shields-config.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload shields-config artifacts if: always() @@ -2090,6 +2176,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -2104,7 +2191,7 @@ jobs: set -euo pipefail npx vitest run --project e2e-live \ test/e2e/live/hermes-shields-config.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload Hermes shields-config artifacts if: always() @@ -2130,6 +2217,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -2165,7 +2253,7 @@ jobs: "$OPENSHELL_BIN" --version npx vitest run --project e2e-live \ test/e2e/live/rebuild-openclaw.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload OpenClaw rebuild artifacts if: always() @@ -2200,6 +2288,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -2219,7 +2308,7 @@ jobs: set -euo pipefail npx vitest run --project e2e-live \ test/e2e/live/rebuild-hermes.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload Hermes rebuild artifacts if: always() @@ -2255,6 +2344,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -2273,7 +2363,7 @@ jobs: set -euo pipefail npx vitest run --project e2e-live \ test/e2e/live/rebuild-hermes.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload Hermes stale-base rebuild artifacts if: always() @@ -2302,6 +2392,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -2335,7 +2426,7 @@ jobs: "$OPENSHELL_BIN" --version npx vitest run --project e2e-live \ test/e2e/live/sandbox-rebuild.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload sandbox rebuild artifacts if: always() @@ -2366,6 +2457,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -2399,7 +2491,7 @@ jobs: "$OPENSHELL_BIN" --version npx vitest run --project e2e-live \ test/e2e/live/sandbox-rlimits-connect.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload sandbox rlimit connect artifacts if: always() @@ -2430,6 +2522,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -2444,7 +2537,7 @@ jobs: set -euo pipefail npx vitest run --project e2e-live \ test/e2e/live/overlayfs-autofix.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload overlayfs autofix artifacts if: always() @@ -2474,6 +2567,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -2507,7 +2601,7 @@ jobs: "$OPENSHELL_BIN" --version npx vitest run --project e2e-live \ test/e2e/live/state-backup-restore.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload state backup restore artifacts if: always() @@ -2537,6 +2631,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -2569,7 +2664,7 @@ jobs: "$OPENSHELL_BIN" --version npx vitest run --project e2e-live \ test/e2e/live/upgrade-stale-sandbox.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload upgrade stale sandbox artifacts if: always() @@ -2596,6 +2691,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -2624,7 +2720,7 @@ jobs: "$OPENSHELL_BIN" --version npx vitest run --project e2e-live \ test/e2e/live/double-onboard.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload double-onboard Vitest artifacts if: always() @@ -2659,6 +2755,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -2687,7 +2784,7 @@ jobs: set -euo pipefail npx vitest run --project e2e-live \ test/e2e/live/jetson-nvmap-gpu.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload Jetson nvmap GPU artifacts if: always() @@ -2714,6 +2811,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -2742,7 +2840,7 @@ jobs: "$OPENSHELL_BIN" --version npx vitest run --project e2e-live \ test/e2e/live/concurrent-gateway-ports.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload concurrent gateway ports Vitest artifacts if: always() @@ -2770,6 +2868,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -2796,7 +2895,7 @@ jobs: "$OPENSHELL_BIN" --version npx vitest run --project e2e-live \ test/e2e/live/onboard-resume.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload onboard-resume artifacts if: always() @@ -2825,6 +2924,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -2853,7 +2953,7 @@ jobs: "$OPENSHELL_BIN" --version npx vitest run --project e2e-live \ test/e2e/live/full-e2e.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload full-e2e artifacts if: always() @@ -2895,6 +2995,7 @@ jobs: # set, and NoNewPrivs remain evidence unless an opt-in expectation is set. NEMOCLAW_E2E_EXPECT_NON_ROOT_HOST: "1" NEMOCLAW_E2E_SECURITY_POSTURE: "1" + NEMOCLAW_E2E_RISK_SHARD: ${{ matrix.agent }} NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS: "60" NEMOCLAW_RECREATE_SANDBOX: "1" @@ -2903,6 +3004,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -2921,7 +3023,7 @@ jobs: export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" npx vitest run --project e2e-live \ "${{ matrix.test_file }}" \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload security posture artifacts if: always() @@ -2950,10 +3052,11 @@ jobs: NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" NEMOCLAW_SANDBOX_NAME: "e2e-cloud-onboard" - NEMOCLAW_PUBLIC_INSTALL_REF: "${{ github.sha }}" + NEMOCLAW_PUBLIC_INSTALL_REF: "${{ inputs.checkout_sha || github.sha }}" steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -2988,7 +3091,7 @@ jobs: "$OPENSHELL_BIN" --version npx vitest run --project e2e-live \ test/e2e/live/cloud-onboard.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Build trusted cloud-onboard timing summary if: always() @@ -3050,6 +3153,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -3076,7 +3180,7 @@ jobs: "$OPENSHELL_BIN" --version npx vitest run --project e2e-live \ test/e2e/live/gpu-double-onboard.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload gpu-double-onboard artifacts if: always() @@ -3104,6 +3208,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -3130,7 +3235,7 @@ jobs: "$OPENSHELL_BIN" --version npx vitest run --project e2e-live \ test/e2e/live/onboard-repair.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload onboard-repair artifacts if: always() @@ -3159,6 +3264,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -3187,7 +3293,7 @@ jobs: "$OPENSHELL_BIN" --version npx vitest run --project e2e-live \ test/e2e/live/issue-4462-scope-upgrade-approval.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload issue-4462-scope-upgrade-approval artifacts if: always() @@ -3212,6 +3318,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -3237,7 +3344,7 @@ jobs: set -euo pipefail npx vitest run --project e2e-live \ test/e2e/live/token-rotation.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload token rotation artifacts if: always() @@ -3264,6 +3371,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -3283,7 +3391,7 @@ jobs: set -euo pipefail npx vitest run --project e2e-live \ test/e2e/live/messaging-compatible-endpoint.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload messaging compatible endpoint artifacts if: always() @@ -3312,6 +3420,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -3331,7 +3440,7 @@ jobs: export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" npx vitest run --project e2e-live \ test/e2e/live/openshell-gateway-upgrade.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload OpenShell gateway upgrade artifacts if: always() @@ -3360,6 +3469,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -3387,7 +3497,7 @@ jobs: set -euo pipefail npx vitest run --project e2e-live \ test/e2e/live/messaging-providers.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload messaging providers artifacts if: always() @@ -3420,6 +3530,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -3437,7 +3548,7 @@ jobs: set -euo pipefail npx vitest run --project e2e-live \ test/e2e/live/launchable-smoke.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload launchable smoke artifacts if: always() @@ -3465,6 +3576,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -3482,7 +3594,7 @@ jobs: set -euo pipefail npx vitest run --project e2e-live \ test/e2e/live/model-router-provider-routed-inference.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload Model Router provider-routed inference artifacts if: always() @@ -3511,6 +3623,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -3530,7 +3643,7 @@ jobs: set -euo pipefail npx vitest run --project e2e-live \ test/e2e/live/snapshot-commands.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload snapshot commands artifacts if: always() @@ -3565,6 +3678,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -3595,7 +3709,7 @@ jobs: export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" npx vitest run --project e2e-live \ test/e2e/live/sandbox-operations.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload sandbox operations artifacts if: always() @@ -3624,6 +3738,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -3642,7 +3757,7 @@ jobs: set -euo pipefail npx vitest run --project e2e-live \ test/e2e/live/sandbox-survival.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload sandbox survival artifacts if: always() @@ -3672,6 +3787,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -3688,7 +3804,7 @@ jobs: NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} run: | set -euo pipefail - npx vitest run --project e2e-live test/e2e/live/diagnostics.test.ts --silent=false --reporter=default + npx vitest run --project e2e-live test/e2e/live/diagnostics.test.ts --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload diagnostics artifacts if: always() @@ -3714,6 +3830,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - name: Prepare E2E workspace @@ -3728,7 +3845,7 @@ jobs: set -euo pipefail npx vitest run --project integration \ test/gateway-drift-preflight.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload gateway drift preflight artifacts if: always() @@ -3755,6 +3872,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -3770,7 +3888,7 @@ jobs: set -euo pipefail npx vitest run --project e2e-live \ test/e2e/live/openclaw-tui-chat-correlation.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload OpenClaw TUI chat correlation artifacts if: always() @@ -3817,6 +3935,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -3857,7 +3976,7 @@ jobs: npx vitest run \ --project e2e-live \ test/e2e/live/gateway-guard-recovery.test.ts \ - --reporter=default --silent=false + --reporter=default --reporter=test/e2e/risk-signal-reporter.ts --silent=false - name: Upload artifacts if: always() @@ -3910,6 +4029,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -3929,7 +4049,7 @@ jobs: set -euo pipefail npx vitest run --project e2e-live \ test/e2e/live/openclaw-inference-switch.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload OpenClaw inference switch artifacts if: always() @@ -3968,6 +4088,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -3985,7 +4106,7 @@ jobs: set -euo pipefail npx vitest run --project e2e-live \ test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload Bedrock Runtime compatible Anthropic artifacts if: always() @@ -4015,6 +4136,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - name: Prepare E2E workspace @@ -4032,7 +4154,7 @@ jobs: export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" npx vitest run --project e2e-live \ test/e2e/live/gateway-health-honest.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload gateway health honesty artifacts if: always() @@ -4057,6 +4179,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -4082,7 +4205,7 @@ jobs: fi export OPENSHELL_BIN "$OPENSHELL_BIN" --version - npx vitest run --project e2e-live test/e2e/live/device-auth-health.test.ts --silent=false --reporter=default + npx vitest run --project e2e-live test/e2e/live/device-auth-health.test.ts --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload device auth health artifacts if: always() @@ -4111,6 +4234,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -4151,7 +4275,7 @@ jobs: "$OPENSHELL_BIN" --version npx vitest run --project e2e-live \ test/e2e/live/channels-add-remove.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload channels add/remove artifacts if: always() @@ -4181,6 +4305,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -4217,7 +4342,7 @@ jobs: "$OPENSHELL_BIN" --version npx vitest run --project e2e-live \ test/e2e/live/telegram-injection.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload Telegram injection artifacts if: always() @@ -4244,6 +4369,7 @@ jobs: NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" + NEMOCLAW_E2E_RISK_SHARD: ${{ matrix.agent }} NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" NEMOCLAW_AGENT: ${{ matrix.agent }} @@ -4253,6 +4379,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -4294,7 +4421,7 @@ jobs: "$OPENSHELL_BIN" --version npx vitest run --project e2e-live \ test/e2e/live/channels-stop-start.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload channels stop/start artifacts if: always() @@ -4327,6 +4454,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -4363,7 +4491,7 @@ jobs: "$OPENSHELL_BIN" --version npx vitest run --project e2e-live \ test/e2e/live/openclaw-slack-pairing.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload OpenClaw Slack pairing artifacts if: always() @@ -4392,6 +4520,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -4420,7 +4549,7 @@ jobs: "$OPENSHELL_BIN" --version npx vitest run --project e2e-live \ test/e2e/live/issue-2478-crash-loop-recovery.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: "Upload issue #2478 crash-loop recovery artifacts" if: always() @@ -4450,6 +4579,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -4485,7 +4615,7 @@ jobs: "$OPENSHELL_BIN" --version npx vitest run --project e2e-live \ test/e2e/live/openclaw-discord-pairing.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload OpenClaw Discord pairing artifacts if: always() @@ -4515,6 +4645,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -4561,7 +4692,7 @@ jobs: set -euo pipefail npx vitest run --project e2e-live \ test/e2e/live/tunnel-lifecycle.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload tunnel lifecycle artifacts if: always() @@ -4593,6 +4724,7 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false - *dockerhub-auth @@ -4613,7 +4745,7 @@ jobs: set -euo pipefail npx vitest run --project e2e-live \ test/e2e/live/spark-install.test.ts \ - --silent=false --reporter=default + --silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts - name: Upload Spark install artifacts if: always() @@ -4712,7 +4844,7 @@ jobs: channels-stop-start, spark-install, ] - if: ${{ always() && github.event_name == 'workflow_dispatch' }} + if: ${{ always() && github.event_name == 'workflow_dispatch' && !inputs.risk_shadow }} permissions: # The issue-comment endpoint accepts pull request write permission for PR comments. # Keep issues: write absent so this job cannot restore general issue routing. @@ -4915,7 +5047,7 @@ jobs: scorecard: runs-on: ubuntu-latest needs: *e2e-result-jobs - if: ${{ always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }} + if: ${{ always() && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && !inputs.risk_shadow)) }} permissions: actions: read contents: read @@ -4923,6 +5055,7 @@ jobs: - name: Checkout scorecard builders uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: + ref: ${{ inputs.checkout_sha || github.sha }} persist-credentials: false sparse-checkout: | ci/onboard-performance-budget.json diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 99d95b663c6..9dc2c207c8b 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -151,6 +151,27 @@ jobs: - name: Run plugin coverage uses: ./.github/actions/ci-plugin-coverage + e2e-support: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + with: + node-version: "22" + cache: npm + + - name: Install E2E support dependencies + run: npm ci --ignore-scripts + + - name: Run E2E support tests + run: npx vitest run --project e2e-support + test-e2e-ollama-proxy: runs-on: ubuntu-latest timeout-minutes: 5 @@ -176,6 +197,7 @@ jobs: - real-openclaw-dist-harness - cli-tests - plugin-tests + - e2e-support - test-e2e-ollama-proxy if: always() runs-on: ubuntu-latest @@ -189,6 +211,7 @@ jobs: REAL_OPENCLAW_DIST_HARNESS_RESULT: ${{ needs['real-openclaw-dist-harness'].result }} CLI_TESTS_RESULT: ${{ needs['cli-tests'].result }} PLUGIN_TESTS_RESULT: ${{ needs['plugin-tests'].result }} + E2E_SUPPORT_RESULT: ${{ needs['e2e-support'].result }} E2E_PROXY_RESULT: ${{ needs['test-e2e-ollama-proxy'].result }} run: | set -euo pipefail @@ -208,6 +231,7 @@ jobs: require_success "real-openclaw-dist-harness" "$REAL_OPENCLAW_DIST_HARNESS_RESULT" require_success "cli-tests" "$CLI_TESTS_RESULT" require_success "plugin-tests" "$PLUGIN_TESTS_RESULT" + require_success "e2e-support" "$E2E_SUPPORT_RESULT" require_success "test-e2e-ollama-proxy" "$E2E_PROXY_RESULT" sandbox-images-and-e2e: diff --git a/.github/workflows/post-merge-e2e-risk-gate-shadow.yaml b/.github/workflows/post-merge-e2e-risk-gate-shadow.yaml new file mode 100644 index 00000000000..25238b43926 --- /dev/null +++ b/.github/workflows/post-merge-e2e-risk-gate-shadow.yaml @@ -0,0 +1,102 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: E2E / Post-merge Risk Gate Shadow + +on: + push: + branches: [main] + +permissions: + actions: write + checks: write + contents: read + +concurrency: + group: e2e-post-merge-risk-gate-${{ github.sha }} + cancel-in-progress: false + +jobs: + shadow: + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' }} + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - name: Checkout trusted controller + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: ${{ github.event.after }} + fetch-depth: 0 + persist-credentials: false + + - name: Setup Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: "22" + cache: npm + + - name: Install trusted controller dependencies + run: npm ci --ignore-scripts + + - id: start + name: Build plan and dispatch exact-commit E2E + continue-on-error: true + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + node --experimental-strip-types tools/e2e-advisor/post-merge-risk-gate.mts + --mode start + --base "${{ github.event.before }}" + --commit "${{ github.event.after }}" + --plan /tmp/post-merge-risk-plan.json + --state /tmp/e2e-risk-gate-state.json + + - name: Upload post-merge risk plan + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: post-merge-risk-plan-${{ github.event.after }} + path: /tmp/post-merge-risk-plan.json + if-no-files-found: ignore + retention-days: 14 + + - name: Close shadow check after controller startup failure + if: ${{ always() && steps.start.outputs.check_id != '' && steps.start.outputs.dispatched != 'true' && steps.start.outputs.finalized != 'true' }} + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + node --experimental-strip-types tools/e2e-advisor/post-merge-risk-gate.mts + --mode abandon + --check-id "${{ steps.start.outputs.check_id }}" + + - name: Propagate controller startup failure + if: ${{ steps.start.outcome == 'failure' }} + run: exit 1 + + - name: Wait for correlated E2E run + if: ${{ steps.start.outputs.dispatched == 'true' }} + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + RUN_ID: ${{ steps.start.outputs.run_id }} + run: >- + timeout --signal=TERM --kill-after=30s 100m + gh run watch "$RUN_ID" --repo "$GITHUB_REPOSITORY" --exit-status + + - name: Download correlated E2E evidence + if: ${{ always() && steps.start.outputs.dispatched == 'true' }} + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + RUN_ID: ${{ steps.start.outputs.run_id }} + run: gh run download "$RUN_ID" --repo "$GITHUB_REPOSITORY" --dir /tmp/e2e-risk-evidence + + - name: Complete exact-commit shadow check + if: ${{ always() && steps.start.outputs.dispatched == 'true' }} + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + node --experimental-strip-types tools/e2e-advisor/post-merge-risk-gate.mts + --mode finish + --state /tmp/e2e-risk-gate-state.json + --evidence /tmp/e2e-risk-evidence diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 5faaf546f9a..54b29582d23 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -308,6 +308,29 @@ jobs: - name: Run plugin coverage uses: ./.trusted-ci-actions/.github/actions/ci-plugin-coverage + e2e-support: + needs: changes + if: needs.changes.outputs.code == 'true' + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + with: + node-version: "22" + cache: npm + + - name: Install E2E support dependencies + run: npm ci --ignore-scripts + + - name: Run E2E support tests + run: npx vitest run --project e2e-support + test-e2e-ollama-proxy: needs: changes if: needs.changes.outputs.code == 'true' @@ -335,6 +358,7 @@ jobs: - installer-integration - cli-tests - plugin-tests + - e2e-support - test-e2e-ollama-proxy if: always() runs-on: ubuntu-latest @@ -350,6 +374,7 @@ jobs: INSTALLER_INTEGRATION_RESULT: ${{ needs['installer-integration'].result }} CLI_TESTS_RESULT: ${{ needs['cli-tests'].result }} PLUGIN_TESTS_RESULT: ${{ needs['plugin-tests'].result }} + E2E_SUPPORT_RESULT: ${{ needs['e2e-support'].result }} E2E_PROXY_RESULT: ${{ needs['test-e2e-ollama-proxy'].result }} run: | set -euo pipefail @@ -384,6 +409,7 @@ jobs: require_success "installer-integration" "$INSTALLER_INTEGRATION_RESULT" require_success "cli-tests" "$CLI_TESTS_RESULT" require_success "plugin-tests" "$PLUGIN_TESTS_RESULT" + require_success "e2e-support" "$E2E_SUPPORT_RESULT" require_success "test-e2e-ollama-proxy" "$E2E_PROXY_RESULT" else require_success "docs-only-checks" "$DOCS_ONLY_RESULT" @@ -392,6 +418,7 @@ jobs: allow_success_or_skipped "installer-integration" "$INSTALLER_INTEGRATION_RESULT" allow_success_or_skipped "cli-tests" "$CLI_TESTS_RESULT" allow_success_or_skipped "plugin-tests" "$PLUGIN_TESTS_RESULT" + allow_success_or_skipped "e2e-support" "$E2E_SUPPORT_RESULT" allow_success_or_skipped "test-e2e-ollama-proxy" "$E2E_PROXY_RESULT" fi diff --git a/AGENTS.md b/AGENTS.md index 4d99c7037a2..6311369e914 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -52,6 +52,7 @@ Package-specific guides: | Run fast source tests | `npm run test:fast` | | Run integration tests | `npm run test:integration` | | Run package contracts | `npm run test:package` | +| Run E2E support tests | `npx vitest run --project e2e-support` | | Run live E2E targets | `npm run test:live-e2e` | | Run plugin tests | `cd nemoclaw && npm test` | | Run repo-wide pre-commit and coverage checks | `npm run check` | @@ -81,7 +82,8 @@ Tests are organized into disjoint Vitest projects defined in `vitest.config.ts`: 3. **`installer-integration`** — installer tests that spawn real `install.sh` processes 4. **`package-contract`** — `test/package-contract/**/*.test.ts` — the only non-live lane that imports compiled CLI/plugin artifacts 5. **`plugin`** — `nemoclaw/src/**/*.test.ts` — plugin unit tests co-located with source -6. **`e2e-support`** — fast tests for the E2E fixture/support layer +6. **`e2e-support`** — fast tests for the E2E fixture/support layer; this project runs in the + aggregate checks for code-changing PRs and code-changing pushes to `main` 7. **`e2e-live`** — opt-in live targets that mutate real external state 8. **`e2e-branch-validation`** — opt-in validation on an ephemeral Brev instance diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index afa21ac57bf..eb41ec922fc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -185,6 +185,17 @@ These are the primary npm scripts for day-to-day development: | `npm run docs:preview:watch` | Publish branch-based Fern previews when docs files change | | `npm run docs:deps` | Print the pinned Fern CLI version used by docs commands | +The `e2e-support` Vitest project is part of the aggregate checks for code-changing pull requests +and code-changing pushes to `main`. Run it directly when you change E2E fixtures, support helpers, +registries, or workflow boundary checks: + +```bash +npx vitest run --project e2e-support +``` + +This project is fast and does not run live targets. Live E2E remains opt-in through +`npm run test:live-e2e` or the applicable GitHub Actions workflow. + ### Test Titles as Behavioral Documentation Write `describe` and `it` titles so the Vitest tree reads as behavioral documentation. Start test diff --git a/test/e2e-risk-signal-reporter.test.ts b/test/e2e-risk-signal-reporter.test.ts new file mode 100644 index 00000000000..a662bba4038 --- /dev/null +++ b/test/e2e-risk-signal-reporter.test.ts @@ -0,0 +1,110 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; +import type { TestModule } from "vitest/node"; +import { + configuredEnvironment, + RISK_SIGNAL_FILE, + type RiskSignalEnvironment, + writeRiskSignal, +} from "./e2e/risk-signal-reporter.ts"; + +const EXPECTED_SHA = "a".repeat(40); +const PLAN_HASH = "b".repeat(64); +const CORRELATION_ID = "12345678-1234-4123-8123-123456789abc"; + +function moduleWithStates(states: Array<"passed" | "failed" | "skipped" | "pending">): TestModule { + return { + children: { + *allTests() { + for (const state of states) yield { result: () => ({ state }) }; + }, + }, + } as unknown as TestModule; +} + +function environment(artifactDir: string): RiskSignalEnvironment { + return { + artifactDir, + jobId: "onboard-resume", + shardId: "default", + expectedSha: EXPECTED_SHA, + testedSha: EXPECTED_SHA, + planHash: PLAN_HASH, + correlationId: CORRELATION_ID, + }; +} + +describe("E2E risk signal reporter", () => { + it("stays disabled outside shadow runs", () => { + expect(configuredEnvironment({})).toBeNull(); + }); + + it("fails closed when shadow metadata is incomplete", () => { + expect(() => configuredEnvironment({ NEMOCLAW_E2E_RISK_SHADOW: "1" })).toThrow( + /E2E_ARTIFACT_DIR/u, + ); + }); + + it("attests the checked-out HEAD instead of echoing only the expected SHA", () => { + const env = { + E2E_ARTIFACT_DIR: "/tmp/e2e-risk-signal-test", + E2E_TARGET_ID: "onboard-resume", + GITHUB_WORKSPACE: "/workspace", + NEMOCLAW_E2E_EXPECTED_SHA: EXPECTED_SHA, + NEMOCLAW_E2E_RISK_PLAN_HASH: PLAN_HASH, + NEMOCLAW_E2E_RISK_CORRELATION: CORRELATION_ID, + NEMOCLAW_E2E_RISK_SHARD: "default", + NEMOCLAW_E2E_RISK_SHADOW: "1", + }; + + expect(configuredEnvironment(env, () => EXPECTED_SHA)?.testedSha).toBe(EXPECTED_SHA); + expect(() => configuredEnvironment(env, () => "c".repeat(40))).toThrow(/checked-out HEAD/u); + }); + + it("writes exact-SHA pass, failure, skip, and pending counts", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-risk-signal-")); + try { + const signal = writeRiskSignal( + environment(dir), + [moduleWithStates(["passed", "failed", "skipped", "pending"])], + [new Error("unhandled")], + "failed", + ); + + expect(signal).toMatchObject({ + passed: 1, + failed: 1, + skipped: 1, + pending: 1, + unhandledErrors: 1, + runReason: "failed", + }); + expect(fs.statSync(path.join(dir, RISK_SIGNAL_FILE)).mode & 0o777).toBe(0o600); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it("aggregates multiple Vitest invocations for one workflow job", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-risk-signal-")); + try { + writeRiskSignal(environment(dir), [moduleWithStates(["passed"])], [], "passed"); + const signal = writeRiskSignal( + environment(dir), + [moduleWithStates(["passed", "skipped"])], + [], + "passed", + ); + + expect(signal).toMatchObject({ passed: 2, skipped: 1, failed: 0, pending: 0 }); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/test/e2e/README.md b/test/e2e/README.md index 825b80d6d4f..7847e422253 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -10,6 +10,9 @@ before those targets run; local runners must provide it themselves. - `.github/workflows/e2e.yaml` is the scheduled and manually dispatchable live target workflow. +- `.github/workflows/post-merge-e2e-risk-gate-shadow.yaml` is the trusted post-merge + controller that selects and dispatches a bounded exact-commit subset after + pushes to `main`. - `.github/workflows/e2e-branch-validation.yaml` provisions Brev instances and runs focused E2E targets from source on a clean machine. - Platform workflows such as macOS, WSL, Ollama proxy, sandbox image, and @@ -53,6 +56,37 @@ artifact so baseline aggregation stays stable. Older issue references to Vitest target artifacts under `e2e-artifacts/vitest/` map to this consolidated `e2e-artifacts/live/` registry-target artifact layout. +## Post-merge risk shadow + +Every push to the `main` branch of `NVIDIA/NemoClaw` starts a model-independent +shadow controller. It builds the deterministic risk plan from the exact +`github.event.before` and `github.event.after` range after confirming that its checkout +matches the pushed commit. If the plan matches runtime regression families, +the controller dispatches at most three `automaticJobs` through `e2e.yaml`. +The workflow definition stays on `main`, while every E2E checkout uses the +exact merged commit supplied through `checkout_sha`. + +Before E2E preparation or selected jobs can use repository secrets, +`e2e.yaml` verifies the requested SHA, confirms that checked-out `HEAD` matches +it, proves that the commit is reachable from `main`, and accepts only selective +job dispatch with valid plan and correlation metadata. The shadow-only Vitest +reporter then writes a `risk-signal.json` for each selected job and matrix +shard. Each signal binds the observed checkout SHA, expected SHA, plan hash, +correlation ID, and pass, failure, skip, pending, and unhandled-error counts. +The controller retains `post-merge-risk-plan-` for 14 days, while each +signal travels in the selected job's existing E2E artifact. + +The controller reports `E2E / Post-merge Risk Gate (shadow)` on the merged +commit. It reports success only when every expected shard produces a complete, +unskipped pass and the three-job cap did not omit required jobs. Selected E2E +workflow or exact-SHA test failures report failure. Missing, partial, skipped, +ambiguous, or manual-expansion evidence reports neutral. A plan with no matched +runtime risk reports success without dispatching live E2E. This shadow check runs +after merge and is not a required PR check. It disables PR comments and the +scheduled/manual scorecard, including scorecard Slack reporting. +Controller or evidence-verification errors close an already-created check as +neutral so incomplete evidence cannot appear successful. + ## Onboard performance budget The scheduled/manual scorecard evaluates the trusted `cloud-onboard` timing diff --git a/test/e2e/docs/README.md b/test/e2e/docs/README.md index 05594e21198..4e086fd19c2 100644 --- a/test/e2e/docs/README.md +++ b/test/e2e/docs/README.md @@ -99,9 +99,16 @@ test/e2e/ ## CI Entry Points - `tools/advisors/risk-plan.mts` is the small deterministic selection policy - shared by PR Review Advisor and E2E Advisor. It maps changed runtime surfaces - to invariant families and canonical `e2e.yaml` jobs; it is not a second test - runner or migration-status ledger. + shared by PR Review Advisor, E2E Advisor, and the model-independent post-merge + shadow controller. It maps changed runtime surfaces to invariant families and + canonical `e2e.yaml` jobs; it is not a second test runner or migration-status + ledger. + +- `.github/workflows/post-merge-e2e-risk-gate-shadow.yaml` runs only for trusted pushes to + `main`. `tools/e2e-advisor/post-merge-risk-gate.mts` builds a plan from the exact + before/after SHAs, dispatches at most three automatic jobs, and validates + `risk-signal.json` evidence for every expected job and matrix shard. The + resulting check is post-merge shadow evidence, not a required PR gate. - `.github/workflows/e2e.yaml` runs selected or all supported live E2E targets and uploads an explicit artifact allowlist with @@ -114,11 +121,17 @@ test/e2e/ These per-target timing summaries are artifact evidence only. The Slack and GitHub scorecard timing comparison remains scoped to the dedicated `cloud-onboard` artifact. + Exact-commit shadow dispatches validate that the requested checkout is + reachable from `main` before preparation, attach + `test/e2e/risk-signal-reporter.ts` to live Vitest invocations, and suppress + PR reporting and scorecards. - `.github/workflows/e2e-branch-validation.yaml`, `macos-e2e.yaml`, `wsl-e2e.yaml`, `ollama-proxy-e2e.yaml`, and `regression-e2e.yaml` call focused E2E targets directly for their E2E coverage. -- `vitest.config.ts` contains `e2e-support` for fast fixture/support - tests and `e2e-live` for opt-in live target execution. +- `vitest.config.ts` contains `e2e-support` for fast fixture/support tests and + `e2e-live` for opt-in live target execution. The PR and `main` aggregate + checks require `e2e-support` for code changes; the project never opts into + live targets. ## Migration Tracking diff --git a/test/e2e/risk-signal-reporter.ts b/test/e2e/risk-signal-reporter.ts new file mode 100644 index 00000000000..66a3627a77e --- /dev/null +++ b/test/e2e/risk-signal-reporter.ts @@ -0,0 +1,161 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; + +import type { TestModule } from "vitest/node"; +import type { Reporter, TestRunEndReason } from "vitest/reporters"; +import type { E2eRiskSignal } from "../../tools/e2e-advisor/risk-signal.ts"; + +export const RISK_SIGNAL_FILE = "risk-signal.json"; + +export type RiskSignalEnvironment = { + artifactDir: string; + jobId: string; + shardId: string; + expectedSha: string; + testedSha: string; + planHash: string; + correlationId: string; +}; + +const SHA_PATTERN = /^[a-f0-9]{40}$/u; +const HASH_PATTERN = /^[a-f0-9]{64}$/u; +const CORRELATION_PATTERN = + /^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/u; +const JOB_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]*$/u; +const SHARD_PATTERN = /^(?:default|[A-Za-z0-9][A-Za-z0-9_-]*)$/u; + +function checkedOutSha(workspace: string): string { + return execFileSync("git", ["rev-parse", "--verify", "HEAD"], { + cwd: workspace, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }).trim(); +} + +export function configuredEnvironment( + env: NodeJS.ProcessEnv, + resolveHead: (workspace: string) => string = checkedOutSha, +): RiskSignalEnvironment | null { + if (env.NEMOCLAW_E2E_RISK_SHADOW !== "1") return null; + const values = { + artifactDir: env.E2E_ARTIFACT_DIR ?? "", + jobId: env.E2E_TARGET_ID ?? "", + shardId: env.NEMOCLAW_E2E_RISK_SHARD ?? "", + expectedSha: env.NEMOCLAW_E2E_EXPECTED_SHA ?? "", + planHash: env.NEMOCLAW_E2E_RISK_PLAN_HASH ?? "", + correlationId: env.NEMOCLAW_E2E_RISK_CORRELATION ?? "", + }; + if (!values.artifactDir) throw new Error("risk signal requires E2E_ARTIFACT_DIR"); + if (!JOB_PATTERN.test(values.jobId)) throw new Error("risk signal requires a safe E2E_TARGET_ID"); + if (!SHARD_PATTERN.test(values.shardId)) { + throw new Error("risk signal requires a safe shard id"); + } + if (!SHA_PATTERN.test(values.expectedSha)) { + throw new Error("risk signal requires a 40-character lowercase expected SHA"); + } + if (!HASH_PATTERN.test(values.planHash)) { + throw new Error("risk signal requires a 64-character lowercase plan hash"); + } + if (!CORRELATION_PATTERN.test(values.correlationId)) { + throw new Error("risk signal requires a lowercase UUIDv4 correlation id"); + } + const testedSha = resolveHead(env.GITHUB_WORKSPACE ?? process.cwd()); + if (!SHA_PATTERN.test(testedSha) || testedSha !== values.expectedSha) { + throw new Error("risk signal checked-out HEAD does not match the expected SHA"); + } + return { ...values, testedSha }; +} + +function counts(testModules: ReadonlyArray) { + const result = { passed: 0, failed: 0, skipped: 0, pending: 0 }; + for (const module of testModules) { + for (const test of module.children.allTests()) { + result[test.result().state] += 1; + } + } + return result; +} + +function mergeSignal(previous: E2eRiskSignal | null, current: E2eRiskSignal): E2eRiskSignal { + if (!previous) return current; + if ( + previous.version !== current.version || + previous.jobId !== current.jobId || + previous.shardId !== current.shardId || + previous.expectedSha !== current.expectedSha || + previous.testedSha !== current.testedSha || + previous.planHash !== current.planHash || + previous.correlationId !== current.correlationId + ) { + throw new Error("risk signal metadata changed between Vitest invocations"); + } + return { + ...current, + passed: previous.passed + current.passed, + failed: previous.failed + current.failed, + skipped: previous.skipped + current.skipped, + pending: previous.pending + current.pending, + unhandledErrors: previous.unhandledErrors + current.unhandledErrors, + runReason: + previous.runReason === "failed" || current.runReason === "failed" + ? "failed" + : previous.runReason === "interrupted" || current.runReason === "interrupted" + ? "interrupted" + : "passed", + }; +} + +function readPrevious(file: string): E2eRiskSignal | null { + if (!fs.existsSync(file)) return null; + const stat = fs.lstatSync(file); + if (!stat.isFile() || stat.isSymbolicLink()) { + throw new Error("risk signal output must be a regular file"); + } + return JSON.parse(fs.readFileSync(file, "utf8")) as E2eRiskSignal; +} + +export function writeRiskSignal( + environment: RiskSignalEnvironment, + testModules: ReadonlyArray, + unhandledErrors: ReadonlyArray, + runReason: TestRunEndReason, +): E2eRiskSignal { + const signal: E2eRiskSignal = { + version: 1, + jobId: environment.jobId, + shardId: environment.shardId, + expectedSha: environment.expectedSha, + testedSha: environment.testedSha, + planHash: environment.planHash, + correlationId: environment.correlationId, + ...counts(testModules), + unhandledErrors: unhandledErrors.length, + runReason, + }; + fs.mkdirSync(environment.artifactDir, { recursive: true }); + const file = path.join(environment.artifactDir, RISK_SIGNAL_FILE); + const merged = mergeSignal(readPrevious(file), signal); + fs.writeFileSync(file, `${JSON.stringify(merged, null, 2)}\n`, { mode: 0o600 }); + return merged; +} + +export default class E2eRiskSignalReporter implements Reporter { + private readonly environment: RiskSignalEnvironment | null; + + constructor() { + this.environment = configuredEnvironment(process.env); + } + + onTestRunEnd( + testModules: ReadonlyArray, + unhandledErrors: ReadonlyArray, + reason: TestRunEndReason, + ): void { + if (!this.environment) return; + writeRiskSignal(this.environment, testModules, unhandledErrors, reason); + } +} diff --git a/test/e2e/support/e2e-operations-workflow-boundary.test.ts b/test/e2e/support/e2e-operations-workflow-boundary.test.ts index d30f7d6616a..ec616eb535b 100644 --- a/test/e2e/support/e2e-operations-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-operations-workflow-boundary.test.ts @@ -57,6 +57,21 @@ describe("E2E operations workflow boundary", () => { ); }); + it("keeps PR reporting and scorecards disabled for E2E risk shadow runs", () => { + const workflow = readE2eOperationsWorkflow(); + workflow.jobs["report-to-pr"].if = + "${{ always() && github.event_name == 'workflow_dispatch' }}"; + workflow.jobs.scorecard.if = + "${{ always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }}"; + + expect(validateE2eOperationsWorkflow(workflow)).toEqual( + expect.arrayContaining([ + "report-to-pr must run only for manual workflow dispatches", + "scorecard must run after scheduled and manual E2E executions", + ]), + ); + }); + it("rejects restoration of scheduled issue routing or broad issue-write access", () => { const workflow = readE2eOperationsWorkflow(); workflow.permissions = "write-all"; diff --git a/test/post-merge-e2e-risk-gate-workflow.test.ts b/test/post-merge-e2e-risk-gate-workflow.test.ts new file mode 100644 index 00000000000..a3943145920 --- /dev/null +++ b/test/post-merge-e2e-risk-gate-workflow.test.ts @@ -0,0 +1,163 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { RISK_RULES } from "../tools/advisors/risk-plan.mts"; +import { + readYaml, + type Workflow, + type WorkflowJob, + type WorkflowStep, +} from "./helpers/e2e-workflow-contract.ts"; + +const SHADOW_PATH = ".github/workflows/post-merge-e2e-risk-gate-shadow.yaml"; +const E2E_PATH = ".github/workflows/e2e.yaml"; + +type TriggeredWorkflow = Workflow & { + on?: Record; + permissions?: Record; +}; + +function step(job: WorkflowJob, name: string): WorkflowStep { + const match = job.steps?.find((candidate) => candidate.name === name); + expect(match, `missing workflow step ${name}`).toBeDefined(); + return match!; +} + +function collectStrings(value: unknown): string[] { + if (typeof value === "string") return [value]; + if (Array.isArray(value)) return value.flatMap(collectStrings); + if (value && typeof value === "object") { + return Object.values(value).flatMap(collectStrings); + } + return []; +} + +describe("post-merge E2E risk gate shadow workflow", () => { + it("uses a trusted main-push controller with minimal write permissions", () => { + const workflow = readYaml(SHADOW_PATH); + const job = workflow.jobs.shadow; + + expect(workflow.on).toEqual({ + push: { branches: ["main"] }, + }); + expect(workflow.permissions).toEqual({ + actions: "write", + checks: "write", + contents: "read", + }); + expect(job.if).toContain("github.repository == 'NVIDIA/NemoClaw'"); + expect(job.if).toContain("github.ref == 'refs/heads/main'"); + expect(collectStrings(workflow).some((value) => value.includes("${{ secrets."))).toBe(false); + }); + + it("builds the plan from the exact trusted push and bounds child-run waiting", () => { + const workflow = readYaml(SHADOW_PATH); + const job = workflow.jobs.shadow; + const checkout = step(job, "Checkout trusted controller"); + const start = step(job, "Build plan and dispatch exact-commit E2E"); + const wait = step(job, "Wait for correlated E2E run"); + const finish = step(job, "Complete exact-commit shadow check"); + + expect(checkout.with).toMatchObject({ + ref: "${{ github.event.after }}", + "fetch-depth": 0, + "persist-credentials": false, + }); + expect(start.run).toContain("post-merge-risk-gate.mts --mode start"); + expect(start.run).toContain('--base "${{ github.event.before }}"'); + expect(start.run).toContain('--commit "${{ github.event.after }}"'); + expect(wait.run).toContain("timeout --signal=TERM --kill-after=30s 100m"); + expect(finish.if).toContain("always()"); + expect(finish.run).toContain("post-merge-risk-gate.mts --mode finish"); + }); + + it("binds every E2E checkout and test signal to the exact merged commit", () => { + const workflow = readYaml; "run-name"?: string }>( + E2E_PATH, + ); + const allSteps = Object.values(workflow.jobs).flatMap((job) => job.steps ?? []); + const checkouts = allSteps.filter((candidate) => + candidate.uses?.startsWith("actions/checkout@"), + ); + const testCommands = allSteps + .map((candidate) => candidate.run ?? "") + .filter((run) => run.includes("npx vitest run --project e2e-live")); + + expect(workflow["run-name"]).toContain("inputs.risk_correlation"); + expect(workflow.env).toMatchObject({ + NEMOCLAW_E2E_EXPECTED_SHA: "${{ inputs.checkout_sha }}", + NEMOCLAW_E2E_RISK_PLAN_HASH: "${{ inputs.risk_plan_hash }}", + NEMOCLAW_E2E_RISK_CORRELATION: "${{ inputs.risk_correlation }}", + NEMOCLAW_E2E_RISK_SHARD: "default", + }); + expect(checkouts.length).toBeGreaterThan(50); + expect( + checkouts.every( + (checkout) => checkout.with?.ref === "${{ inputs.checkout_sha || github.sha }}", + ), + ).toBe(true); + expect(testCommands.length).toBeGreaterThan(50); + expect( + testCommands.every((run) => run.includes("--reporter=test/e2e/risk-signal-reporter.ts")), + ).toBe(true); + expect(workflow.jobs["cloud-onboard"].env?.NEMOCLAW_PUBLIC_INSTALL_REF).toBe( + "${{ inputs.checkout_sha || github.sha }}", + ); + }); + + it("keeps every deterministic risk job signal-bearing and artifact-backed", () => { + const workflow = readYaml(E2E_PATH); + const requiredJobs = [...new Set(RISK_RULES.flatMap((rule) => rule.requiredJobs))]; + + for (const jobId of requiredJobs) { + const job = workflow.jobs[jobId]; + expect(job, `missing risk-plan job ${jobId}`).toBeDefined(); + expect( + Array.isArray(job.needs) ? job.needs : [job.needs], + `${jobId} must wait for exact-commit validation`, + ).toContain("generate-matrix"); + expect(job.env?.E2E_TARGET_ID, `${jobId} must identify its risk signal`).toBe(jobId); + const liveRuns = (job.steps ?? []) + .map((candidate) => candidate.run ?? "") + .filter((run) => run.includes("--project e2e-live")); + expect(liveRuns.length, `${jobId} must execute a live Vitest target`).toBeGreaterThan(0); + expect( + liveRuns.every((run) => run.includes("--reporter=test/e2e/risk-signal-reporter.ts")), + `${jobId} must write risk evidence for every live Vitest invocation`, + ).toBe(true); + const upload = (job.steps ?? []).find((candidate) => + candidate.uses?.includes("/.github/actions/upload-e2e-artifacts@"), + ); + expect(upload, `${jobId} must upload its risk signal`).toBeDefined(); + expect(upload?.if).toBe("always()"); + } + + expect(workflow.jobs["security-posture"].env?.NEMOCLAW_E2E_RISK_SHARD).toBe( + "${{ matrix.agent }}", + ); + expect(workflow.jobs["channels-stop-start"].env?.NEMOCLAW_E2E_RISK_SHARD).toBe( + "${{ matrix.agent }}", + ); + }); + + it("validates shadow inputs before preparing or executing the selected workspace", () => { + const workflow = readYaml(E2E_PATH); + const steps = workflow.jobs["generate-matrix"].steps ?? []; + const validateIndex = steps.findIndex( + (candidate) => candidate.name === "Validate exact-commit dispatch", + ); + const prepareIndex = steps.findIndex((candidate) => candidate.name === "Prepare E2E workspace"); + const validate = steps[validateIndex]; + + expect(validateIndex).toBeGreaterThan(0); + expect(validateIndex).toBeLessThan(prepareIndex); + expect(validate?.run).toContain("checkout_sha must be a lowercase 40-character SHA"); + expect(validate?.run).toContain('"$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA"'); + expect(validate?.run).toContain('git merge-base --is-ancestor "$CHECKOUT_SHA" origin/main'); + expect(validate?.run).toContain("forbid targets/fan-out"); + expect(steps[0]?.with?.["fetch-depth"]).toBe(0); + expect(workflow.jobs["report-to-pr"].if).toContain("!inputs.risk_shadow"); + expect(workflow.jobs.scorecard.if).toContain("!inputs.risk_shadow"); + }); +}); diff --git a/test/post-merge-e2e-risk-gate.test.ts b/test/post-merge-e2e-risk-gate.test.ts new file mode 100644 index 00000000000..7b4a7428f2c --- /dev/null +++ b/test/post-merge-e2e-risk-gate.test.ts @@ -0,0 +1,241 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; +import { buildRiskPlan, RISK_RULES } from "../tools/advisors/risk-plan.mts"; +import { + assertTrustedMainPush, + changedFilesBetween, + classifyRiskEvidence, + expectedRiskSignalShards, + type RiskGateState, + validateRiskPlan, + validateSignal, +} from "../tools/e2e-advisor/post-merge-risk-gate.mts"; +import type { E2eRiskSignal } from "../tools/e2e-advisor/risk-signal.ts"; + +const HEAD_SHA = "a".repeat(40); +const ALLOWED = new Set(["onboard-repair", "onboard-resume"]); + +function state(): RiskGateState { + return { + version: 1, + repository: "NVIDIA/NemoClaw", + commitSha: HEAD_SHA, + planHash: buildRiskPlan({ headSha: HEAD_SHA, changedFiles: ["src/lib/onboard.ts"] }).planHash, + correlationId: "12345678-1234-4123-8123-123456789abc", + expectedJobs: ["onboard-repair", "onboard-resume"], + expectedShards: { + "onboard-repair": ["default"], + "onboard-resume": ["default"], + }, + requiresManualExpansion: false, + checkRunId: 1, + childRunId: 2, + childRunUrl: "https://github.com/NVIDIA/NemoClaw/actions/runs/2", + }; +} + +function signal(jobId: string, overrides: Partial = {}): E2eRiskSignal { + const gate = state(); + return { + version: 1, + jobId, + shardId: "default", + expectedSha: gate.commitSha, + testedSha: gate.commitSha, + planHash: gate.planHash, + correlationId: gate.correlationId, + passed: 1, + failed: 0, + skipped: 0, + pending: 0, + unhandledErrors: 0, + runReason: "passed", + ...overrides, + }; +} + +describe("post-merge E2E risk gate", () => { + it("accepts only the exact trusted main-push context", () => { + const trusted = { + eventName: "push", + ref: "refs/heads/main", + sha: HEAD_SHA, + commitSha: HEAD_SHA, + }; + + expect(() => assertTrustedMainPush(trusted)).not.toThrow(); + expect(() => assertTrustedMainPush({ ...trusted, eventName: "pull_request" })).toThrow( + /exact trusted main push/u, + ); + expect(() => assertTrustedMainPush({ ...trusted, ref: "refs/heads/feature" })).toThrow( + /exact trusted main push/u, + ); + expect(() => assertTrustedMainPush({ ...trusted, sha: "b".repeat(40) })).toThrow( + /exact trusted main push/u, + ); + }); + + it("derives changed files from an exact checked-out commit range", () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-risk-git-")); + const git = (...args: string[]) => + execFileSync("git", args, { cwd: directory, encoding: "utf8" }).trim(); + try { + git("init", "--quiet"); + git("config", "user.name", "Risk Gate Test"); + git("config", "user.email", "risk-gate@example.invalid"); + fs.writeFileSync(path.join(directory, "README.md"), "base\n"); + fs.mkdirSync(path.join(directory, "src", "lib", "credentials"), { recursive: true }); + fs.writeFileSync(path.join(directory, "src", "lib", "credentials", "token.ts"), "base\n"); + git("add", "README.md", "src/lib/credentials/token.ts"); + git("-c", "commit.gpgsign=false", "commit", "--quiet", "-m", "base"); + const base = git("rev-parse", "HEAD"); + fs.mkdirSync(path.join(directory, "docs")); + git("mv", "src/lib/credentials/token.ts", "docs/token.ts"); + fs.writeFileSync(path.join(directory, "src", "feature.ts"), "export {};\n"); + git("add", "docs/token.ts", "src/feature.ts"); + git("-c", "commit.gpgsign=false", "commit", "--quiet", "-m", "feature"); + const head = git("rev-parse", "HEAD"); + + expect(changedFilesBetween(base, head, directory)).toEqual([ + "docs/token.ts", + "src/feature.ts", + "src/lib/credentials/token.ts", + ]); + expect(() => changedFilesBetween(head, base, directory)).toThrow(/does not match/u); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }); + + it("accepts only the deterministic exact-SHA plan", () => { + const plan = buildRiskPlan({ headSha: HEAD_SHA, changedFiles: ["src/lib/onboard.ts"] }); + + expect(validateRiskPlan(plan, ALLOWED)).toEqual(plan); + expect(() => validateRiskPlan({ ...plan, planHash: "b".repeat(64) }, ALLOWED)).toThrow( + /deterministic hash/u, + ); + expect(() => validateRiskPlan(plan, new Set())).toThrow(/unknown E2E job/u); + }); + + it("accepts only signals bound to the expected job, SHA, plan, and correlation", () => { + const gate = state(); + + expect(validateSignal(signal("onboard-resume"), gate).jobId).toBe("onboard-resume"); + expect(() => + validateSignal(signal("onboard-resume", { expectedSha: "b".repeat(40) }), gate), + ).toThrow(/SHA mismatch/u); + expect(() => validateSignal(signal("other"), gate)).toThrow(/unexpected/u); + expect(() => validateSignal(signal("onboard-resume", { shardId: "unexpected" }), gate)).toThrow( + /shard/u, + ); + }); + + it("derives expected evidence shards from the trusted E2E workflow", () => { + const jobIds = [...new Set(RISK_RULES.flatMap((rule) => rule.requiredJobs))]; + const shards = expectedRiskSignalShards(jobIds); + + expect(Object.keys(shards).sort()).toEqual(jobIds.sort()); + expect(shards["onboard-resume"]).toEqual(["default"]); + expect(shards["security-posture"]).toEqual(["openclaw", "hermes"]); + expect(shards["channels-stop-start"]).toEqual(["openclaw", "hermes"]); + }); + + it("reports success only for complete unskipped evidence", () => { + const verdict = classifyRiskEvidence({ + workflowConclusion: "success", + expectedJobs: ["onboard-repair", "onboard-resume"], + expectedShards: state().expectedShards, + signals: [signal("onboard-repair"), signal("onboard-resume")], + requiresManualExpansion: false, + }); + + expect(verdict.conclusion).toBe("success"); + }); + + it.each([ + { + label: "missing signal", + signals: [signal("onboard-repair")], + manual: false, + }, + { + label: "skipped test", + signals: [signal("onboard-repair"), signal("onboard-resume", { skipped: 1 })], + manual: false, + }, + { + label: "manual expansion", + signals: [signal("onboard-repair"), signal("onboard-resume")], + manual: true, + }, + { + label: "duplicate signal", + signals: [signal("onboard-repair"), signal("onboard-repair"), signal("onboard-resume")], + manual: false, + }, + ])("reports neutral for $label", ({ signals, manual }) => { + const verdict = classifyRiskEvidence({ + workflowConclusion: "success", + expectedJobs: ["onboard-repair", "onboard-resume"], + expectedShards: state().expectedShards, + signals, + requiresManualExpansion: manual, + }); + + expect(verdict.conclusion).toBe("neutral"); + }); + + it("reports a product workflow failure as failure", () => { + const verdict = classifyRiskEvidence({ + workflowConclusion: "failure", + expectedJobs: ["onboard-repair"], + expectedShards: { "onboard-repair": ["default"] }, + signals: [], + requiresManualExpansion: false, + }); + + expect(verdict.conclusion).toBe("failure"); + }); + + it("reports failed exact-SHA test evidence as failure even when the workflow is green", () => { + const verdict = classifyRiskEvidence({ + workflowConclusion: "success", + expectedJobs: ["onboard-repair"], + expectedShards: { "onboard-repair": ["default"] }, + signals: [signal("onboard-repair", { failed: 1, runReason: "failed" })], + requiresManualExpansion: false, + }); + + expect(verdict.conclusion).toBe("failure"); + }); + + it("requires every expected matrix shard to pass", () => { + const complete = classifyRiskEvidence({ + workflowConclusion: "success", + expectedJobs: ["security-posture"], + expectedShards: { "security-posture": ["openclaw", "hermes"] }, + signals: [ + signal("security-posture", { shardId: "openclaw" }), + signal("security-posture", { shardId: "hermes" }), + ], + requiresManualExpansion: false, + }); + const missingShard = classifyRiskEvidence({ + workflowConclusion: "success", + expectedJobs: ["security-posture"], + expectedShards: { "security-posture": ["openclaw", "hermes"] }, + signals: [signal("security-posture", { shardId: "openclaw" })], + requiresManualExpansion: false, + }); + + expect(complete.conclusion).toBe("success"); + expect(missingShard.conclusion).toBe("neutral"); + }); +}); diff --git a/test/pr-workflow-contract.test.ts b/test/pr-workflow-contract.test.ts index 7151c57048f..936f9de2b94 100644 --- a/test/pr-workflow-contract.test.ts +++ b/test/pr-workflow-contract.test.ts @@ -1007,6 +1007,12 @@ describe("pull request and main workflow contracts", () => { expect(vitestConfig).toContain('name: "integration"'); expect(vitestConfig).toContain('include: ["test/**/*.test.{js,ts}"]'); expect(vitestConfig).toContain('name: "e2e-support"'); + expect(stepRuns(prWorkflow.jobs["e2e-support"]).join("\n")).toContain( + "npx vitest run --project e2e-support", + ); + expect(stepRuns(mainWorkflow.jobs["e2e-support"]).join("\n")).toContain( + "npx vitest run --project e2e-support", + ); expect(vitestConfig).toContain('name: "package-contract"'); expect(vitestConfig).toContain('"test/e2e/**"'); expect(vitestConfig).toContain('"test/install-express-prompt.test.ts"'); @@ -1201,6 +1207,7 @@ describe("pull request and main workflow contracts", () => { "installer-integration", "cli-tests", "plugin-tests", + "e2e-support", "test-e2e-ollama-proxy", ]); expect(prWorkflow.jobs["cli-tests"].needs).toEqual(["changes", "cli-test-shards"]); @@ -1212,6 +1219,7 @@ describe("pull request and main workflow contracts", () => { "installer-integration", "cli-tests", "plugin-tests", + "e2e-support", "test-e2e-ollama-proxy", ]) { expect(prChecksRun).toContain(`require_success "${jobName}"`); @@ -1226,6 +1234,7 @@ describe("pull request and main workflow contracts", () => { "real-openclaw-dist-harness", "cli-tests", "plugin-tests", + "e2e-support", "test-e2e-ollama-proxy", ]); expect(mainWorkflow.jobs["cli-tests"].needs).toBe("cli-test-shards"); @@ -1236,6 +1245,7 @@ describe("pull request and main workflow contracts", () => { "real-openclaw-dist-harness", "cli-tests", "plugin-tests", + "e2e-support", "test-e2e-ollama-proxy", ]) { expect(mainChecksRun).toContain(`require_success "${jobName}"`); diff --git a/tools/e2e-advisor/README.md b/tools/e2e-advisor/README.md index 02d1e1b3d57..f32f17258a5 100644 --- a/tools/e2e-advisor/README.md +++ b/tools/e2e-advisor/README.md @@ -37,6 +37,32 @@ the trusted timing signal. `.github/workflows/e2e.yaml`, but the advisor job does not trigger those commands automatically. +## Post-merge shadow controller + +The model-independent `.github/workflows/post-merge-e2e-risk-gate-shadow.yaml` workflow +uses the same checked-in risk-plan policy after a commit lands on `main`. It +does not consume model output or the advisor artifact. Instead, +`tools/e2e-advisor/post-merge-risk-gate.mts` confirms the trusted controller checkout, +builds a new plan from the exact `github.event.before` and `github.event.after` range, +and dispatches at most three `automaticJobs` to `e2e.yaml` against the merged +commit. + +The child workflow validates the exact checkout SHA, its reachability from +`main`, selective-job inputs, plan hash, and correlation ID before E2E +preparation or secret-bearing jobs can run. The shadow-only Vitest reporter +records the observed checkout SHA and pass, failure, skip, pending, and +unhandled-error counts for each job and matrix shard. The controller accepts +only signals bound to the expected SHA, plan hash, correlation ID, job, and +shard. + +The controller writes a check on the merged commit without posting a PR +comment or running the scheduled/manual scorecard. Complete unskipped evidence +reports success, selected E2E workflow or exact-SHA test failures report +failure, and incomplete, ambiguous, skipped, or cap-limited evidence that +requires manual expansion reports neutral. If no runtime risk family matches, +it reports success without dispatching E2E. This is post-merge shadow evidence, +not a required pre-merge check. + ## Required secret Configure this repository secret for E2E recommendations: @@ -61,8 +87,9 @@ dispatch commands; it does not trigger E2E workflows automatically. - `e2e-advisor-prompt.md` — task prompt sent to the advisor. Diff, changed files, metadata, and schema are injected into the Pi session as deterministic synthetic tool results and captured in the session transcript. - `risk-plan.json` — stable exact-SHA risk families, invariants, required jobs, a capped - `automaticJobs` candidate subset, manual-expansion state, and the plan hash. Both E2E - advisor projections consume the required-job floor, but neither dispatches it. + `automaticJobs` subset, manual-expansion state, and the plan hash. Both E2E advisor + projections consume the required-job floor, while the separate post-merge controller + rebuilds the plan and dispatches the automatic subset. - `e2e-advisor-raw-output.txt` — raw advisor transcript and diagnostics. - `e2e-advisor-result.json` — parsed advisor response or execution metadata. - `e2e-advisor-session.html` — exported advisor session transcript. @@ -94,5 +121,6 @@ secret. Run `npm install` first so the Pi SDK dependency is available. `tools/e2e-advisor/schema.json` defines the normalized coverage recommendation shape. `tools/e2e-advisor/targets-schema.json` defines the normalized target recommendation shape used by the `targets` and `jobs` dispatch commands. -Future enforcement should be implemented as a single dynamic required check that verifies the -recommended E2E jobs passed for the same PR head SHA. +The post-merge shadow check does not establish pre-merge enforcement. Any future required check +must verify complete E2E evidence for the same PR head SHA without making model availability part +of the merge authority. diff --git a/tools/e2e-advisor/post-merge-risk-gate.mts b/tools/e2e-advisor/post-merge-risk-gate.mts new file mode 100755 index 00000000000..ddc38026c21 --- /dev/null +++ b/tools/e2e-advisor/post-merge-risk-gate.mts @@ -0,0 +1,630 @@ +#!/usr/bin/env node + +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { execFileSync } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +import YAML from "yaml"; + +import { githubApi } from "../advisors/github.mts"; +import { parseArgs } from "../advisors/io.mts"; +import { buildRiskPlan, type RiskPlan } from "../advisors/risk-plan.mts"; +import { readFreeStandingJobsInventory } from "../e2e/workflow-boundary.mts"; +import type { E2eRiskSignal } from "./risk-signal.ts"; + +const E2E_WORKFLOW = "e2e.yaml"; +const CHECK_NAME = "E2E / Post-merge Risk Gate (shadow)"; +const SHA_PATTERN = /^[a-f0-9]{40}$/u; +const HASH_PATTERN = /^[a-f0-9]{64}$/u; +const JOB_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_-]*$/u; +const SHARD_PATTERN = /^(?:default|[A-Za-z0-9][A-Za-z0-9_-]*)$/u; +const CORRELATION_PATTERN = + /^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/u; +const MAX_PLAN_BYTES = 1024 * 1024; + +type CheckConclusion = "success" | "failure" | "neutral"; + +type WorkflowRun = { + id: number; + name: string; + event: string; + head_sha: string; + status: string; + conclusion: string | null; + created_at: string; + display_title: string; + html_url: string; +}; + +type CheckRun = { id: number }; + +export type RiskGateState = { + version: 1; + repository: string; + commitSha: string; + planHash: string; + correlationId: string; + expectedJobs: string[]; + expectedShards: Record; + requiresManualExpansion: boolean; + checkRunId: number; + childRunId: number; + childRunUrl: string; +}; + +export type RiskEvidenceVerdict = { + conclusion: CheckConclusion; + title: string; + summary: string; +}; + +export function assertTrustedMainPush(options: { + eventName: string | undefined; + ref: string | undefined; + sha: string | undefined; + commitSha: string; +}): void { + if ( + options.eventName !== "push" || + options.ref !== "refs/heads/main" || + options.sha !== options.commitSha + ) { + throw new Error("post-merge risk dispatch requires the exact trusted main push context"); + } +} + +function isRecord(value: unknown): value is Record { + return !!value && typeof value === "object" && !Array.isArray(value); +} + +function readRegularJson(file: string, maxBytes = MAX_PLAN_BYTES): unknown { + const stat = fs.lstatSync(file); + if (!stat.isFile() || stat.isSymbolicLink()) throw new Error(`${file} must be a regular file`); + if (stat.size > maxBytes) throw new Error(`${file} exceeds ${maxBytes} bytes`); + return JSON.parse(fs.readFileSync(file, "utf8")); +} + +export function validateRiskPlan(value: unknown, allowedJobs: ReadonlySet): RiskPlan { + if (!isRecord(value)) throw new Error("risk plan must be an object"); + if (value.version !== 1) throw new Error("unsupported risk-plan version"); + if (typeof value.headSha !== "string" || !SHA_PATTERN.test(value.headSha)) { + throw new Error("risk plan headSha must be a lowercase 40-character SHA"); + } + if ( + !Array.isArray(value.changedFiles) || + !value.changedFiles.every((file) => typeof file === "string") + ) { + throw new Error("risk plan changedFiles must be strings"); + } + if (value.maxAutomaticJobs !== 3) throw new Error("risk plan automatic-job cap must be 3"); + const rebuilt = buildRiskPlan({ + headSha: value.headSha, + changedFiles: value.changedFiles, + maxAutomaticJobs: value.maxAutomaticJobs, + }); + if (JSON.stringify(value) !== JSON.stringify(rebuilt)) { + throw new Error("risk plan does not match its deterministic hash and inputs"); + } + if (!HASH_PATTERN.test(rebuilt.planHash)) throw new Error("risk plan hash is invalid"); + const automatic = new Set(rebuilt.automaticJobs); + if (automatic.size !== rebuilt.automaticJobs.length) { + throw new Error("risk plan automatic jobs must be unique"); + } + for (const job of rebuilt.requiredJobs) { + if (!JOB_PATTERN.test(job.id) || !allowedJobs.has(job.id)) { + throw new Error(`risk plan names unknown E2E job: ${job.id}`); + } + } + return rebuilt; +} + +export function validateSignal( + value: unknown, + state: Pick< + RiskGateState, + "commitSha" | "planHash" | "correlationId" | "expectedJobs" | "expectedShards" + >, +): E2eRiskSignal { + if (!isRecord(value) || value.version !== 1) throw new Error("invalid risk signal version"); + const signal = value as E2eRiskSignal; + if (!state.expectedJobs.includes(signal.jobId)) throw new Error("risk signal job is unexpected"); + if (!state.expectedShards[signal.jobId]?.includes(signal.shardId)) { + throw new Error("risk signal shard is unexpected"); + } + if (signal.expectedSha !== state.commitSha) throw new Error("risk signal SHA mismatch"); + if (signal.testedSha !== state.commitSha) throw new Error("risk signal tested SHA mismatch"); + if (signal.planHash !== state.planHash) throw new Error("risk signal plan hash mismatch"); + if (signal.correlationId !== state.correlationId) { + throw new Error("risk signal correlation mismatch"); + } + for (const key of ["passed", "failed", "skipped", "pending", "unhandledErrors"] as const) { + if (!Number.isSafeInteger(signal[key]) || signal[key] < 0) { + throw new Error(`risk signal ${key} must be a non-negative integer`); + } + } + if (!(["passed", "failed", "interrupted"] as const).includes(signal.runReason)) { + throw new Error("risk signal runReason is invalid"); + } + return signal; +} + +export function classifyRiskEvidence(options: { + workflowConclusion: string | null; + expectedJobs: readonly string[]; + expectedShards: Readonly>; + signals: readonly E2eRiskSignal[]; + requiresManualExpansion: boolean; +}): RiskEvidenceVerdict { + if ( + ["failure", "timed_out", "action_required", "startup_failure"].includes( + options.workflowConclusion ?? "", + ) + ) { + return { + conclusion: "failure", + title: "Selected E2E workflow failed", + summary: `The correlated workflow concluded ${options.workflowConclusion}.`, + }; + } + if (options.workflowConclusion !== "success") { + return { + conclusion: "neutral", + title: "Selected E2E workflow produced no complete signal", + summary: `The correlated workflow concluded ${options.workflowConclusion ?? "without a conclusion"}.`, + }; + } + const byJobShard = new Map(); + const duplicates = new Set(); + for (const signal of options.signals) { + const key = `${signal.jobId}:${signal.shardId}`; + if (byJobShard.has(key)) duplicates.add(key); + byJobShard.set(key, signal); + } + if (duplicates.size > 0) { + return { + conclusion: "neutral", + title: "Selected E2E jobs produced ambiguous evidence", + summary: `Multiple risk signals were uploaded for: ${[...duplicates].sort().join(", ")}.`, + }; + } + const expectedEvidence = options.expectedJobs.flatMap((job) => + (options.expectedShards[job] ?? []).map((shard) => `${job}:${shard}`), + ); + const jobsWithoutShardPolicy = options.expectedJobs.filter( + (job) => (options.expectedShards[job]?.length ?? 0) === 0, + ); + if (jobsWithoutShardPolicy.length > 0) { + return { + conclusion: "neutral", + title: "Selected E2E jobs lack an evidence policy", + summary: `No trusted shard policy was found for: ${jobsWithoutShardPolicy.join(", ")}.`, + }; + } + const missing = expectedEvidence.filter((key) => !byJobShard.has(key)); + if (missing.length > 0) { + return { + conclusion: "neutral", + title: "Selected E2E jobs are missing test evidence", + summary: `No risk signal was uploaded for: ${missing.join(", ")}.`, + }; + } + const failed = expectedEvidence.filter((key) => { + const signal = byJobShard.get(key)!; + return signal.failed > 0 || signal.unhandledErrors > 0 || signal.runReason === "failed"; + }); + if (failed.length > 0) { + return { + conclusion: "failure", + title: "Selected E2E jobs reported test failures", + summary: `Failed exact-SHA evidence was reported for: ${failed.join(", ")}.`, + }; + } + const partial = expectedEvidence.filter((key) => { + const signal = byJobShard.get(key)!; + return ( + signal.passed < 1 || signal.skipped > 0 || signal.pending > 0 || signal.runReason !== "passed" + ); + }); + if (partial.length > 0) { + return { + conclusion: "neutral", + title: "Selected E2E jobs produced partial or skipped evidence", + summary: `The following jobs did not produce an unskipped pass: ${partial.join(", ")}.`, + }; + } + if (options.requiresManualExpansion) { + return { + conclusion: "neutral", + title: "Automatic shadow subset passed; broader evidence is required", + summary: + "The risk plan exceeded the three-job automatic cap, so this passing subset is not complete merge evidence.", + }; + } + return { + conclusion: "success", + title: "All risk-selected E2E jobs passed", + summary: `${expectedEvidence.length} exact-SHA job shard(s) produced complete, unskipped evidence.`, + }; +} + +function appendOutput(name: string, value: string): void { + const output = process.env.GITHUB_OUTPUT; + if (output) fs.appendFileSync(output, `${name}=${value}\n`); +} + +async function createCheck( + repository: string, + token: string, + headSha: string, + title: string, + summary: string, +): Promise { + const check = await githubApi(`repos/${repository}/check-runs`, token, { + method: "POST", + body: { + name: CHECK_NAME, + head_sha: headSha, + status: "in_progress", + output: { title, summary }, + }, + userAgent: "nemoclaw-e2e-risk-gate", + }); + return check.id; +} + +async function completeCheck( + state: Pick, + token: string, + verdict: RiskEvidenceVerdict, + detailsUrl?: string, +): Promise { + await githubApi(`repos/${state.repository}/check-runs/${state.checkRunId}`, token, { + method: "PATCH", + body: { + status: "completed", + conclusion: verdict.conclusion, + completed_at: new Date().toISOString(), + details_url: detailsUrl, + output: { title: verdict.title, summary: verdict.summary }, + }, + userAgent: "nemoclaw-e2e-risk-gate", + }); +} + +async function completeNeutralAfterControllerError( + state: Pick, + token: string, + title: string, + detailsUrl?: string, +): Promise { + try { + await completeCheck( + state, + token, + { + conclusion: "neutral", + title, + summary: + "The shadow controller could not produce complete, trustworthy evidence. Inspect the controller workflow for details.", + }, + detailsUrl, + ); + return true; + } catch (error) { + console.error( + `failed to close shadow check after controller error: ${error instanceof Error ? error.message : String(error)}`, + ); + return false; + } +} + +export function changedFilesBetween( + baseSha: string, + commitSha: string, + workspace = process.cwd(), +): string[] { + if (!SHA_PATTERN.test(baseSha) || !SHA_PATTERN.test(commitSha)) { + throw new Error("base and tested commits must be lowercase 40-character SHAs"); + } + const checkedOutSha = execFileSync("git", ["rev-parse", "--verify", "HEAD"], { + cwd: workspace, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }).trim(); + if (checkedOutSha !== commitSha) { + throw new Error("trusted controller checkout does not match the tested commit"); + } + const output = execFileSync( + "git", + ["diff", "--no-renames", "--name-only", "-z", baseSha, commitSha], + { + cwd: workspace, + encoding: "utf8", + maxBuffer: 16 * 1024 * 1024, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + const files = output.split("\0").filter(Boolean); + if (files.length > 5000) throw new Error("post-merge risk plan exceeds 5000 changed files"); + if (files.some((file) => file.startsWith("/") || file.split("/").includes(".."))) { + throw new Error("post-merge diff contains an unsafe repository path"); + } + return files; +} + +export function expectedRiskSignalShards( + jobIds: readonly string[], + workflowPath = ".github/workflows/e2e.yaml", +): Record { + const workflow = YAML.parse(fs.readFileSync(workflowPath, "utf8")) as unknown; + const jobs = isRecord(workflow) && isRecord(workflow.jobs) ? workflow.jobs : {}; + return Object.fromEntries( + jobIds.map((jobId) => { + const job = isRecord(jobs[jobId]) ? jobs[jobId] : {}; + const strategy = isRecord(job.strategy) ? job.strategy : {}; + const matrix = isRecord(strategy.matrix) ? strategy.matrix : null; + let shards = ["default"]; + if (matrix) { + const keys = Object.keys(matrix); + if (keys.length === 1 && Array.isArray(matrix.agent)) { + shards = matrix.agent.filter((value): value is string => typeof value === "string"); + if (shards.length !== matrix.agent.length) { + throw new Error(`${jobId} risk matrix agent values must be strings`); + } + } else if (keys.length === 1 && Array.isArray(matrix.include)) { + shards = matrix.include.map((entry) => { + if (!isRecord(entry) || typeof entry.agent !== "string") { + throw new Error(`${jobId} risk matrix include entries must name an agent`); + } + return entry.agent; + }); + } else { + throw new Error(`${jobId} uses an unsupported risk-evidence matrix`); + } + } + if ( + shards.length === 0 || + new Set(shards).size !== shards.length || + shards.some((shard) => !SHARD_PATTERN.test(shard)) + ) { + throw new Error(`${jobId} risk evidence shards must be unique safe identifiers`); + } + return [jobId, shards]; + }), + ); +} + +async function findCorrelatedRun( + repository: string, + token: string, + correlationId: string, + dispatchedAt: number, +): Promise { + const expectedTitle = `E2E risk ${correlationId}`; + for (let attempt = 0; attempt < 60; attempt += 1) { + const payload = await githubApi<{ workflow_runs: WorkflowRun[] }>( + `repos/${repository}/actions/workflows/${E2E_WORKFLOW}/runs?event=workflow_dispatch&branch=main&per_page=50`, + token, + { userAgent: "nemoclaw-e2e-risk-gate" }, + ); + const match = payload.workflow_runs.find( + (run) => + run.display_title === expectedTitle && Date.parse(run.created_at) >= dispatchedAt - 5000, + ); + if (match) return match; + await new Promise((resolve) => setTimeout(resolve, 5000)); + } + throw new Error("timed out locating the correlated E2E workflow run"); +} + +async function start(options: { + baseSha: string; + commitSha: string; + planPath: string; + statePath: string; +}): Promise { + const token = process.env.GITHUB_TOKEN ?? ""; + const repository = process.env.GITHUB_REPOSITORY ?? ""; + if (!token || !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository)) { + throw new Error("GITHUB_TOKEN and a safe GITHUB_REPOSITORY are required"); + } + assertTrustedMainPush({ + eventName: process.env.GITHUB_EVENT_NAME, + ref: process.env.GITHUB_REF, + sha: process.env.GITHUB_SHA, + commitSha: options.commitSha, + }); + + const allowedJobs = new Set(readFreeStandingJobsInventory().allowedJobs); + const plan = validateRiskPlan( + buildRiskPlan({ + headSha: options.commitSha, + changedFiles: changedFilesBetween(options.baseSha, options.commitSha), + }), + allowedJobs, + ); + fs.writeFileSync(options.planPath, `${JSON.stringify(plan, null, 2)}\n`, { mode: 0o600 }); + + const checkRunId = await createCheck( + repository, + token, + options.commitSha, + "Post-merge risk-selected E2E is being dispatched", + `Plan ${plan.planHash.slice(0, 12)} selected ${plan.automaticJobs.length} automatic job(s).`, + ); + appendOutput("check_id", String(checkRunId)); + try { + const expectedShards = expectedRiskSignalShards(plan.automaticJobs); + if (plan.requiredJobs.length === 0) { + await completeCheck({ repository, checkRunId }, token, { + conclusion: "success", + title: "No post-merge runtime E2E required", + summary: "The deterministic risk plan matched no runtime regression family.", + }); + appendOutput("dispatched", "false"); + appendOutput("finalized", "true"); + return; + } + + const correlationId = randomUUID(); + if (!CORRELATION_PATTERN.test(correlationId)) { + throw new Error("generated correlation id is invalid"); + } + const dispatchedAt = Date.now(); + await githubApi(`repos/${repository}/actions/workflows/${E2E_WORKFLOW}/dispatches`, token, { + method: "POST", + body: { + ref: "main", + inputs: { + jobs: plan.automaticJobs.join(","), + checkout_sha: options.commitSha, + risk_plan_hash: plan.planHash, + risk_correlation: correlationId, + risk_shadow: "true", + }, + }, + userAgent: "nemoclaw-e2e-risk-gate", + }); + const child = await findCorrelatedRun(repository, token, correlationId, dispatchedAt); + const state: RiskGateState = { + version: 1, + repository, + commitSha: options.commitSha, + planHash: plan.planHash, + correlationId, + expectedJobs: plan.automaticJobs, + expectedShards, + requiresManualExpansion: plan.requiresManualExpansion, + checkRunId, + childRunId: child.id, + childRunUrl: child.html_url, + }; + fs.writeFileSync(options.statePath, `${JSON.stringify(state, null, 2)}\n`, { mode: 0o600 }); + appendOutput("dispatched", "true"); + appendOutput("run_id", String(child.id)); + } catch (error) { + const finalized = await completeNeutralAfterControllerError( + { repository, checkRunId }, + token, + "Risk-selected E2E could not be dispatched", + ); + if (finalized) appendOutput("finalized", "true"); + throw error; + } +} + +function findSignalFiles(root: string): string[] { + if (!fs.existsSync(root)) return []; + const files: string[] = []; + const visit = (directory: string): void => { + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const full = path.join(directory, entry.name); + if (entry.isSymbolicLink()) throw new Error("risk evidence must not contain symlinks"); + if (entry.isDirectory()) visit(full); + else if (entry.isFile() && entry.name === "risk-signal.json") files.push(full); + } + }; + visit(root); + return files.sort((left, right) => left.localeCompare(right)); +} + +async function finish(options: { statePath: string; evidencePath: string }): Promise { + const token = process.env.GITHUB_TOKEN ?? ""; + const state = readRegularJson(options.statePath) as RiskGateState; + if (!token || state.version !== 1 || !SHA_PATTERN.test(state.commitSha)) { + throw new Error("valid gate state and GITHUB_TOKEN are required"); + } + try { + const child = await githubApi( + `repos/${state.repository}/actions/runs/${state.childRunId}`, + token, + { userAgent: "nemoclaw-e2e-risk-gate" }, + ); + if ( + child.id !== state.childRunId || + child.name !== "E2E" || + child.event !== "workflow_dispatch" || + child.display_title !== `E2E risk ${state.correlationId}` + ) { + throw new Error("correlated E2E workflow identity changed"); + } + const signals = + child.conclusion === "success" + ? findSignalFiles(options.evidencePath).map((file) => + validateSignal(readRegularJson(file), state), + ) + : []; + const verdict = classifyRiskEvidence({ + workflowConclusion: child.conclusion, + expectedJobs: state.expectedJobs, + expectedShards: state.expectedShards, + signals, + requiresManualExpansion: state.requiresManualExpansion, + }); + await completeCheck(state, token, verdict, state.childRunUrl); + } catch (error) { + await completeNeutralAfterControllerError( + state, + token, + "Risk-selected E2E evidence could not be verified", + state.childRunUrl, + ); + throw error; + } +} + +async function abandon(checkId: string): Promise { + const token = process.env.GITHUB_TOKEN ?? ""; + const repository = process.env.GITHUB_REPOSITORY ?? ""; + if ( + !token || + !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository) || + !/^[1-9][0-9]*$/u.test(checkId) + ) { + throw new Error("GITHUB_TOKEN, a safe GITHUB_REPOSITORY, and numeric --check-id are required"); + } + await completeCheck({ repository, checkRunId: Number(checkId) }, token, { + conclusion: "neutral", + title: "Risk-selected E2E controller stopped early", + summary: + "The shadow controller stopped before it could produce complete evidence. Inspect the controller workflow for details.", + }); +} + +async function main(): Promise { + const args = parseArgs(process.argv.slice(2)); + const mode = args.mode; + if (mode === "start") { + await start({ + baseSha: args.base || "", + commitSha: args.commit || "", + planPath: args.plan || "/tmp/post-merge-risk-plan.json", + statePath: args.state || "/tmp/e2e-risk-gate-state.json", + }); + return; + } + if (mode === "finish") { + await finish({ + statePath: args.state || "/tmp/e2e-risk-gate-state.json", + evidencePath: args.evidence || "/tmp/e2e-risk-evidence", + }); + return; + } + if (mode === "abandon") { + await abandon(args["check-id"] || ""); + return; + } + throw new Error("--mode must be start, finish, or abandon"); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + }); +} diff --git a/tools/e2e-advisor/risk-signal.ts b/tools/e2e-advisor/risk-signal.ts new file mode 100644 index 00000000000..db1ab845a00 --- /dev/null +++ b/tools/e2e-advisor/risk-signal.ts @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export type E2eRiskSignal = { + version: 1; + jobId: string; + shardId: string; + expectedSha: string; + testedSha: string; + planHash: string; + correlationId: string; + passed: number; + failed: number; + skipped: number; + pending: number; + unhandledErrors: number; + runReason: "passed" | "failed" | "interrupted"; +}; diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index 6b15f394432..35dd668a698 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -148,7 +148,10 @@ function validateIssueRoutingRetirement(errors: string[], workflow: OperationsWo ) { errors.push("report-to-pr must hold only pull-requests: write"); } - if (job.if !== "${{ always() && github.event_name == 'workflow_dispatch' }}") { + if ( + job.if !== + "${{ always() && github.event_name == 'workflow_dispatch' && !inputs.risk_shadow }}" + ) { errors.push("report-to-pr must run only for manual workflow dispatches"); } const report = findStep(job, "Post E2E target results to PR"); @@ -212,7 +215,7 @@ function validateScorecard(errors: string[], workflow: OperationsWorkflow): void const permissions = permissionMap(job.permissions); if ( job.if !== - "${{ always() && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') }}" + "${{ always() && (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && !inputs.risk_shadow)) }}" ) { errors.push("scorecard must run after scheduled and manual E2E executions"); } diff --git a/tools/pr-review-advisor/README.md b/tools/pr-review-advisor/README.md index f700c521728..e7356e97356 100644 --- a/tools/pr-review-advisor/README.md +++ b/tools/pr-review-advisor/README.md @@ -40,7 +40,9 @@ It intentionally does not report GitHub mergeability, branch protection, CI stat The workflow is advisory and must not be configured as a required status check. It uses the deterministic plan as review context but does not run its jobs. E2E Advisor emits the corresponding plan-backed recommendations separately and likewise does not dispatch E2E. Model availability must -not become the authority for whether a pull request can merge. +not become the authority for whether a pull request can merge. After a commit lands, a separate +model-independent shadow controller rebuilds the plan from the exact `main` push range and runs its +capped automatic subset. That post-merge check does not make PR Review Advisor a merge gate. Required-check status is point-in-time context, not a settled-CI gate. Earlier `PR_REVIEW_ADVISOR_WAIT_*` workflow variables were inert and have been removed; any future waiting From 74f5fdbf1fef84826e56204efe93ee74967da956 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Wed, 8 Jul 2026 00:11:59 -0700 Subject: [PATCH 2/9] fix(ci): build E2E support artifacts Signed-off-by: Carlos Villela --- .github/workflows/main.yaml | 3 +++ .github/workflows/pr.yaml | 3 +++ test/post-merge-e2e-risk-gate-workflow.test.ts | 13 +++++++------ test/pr-workflow-contract.test.ts | 12 ++++++++---- 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 9dc2c207c8b..d2b65b414fb 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -169,6 +169,9 @@ jobs: - name: Install E2E support dependencies run: npm ci --ignore-scripts + - name: Build CLI artifacts for E2E support + run: npm run build:cli + - name: Run E2E support tests run: npx vitest run --project e2e-support diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 54b29582d23..77cf9a9337c 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -328,6 +328,9 @@ jobs: - name: Install E2E support dependencies run: npm ci --ignore-scripts + - name: Build CLI artifacts for E2E support + run: npm run build:cli + - name: Run E2E support tests run: npx vitest run --project e2e-support diff --git a/test/post-merge-e2e-risk-gate-workflow.test.ts b/test/post-merge-e2e-risk-gate-workflow.test.ts index a3943145920..60c5764ea7e 100644 --- a/test/post-merge-e2e-risk-gate-workflow.test.ts +++ b/test/post-merge-e2e-risk-gate-workflow.test.ts @@ -25,12 +25,13 @@ function step(job: WorkflowJob, name: string): WorkflowStep { } function collectStrings(value: unknown): string[] { - if (typeof value === "string") return [value]; - if (Array.isArray(value)) return value.flatMap(collectStrings); - if (value && typeof value === "object") { - return Object.values(value).flatMap(collectStrings); - } - return []; + return typeof value === "string" + ? [value] + : Array.isArray(value) + ? value.flatMap(collectStrings) + : value && typeof value === "object" + ? Object.values(value).flatMap(collectStrings) + : []; } describe("post-merge E2E risk gate shadow workflow", () => { diff --git a/test/pr-workflow-contract.test.ts b/test/pr-workflow-contract.test.ts index 936f9de2b94..c4e52e803f2 100644 --- a/test/pr-workflow-contract.test.ts +++ b/test/pr-workflow-contract.test.ts @@ -1007,12 +1007,16 @@ describe("pull request and main workflow contracts", () => { expect(vitestConfig).toContain('name: "integration"'); expect(vitestConfig).toContain('include: ["test/**/*.test.{js,ts}"]'); expect(vitestConfig).toContain('name: "e2e-support"'); - expect(stepRuns(prWorkflow.jobs["e2e-support"]).join("\n")).toContain( + expect(stepRuns(prWorkflow.jobs["e2e-support"])).toEqual([ + "npm ci --ignore-scripts", + "npm run build:cli", "npx vitest run --project e2e-support", - ); - expect(stepRuns(mainWorkflow.jobs["e2e-support"]).join("\n")).toContain( + ]); + expect(stepRuns(mainWorkflow.jobs["e2e-support"])).toEqual([ + "npm ci --ignore-scripts", + "npm run build:cli", "npx vitest run --project e2e-support", - ); + ]); expect(vitestConfig).toContain('name: "package-contract"'); expect(vitestConfig).toContain('"test/e2e/**"'); expect(vitestConfig).toContain('"test/install-express-prompt.test.ts"'); From bfc0b093aa4989c4d05ba63081a790ba041bde68 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Wed, 8 Jul 2026 00:34:44 -0700 Subject: [PATCH 3/9] fix(ci): harden post-merge risk evidence Signed-off-by: Carlos Villela --- .github/workflows/e2e.yaml | 2 +- .../post-merge-e2e-risk-gate-shadow.yaml | 20 +- test/e2e-private-file.test.ts | 76 ++++++++ test/e2e-risk-signal-reporter.test.ts | 34 ++++ test/e2e/risk-signal-reporter.ts | 17 +- .../post-merge-e2e-risk-gate-workflow.test.ts | 22 ++- test/post-merge-e2e-risk-gate.test.ts | 50 +++++ tools/advisors/github.mts | 8 +- tools/e2e-advisor/post-merge-risk-gate.mts | 172 ++++++++++++++++-- tools/e2e-advisor/private-file.ts | 56 ++++++ 10 files changed, 422 insertions(+), 35 deletions(-) create mode 100644 test/e2e-private-file.test.ts create mode 100644 tools/e2e-advisor/private-file.ts diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 8f9fd74f5d6..c7d03ab75e3 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -54,7 +54,7 @@ permissions: contents: read concurrency: - group: e2e-${{ github.ref }}-${{ inputs.risk_correlation || inputs.targets || 'supported' }}-${{ inputs.jobs || 'all-jobs' }} + group: e2e-${{ github.ref }}-${{ inputs.risk_shadow && github.run_id || inputs.targets || 'supported' }}-${{ inputs.risk_shadow && 'risk-shadow' || inputs.jobs || 'all-jobs' }} cancel-in-progress: false env: diff --git a/.github/workflows/post-merge-e2e-risk-gate-shadow.yaml b/.github/workflows/post-merge-e2e-risk-gate-shadow.yaml index 25238b43926..769928d21c7 100644 --- a/.github/workflows/post-merge-e2e-risk-gate-shadow.yaml +++ b/.github/workflows/post-merge-e2e-risk-gate-shadow.yaml @@ -8,6 +8,7 @@ on: branches: [main] permissions: + # Required to dispatch the separate E2E workflow; no other Actions mutation is performed. actions: write checks: write contents: read @@ -80,7 +81,7 @@ jobs: GH_TOKEN: ${{ github.token }} RUN_ID: ${{ steps.start.outputs.run_id }} run: >- - timeout --signal=TERM --kill-after=30s 100m + timeout --signal=TERM --kill-after=30s 105m gh run watch "$RUN_ID" --repo "$GITHUB_REPOSITORY" --exit-status - name: Download correlated E2E evidence @@ -91,8 +92,10 @@ jobs: RUN_ID: ${{ steps.start.outputs.run_id }} run: gh run download "$RUN_ID" --repo "$GITHUB_REPOSITORY" --dir /tmp/e2e-risk-evidence - - name: Complete exact-commit shadow check + - id: finish + name: Complete exact-commit shadow check if: ${{ always() && steps.start.outputs.dispatched == 'true' }} + continue-on-error: true env: GITHUB_TOKEN: ${{ github.token }} run: >- @@ -100,3 +103,16 @@ jobs: --mode finish --state /tmp/e2e-risk-gate-state.json --evidence /tmp/e2e-risk-evidence + + - name: Close shadow check after completion failure + if: ${{ always() && steps.start.outputs.check_id != '' && steps.start.outputs.dispatched == 'true' && steps.finish.outcome == 'failure' }} + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + node --experimental-strip-types tools/e2e-advisor/post-merge-risk-gate.mts + --mode abandon + --check-id "${{ steps.start.outputs.check_id }}" + + - name: Propagate shadow completion failure + if: ${{ steps.finish.outcome == 'failure' }} + run: exit 1 diff --git a/test/e2e-private-file.test.ts b/test/e2e-private-file.test.ts new file mode 100644 index 00000000000..48ab24aea39 --- /dev/null +++ b/test/e2e-private-file.test.ts @@ -0,0 +1,76 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { execFileSync, spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +import { describe, expect, it } from "vitest"; +import { + readPrivateRegularFile, + writePrivateRegularFile, +} from "../tools/e2e-advisor/private-file.ts"; + +describe("private E2E controller files", () => { + it("writes private regular files without following links or truncating hardlink targets", () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-private-file-")); + const regular = path.join(directory, "regular.json"); + const target = path.join(directory, "target.json"); + const symlink = path.join(directory, "symlink.json"); + const hardlink = path.join(directory, "hardlink.json"); + try { + writePrivateRegularFile(regular, "regular\n"); + expect(readPrivateRegularFile(regular, { maxBytes: 64 })).toBe("regular\n"); + expect(fs.statSync(regular).mode & 0o777).toBe(0o600); + + fs.writeFileSync(target, "protected\n"); + fs.symlinkSync(target, symlink); + fs.linkSync(target, hardlink); + + expect(() => writePrivateRegularFile(symlink, "replaced\n")).toThrow(); + expect(() => writePrivateRegularFile(hardlink, "replaced\n")).toThrow(/private regular/u); + expect(fs.readFileSync(target, "utf8")).toBe("protected\n"); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }); + + it.skipIf(process.platform === "win32")("rejects FIFO paths without blocking", () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-private-fifo-")); + const fifo = path.join(directory, "state.json"); + try { + execFileSync("mkfifo", [fifo]); + const moduleUrl = pathToFileURL(path.resolve("tools/e2e-advisor/private-file.ts")).href; + const read = spawnSync( + process.execPath, + [ + "--experimental-strip-types", + "--input-type=module", + "--eval", + `import { readPrivateRegularFile } from ${JSON.stringify(moduleUrl)}; readPrivateRegularFile(${JSON.stringify(fifo)}, { maxBytes: 64 });`, + ], + { encoding: "utf8", timeout: 2_000 }, + ); + const write = spawnSync( + process.execPath, + [ + "--experimental-strip-types", + "--input-type=module", + "--eval", + `import { writePrivateRegularFile } from ${JSON.stringify(moduleUrl)}; writePrivateRegularFile(${JSON.stringify(fifo)}, "replaced\\n");`, + ], + { encoding: "utf8", timeout: 2_000 }, + ); + + expect(read.error).toBeUndefined(); + expect(read.status).not.toBe(0); + expect(write.error).toBeUndefined(); + expect(write.status).not.toBe(0); + expect(fs.lstatSync(fifo).isFIFO()).toBe(true); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }); +}); diff --git a/test/e2e-risk-signal-reporter.test.ts b/test/e2e-risk-signal-reporter.test.ts index a662bba4038..bf73da84dcd 100644 --- a/test/e2e-risk-signal-reporter.test.ts +++ b/test/e2e-risk-signal-reporter.test.ts @@ -107,4 +107,38 @@ describe("E2E risk signal reporter", () => { fs.rmSync(dir, { recursive: true, force: true }); } }); + + it("refuses a symlinked prior signal without modifying its target", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-risk-signal-")); + const target = path.join(dir, "target.json"); + const signalFile = path.join(dir, RISK_SIGNAL_FILE); + try { + fs.writeFileSync(target, "protected\n"); + fs.symlinkSync(target, signalFile); + + expect(() => + writeRiskSignal(environment(dir), [moduleWithStates(["passed"])], [], "passed"), + ).toThrow(); + expect(fs.readFileSync(target, "utf8")).toBe("protected\n"); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it("refuses a hardlinked prior signal before truncating its target", () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-risk-signal-")); + const target = path.join(dir, "target.json"); + const signalFile = path.join(dir, RISK_SIGNAL_FILE); + try { + fs.writeFileSync(target, "protected\n"); + fs.linkSync(target, signalFile); + + expect(() => + writeRiskSignal(environment(dir), [moduleWithStates(["passed"])], [], "passed"), + ).toThrow(/private regular file/u); + expect(fs.readFileSync(target, "utf8")).toBe("protected\n"); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); }); diff --git a/test/e2e/risk-signal-reporter.ts b/test/e2e/risk-signal-reporter.ts index 66a3627a77e..138ffeec14b 100644 --- a/test/e2e/risk-signal-reporter.ts +++ b/test/e2e/risk-signal-reporter.ts @@ -7,6 +7,10 @@ import path from "node:path"; import type { TestModule } from "vitest/node"; import type { Reporter, TestRunEndReason } from "vitest/reporters"; +import { + readPrivateRegularFile, + writePrivateRegularFile, +} from "../../tools/e2e-advisor/private-file.ts"; import type { E2eRiskSignal } from "../../tools/e2e-advisor/risk-signal.ts"; export const RISK_SIGNAL_FILE = "risk-signal.json"; @@ -93,6 +97,9 @@ function mergeSignal(previous: E2eRiskSignal | null, current: E2eRiskSignal): E2 ) { throw new Error("risk signal metadata changed between Vitest invocations"); } + // Each call represents a separate Vitest command in the same job/shard; + // Vitest has already collapsed retries inside that command. Summing keeps + // failures sticky, because any failed or unhandled count makes the gate red. return { ...current, passed: previous.passed + current.passed, @@ -110,12 +117,8 @@ function mergeSignal(previous: E2eRiskSignal | null, current: E2eRiskSignal): E2 } function readPrevious(file: string): E2eRiskSignal | null { - if (!fs.existsSync(file)) return null; - const stat = fs.lstatSync(file); - if (!stat.isFile() || stat.isSymbolicLink()) { - throw new Error("risk signal output must be a regular file"); - } - return JSON.parse(fs.readFileSync(file, "utf8")) as E2eRiskSignal; + const contents = readPrivateRegularFile(file, { allowMissing: true, maxBytes: 64 * 1024 }); + return contents === null ? null : (JSON.parse(contents) as E2eRiskSignal); } export function writeRiskSignal( @@ -139,7 +142,7 @@ export function writeRiskSignal( fs.mkdirSync(environment.artifactDir, { recursive: true }); const file = path.join(environment.artifactDir, RISK_SIGNAL_FILE); const merged = mergeSignal(readPrevious(file), signal); - fs.writeFileSync(file, `${JSON.stringify(merged, null, 2)}\n`, { mode: 0o600 }); + writePrivateRegularFile(file, `${JSON.stringify(merged, null, 2)}\n`); return merged; } diff --git a/test/post-merge-e2e-risk-gate-workflow.test.ts b/test/post-merge-e2e-risk-gate-workflow.test.ts index 60c5764ea7e..5ed8cb1614d 100644 --- a/test/post-merge-e2e-risk-gate-workflow.test.ts +++ b/test/post-merge-e2e-risk-gate-workflow.test.ts @@ -16,6 +16,7 @@ const E2E_PATH = ".github/workflows/e2e.yaml"; type TriggeredWorkflow = Workflow & { on?: Record; permissions?: Record; + concurrency?: { group: string; "cancel-in-progress": boolean }; }; function step(job: WorkflowJob, name: string): WorkflowStep { @@ -59,6 +60,8 @@ describe("post-merge E2E risk gate shadow workflow", () => { const start = step(job, "Build plan and dispatch exact-commit E2E"); const wait = step(job, "Wait for correlated E2E run"); const finish = step(job, "Complete exact-commit shadow check"); + const completionFallback = step(job, "Close shadow check after completion failure"); + const propagateFailure = step(job, "Propagate shadow completion failure"); expect(checkout.with).toMatchObject({ ref: "${{ github.event.after }}", @@ -68,15 +71,24 @@ describe("post-merge E2E risk gate shadow workflow", () => { expect(start.run).toContain("post-merge-risk-gate.mts --mode start"); expect(start.run).toContain('--base "${{ github.event.before }}"'); expect(start.run).toContain('--commit "${{ github.event.after }}"'); - expect(wait.run).toContain("timeout --signal=TERM --kill-after=30s 100m"); + expect(wait.run).toContain("timeout --signal=TERM --kill-after=30s 105m"); + expect(finish.id).toBe("finish"); expect(finish.if).toContain("always()"); + expect(finish["continue-on-error"]).toBe(true); expect(finish.run).toContain("post-merge-risk-gate.mts --mode finish"); + expect(completionFallback.if).toContain("steps.finish.outcome == 'failure'"); + expect(completionFallback.run).toContain("post-merge-risk-gate.mts --mode abandon"); + expect(propagateFailure.if).toContain("steps.finish.outcome == 'failure'"); }); it("binds every E2E checkout and test signal to the exact merged commit", () => { - const workflow = readYaml; "run-name"?: string }>( - E2E_PATH, - ); + const workflow = readYaml< + Workflow & { + env?: Record; + "run-name"?: string; + concurrency?: { group: string; "cancel-in-progress": boolean }; + } + >(E2E_PATH); const allSteps = Object.values(workflow.jobs).flatMap((job) => job.steps ?? []); const checkouts = allSteps.filter((candidate) => candidate.uses?.startsWith("actions/checkout@"), @@ -86,6 +98,8 @@ describe("post-merge E2E risk gate shadow workflow", () => { .filter((run) => run.includes("npx vitest run --project e2e-live")); expect(workflow["run-name"]).toContain("inputs.risk_correlation"); + expect(workflow.concurrency?.group).not.toContain("inputs.risk_correlation"); + expect(workflow.concurrency?.group).toContain("inputs.risk_shadow && github.run_id"); expect(workflow.env).toMatchObject({ NEMOCLAW_E2E_EXPECTED_SHA: "${{ inputs.checkout_sha }}", NEMOCLAW_E2E_RISK_PLAN_HASH: "${{ inputs.risk_plan_hash }}", diff --git a/test/post-merge-e2e-risk-gate.test.ts b/test/post-merge-e2e-risk-gate.test.ts index 7b4a7428f2c..79c347cf152 100644 --- a/test/post-merge-e2e-risk-gate.test.ts +++ b/test/post-merge-e2e-risk-gate.test.ts @@ -13,7 +13,9 @@ import { changedFilesBetween, classifyRiskEvidence, expectedRiskSignalShards, + findSignalFiles, type RiskGateState, + validateRiskGateState, validateRiskPlan, validateSignal, } from "../tools/e2e-advisor/post-merge-risk-gate.mts"; @@ -124,6 +126,54 @@ describe("post-merge E2E risk gate", () => { expect(() => validateRiskPlan(plan, new Set())).toThrow(/unknown E2E job/u); }); + it("accepts only bounded gate state for this repository and exact child run", () => { + const gate = state(); + + expect(validateRiskGateState(gate, gate.repository)).toEqual(gate); + expect(() => + validateRiskGateState({ ...gate, repository: "other/repository" }, gate.repository), + ).toThrow(/repository/u); + expect(() => + validateRiskGateState({ ...gate, childRunUrl: "https://example.com/run" }, gate.repository), + ).toThrow(/URL/u); + expect(() => + validateRiskGateState({ ...gate, expectedJobs: ["../unsafe"] }, gate.repository), + ).toThrow(/expected jobs/u); + }); + + it("bounds downloaded risk-evidence traversal by entries, depth, and signals", () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-risk-evidence-")); + const rootLink = `${directory}-link`; + const nested = path.join(directory, "artifact", "live", "job"); + try { + fs.mkdirSync(nested, { recursive: true }); + const signalFile = path.join(nested, "risk-signal.json"); + fs.writeFileSync(signalFile, "{}\n"); + + expect(findSignalFiles(directory)).toEqual([signalFile]); + expect(() => + findSignalFiles(directory, { maxDepth: 2, maxEntries: 10, maxSignalFiles: 2 }), + ).toThrow(/depth limit/u); + expect(() => + findSignalFiles(directory, { maxDepth: 8, maxEntries: 2, maxSignalFiles: 2 }), + ).toThrow(/entry limit/u); + const second = path.join(directory, "artifact-2"); + fs.mkdirSync(second); + fs.writeFileSync(path.join(second, "risk-signal.json"), "{}\n"); + expect(() => + findSignalFiles(directory, { maxDepth: 8, maxEntries: 10, maxSignalFiles: 1 }), + ).toThrow(/signal-file limit/u); + expect(() => + findSignalFiles(directory, { maxDepth: 8, maxEntries: 10, maxSignalFiles: 0 }), + ).toThrow(/limits are invalid/u); + fs.symlinkSync(directory, rootLink, "dir"); + expect(() => findSignalFiles(rootLink)).toThrow(/root must be a directory, not a symlink/u); + } finally { + fs.rmSync(rootLink, { force: true }); + fs.rmSync(directory, { recursive: true, force: true }); + } + }); + it("accepts only signals bound to the expected job, SHA, plan, and correlation", () => { const gate = state(); diff --git a/tools/advisors/github.mts b/tools/advisors/github.mts index 1b786ce1462..762e0e7732e 100644 --- a/tools/advisors/github.mts +++ b/tools/advisors/github.mts @@ -80,9 +80,9 @@ export async function githubApi( token: string, options: GitHubRequestOptions = {}, ): Promise { - // lgtm[js/file-access-to-http] Advisor workflows intentionally send normalized - // artifact summaries and strictly validated dispatch inputs to GitHub APIs. - // Callers construct apiPath from fixed workflow/comment endpoints, not PR text. + // Advisor workflows intentionally send normalized artifact summaries and + // strictly validated dispatch inputs to fixed GitHub API endpoints. + // codeql[js/file-access-to-http] const response = await fetch(`https://api.github.com/${apiPath}`, { method: options.method || "GET", headers: { @@ -92,6 +92,8 @@ export async function githubApi( "X-GitHub-Api-Version": "2022-11-28", ...(options.userAgent ? { "User-Agent": options.userAgent } : {}), }, + // Sending reviewed file-derived summaries is this adapter's purpose. + // codeql[js/file-access-to-http] body: options.body === undefined ? undefined : JSON.stringify(options.body), }); const text = await response.text(); diff --git a/tools/e2e-advisor/post-merge-risk-gate.mts b/tools/e2e-advisor/post-merge-risk-gate.mts index ddc38026c21..d6a7c9e51de 100755 --- a/tools/e2e-advisor/post-merge-risk-gate.mts +++ b/tools/e2e-advisor/post-merge-risk-gate.mts @@ -15,6 +15,7 @@ import { githubApi } from "../advisors/github.mts"; import { parseArgs } from "../advisors/io.mts"; import { buildRiskPlan, type RiskPlan } from "../advisors/risk-plan.mts"; import { readFreeStandingJobsInventory } from "../e2e/workflow-boundary.mts"; +import { readPrivateRegularFile, writePrivateRegularFile } from "./private-file.ts"; import type { E2eRiskSignal } from "./risk-signal.ts"; const E2E_WORKFLOW = "e2e.yaml"; @@ -26,6 +27,11 @@ const SHARD_PATTERN = /^(?:default|[A-Za-z0-9][A-Za-z0-9_-]*)$/u; const CORRELATION_PATTERN = /^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/u; const MAX_PLAN_BYTES = 1024 * 1024; +const DEFAULT_EVIDENCE_LIMITS = { + maxDepth: 8, + maxEntries: 4096, + maxSignalFiles: 12, +} as const; type CheckConclusion = "success" | "failure" | "neutral"; @@ -83,10 +89,63 @@ function isRecord(value: unknown): value is Record { } function readRegularJson(file: string, maxBytes = MAX_PLAN_BYTES): unknown { - const stat = fs.lstatSync(file); - if (!stat.isFile() || stat.isSymbolicLink()) throw new Error(`${file} must be a regular file`); - if (stat.size > maxBytes) throw new Error(`${file} exceeds ${maxBytes} bytes`); - return JSON.parse(fs.readFileSync(file, "utf8")); + return JSON.parse(readPrivateRegularFile(file, { maxBytes })!); +} + +export function validateRiskGateState(value: unknown, expectedRepository: string): RiskGateState { + if (!isRecord(value) || value.version !== 1) throw new Error("invalid risk-gate state version"); + if ( + value.repository !== expectedRepository || + !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(expectedRepository) + ) { + throw new Error("risk-gate state repository does not match this workflow"); + } + if (typeof value.commitSha !== "string" || !SHA_PATTERN.test(value.commitSha)) { + throw new Error("risk-gate state commit SHA is invalid"); + } + if (typeof value.planHash !== "string" || !HASH_PATTERN.test(value.planHash)) { + throw new Error("risk-gate state plan hash is invalid"); + } + if (typeof value.correlationId !== "string" || !CORRELATION_PATTERN.test(value.correlationId)) { + throw new Error("risk-gate state correlation id is invalid"); + } + if ( + !Array.isArray(value.expectedJobs) || + value.expectedJobs.length < 1 || + value.expectedJobs.length > 3 || + !value.expectedJobs.every((job) => typeof job === "string" && JOB_PATTERN.test(job)) || + new Set(value.expectedJobs).size !== value.expectedJobs.length + ) { + throw new Error("risk-gate state expected jobs are invalid"); + } + if (!isRecord(value.expectedShards)) throw new Error("risk-gate state shards are invalid"); + const shardJobs = Object.keys(value.expectedShards).sort(); + if (JSON.stringify(shardJobs) !== JSON.stringify([...value.expectedJobs].sort())) { + throw new Error("risk-gate state shard jobs do not match expected jobs"); + } + for (const job of value.expectedJobs) { + const shards = value.expectedShards[job]; + if ( + !Array.isArray(shards) || + shards.length < 1 || + new Set(shards).size !== shards.length || + !shards.every((shard) => typeof shard === "string" && SHARD_PATTERN.test(shard)) + ) { + throw new Error(`risk-gate state shards are invalid for ${job}`); + } + } + if (typeof value.requiresManualExpansion !== "boolean") { + throw new Error("risk-gate manual-expansion state is invalid"); + } + if (!Number.isSafeInteger(value.checkRunId) || Number(value.checkRunId) < 1) { + throw new Error("risk-gate check id is invalid"); + } + if (!Number.isSafeInteger(value.childRunId) || Number(value.childRunId) < 1) { + throw new Error("risk-gate child run id is invalid"); + } + const expectedUrl = `https://github.com/${expectedRepository}/actions/runs/${value.childRunId}`; + if (value.childRunUrl !== expectedUrl) throw new Error("risk-gate child run URL is invalid"); + return value as RiskGateState; } export function validateRiskPlan(value: unknown, allowedJobs: ReadonlySet): RiskPlan { @@ -207,6 +266,9 @@ export function classifyRiskEvidence(options: { } const missing = expectedEvidence.filter((key) => !byJobShard.has(key)); if (missing.length > 0) { + // Missing bound evidence is unverifiable, not proof that product behavior + // failed. Shadow checks become green only for complete evidence; neutral + // keeps incomplete infrastructure evidence from masquerading as a pass. return { conclusion: "neutral", title: "Selected E2E jobs are missing test evidence", @@ -254,7 +316,26 @@ export function classifyRiskEvidence(options: { function appendOutput(name: string, value: string): void { const output = process.env.GITHUB_OUTPUT; - if (output) fs.appendFileSync(output, `${name}=${value}\n`); + if (!output) return; + if (!/^(?:check_id|dispatched|finalized|run_id)$/u.test(name)) { + throw new Error("invalid controller output name"); + } + if (!/^(?:true|false|[1-9][0-9]*)$/u.test(value)) { + throw new Error("invalid controller output value"); + } + const descriptor = fs.openSync( + output, + fs.constants.O_WRONLY | fs.constants.O_APPEND | (fs.constants.O_NOFOLLOW ?? 0), + ); + try { + if (!fs.fstatSync(descriptor).isFile()) throw new Error("GITHUB_OUTPUT must be a regular file"); + // GitHub supplies this output file; values are restricted above to fixed + // booleans or positive decimal IDs before the descriptor write. + // codeql[js/http-to-file-access] + fs.writeFileSync(descriptor, `${name}=${value}\n`, "utf8"); + } finally { + fs.closeSync(descriptor); + } } async function createCheck( @@ -274,6 +355,8 @@ async function createCheck( }, userAgent: "nemoclaw-e2e-risk-gate", }); + if (!Number.isSafeInteger(check.id) || check.id < 1) + throw new Error("GitHub returned an invalid check id"); return check.id; } @@ -414,7 +497,14 @@ async function findCorrelatedRun( ); const match = payload.workflow_runs.find( (run) => - run.display_title === expectedTitle && Date.parse(run.created_at) >= dispatchedAt - 5000, + run.display_title === expectedTitle && + run.name === "E2E" && + run.event === "workflow_dispatch" && + SHA_PATTERN.test(run.head_sha) && + Number.isSafeInteger(run.id) && + run.id > 0 && + run.html_url === `https://github.com/${repository}/actions/runs/${run.id}` && + Date.parse(run.created_at) >= dispatchedAt - 5000, ); if (match) return match; await new Promise((resolve) => setTimeout(resolve, 5000)); @@ -440,6 +530,9 @@ async function start(options: { commitSha: options.commitSha, }); + // The inventory and controller are both read from the exact trusted main + // commit. A second copied allowlist would drift without adding a trust + // boundary: compromising this inventory already means compromising main. const allowedJobs = new Set(readFreeStandingJobsInventory().allowedJobs); const plan = validateRiskPlan( buildRiskPlan({ @@ -448,7 +541,7 @@ async function start(options: { }), allowedJobs, ); - fs.writeFileSync(options.planPath, `${JSON.stringify(plan, null, 2)}\n`, { mode: 0o600 }); + writePrivateRegularFile(options.planPath, `${JSON.stringify(plan, null, 2)}\n`); const checkRunId = await createCheck( repository, @@ -504,7 +597,7 @@ async function start(options: { childRunId: child.id, childRunUrl: child.html_url, }; - fs.writeFileSync(options.statePath, `${JSON.stringify(state, null, 2)}\n`, { mode: 0o600 }); + writePrivateRegularFile(options.statePath, `${JSON.stringify(state, null, 2)}\n`); appendOutput("dispatched", "true"); appendOutput("run_id", String(child.id)); } catch (error) { @@ -518,25 +611,66 @@ async function start(options: { } } -function findSignalFiles(root: string): string[] { +export function findSignalFiles( + root: string, + limits: { + maxDepth: number; + maxEntries: number; + maxSignalFiles: number; + } = DEFAULT_EVIDENCE_LIMITS, +): string[] { if (!fs.existsSync(root)) return []; + if ( + !Number.isSafeInteger(limits.maxDepth) || + limits.maxDepth < 0 || + !Number.isSafeInteger(limits.maxEntries) || + limits.maxEntries < 1 || + !Number.isSafeInteger(limits.maxSignalFiles) || + limits.maxSignalFiles < 1 + ) { + throw new Error("risk evidence traversal limits are invalid"); + } + const rootStat = fs.lstatSync(root); + if (!rootStat.isDirectory() || rootStat.isSymbolicLink()) { + throw new Error("risk evidence root must be a directory, not a symlink"); + } const files: string[] = []; - const visit = (directory: string): void => { - for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { - const full = path.join(directory, entry.name); - if (entry.isSymbolicLink()) throw new Error("risk evidence must not contain symlinks"); - if (entry.isDirectory()) visit(full); - else if (entry.isFile() && entry.name === "risk-signal.json") files.push(full); + let entriesVisited = 0; + const visit = (directory: string, depth: number): void => { + const handle = fs.opendirSync(directory); + try { + let entry = handle.readSync(); + while (entry !== null) { + entriesVisited += 1; + if (entriesVisited > limits.maxEntries) { + throw new Error("risk evidence exceeds the entry limit"); + } + const full = path.join(directory, entry.name); + if (entry.isSymbolicLink()) throw new Error("risk evidence must not contain symlinks"); + if (entry.isDirectory()) { + if (depth >= limits.maxDepth) throw new Error("risk evidence exceeds the depth limit"); + visit(full, depth + 1); + } else if (entry.isFile() && entry.name === "risk-signal.json") { + files.push(full); + if (files.length > limits.maxSignalFiles) { + throw new Error("risk evidence exceeds the signal-file limit"); + } + } + entry = handle.readSync(); + } + } finally { + handle.closeSync(); } }; - visit(root); + visit(root, 0); return files.sort((left, right) => left.localeCompare(right)); } async function finish(options: { statePath: string; evidencePath: string }): Promise { const token = process.env.GITHUB_TOKEN ?? ""; - const state = readRegularJson(options.statePath) as RiskGateState; - if (!token || state.version !== 1 || !SHA_PATTERN.test(state.commitSha)) { + const repository = process.env.GITHUB_REPOSITORY ?? ""; + const state = validateRiskGateState(readRegularJson(options.statePath), repository); + if (!token) { throw new Error("valid gate state and GITHUB_TOKEN are required"); } try { @@ -549,6 +683,8 @@ async function finish(options: { statePath: string; evidencePath: string }): Pro child.id !== state.childRunId || child.name !== "E2E" || child.event !== "workflow_dispatch" || + !SHA_PATTERN.test(child.head_sha) || + child.html_url !== state.childRunUrl || child.display_title !== `E2E risk ${state.correlationId}` ) { throw new Error("correlated E2E workflow identity changed"); diff --git a/tools/e2e-advisor/private-file.ts b/tools/e2e-advisor/private-file.ts new file mode 100644 index 00000000000..fa6b0662acf --- /dev/null +++ b/tools/e2e-advisor/private-file.ts @@ -0,0 +1,56 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; + +const NO_FOLLOW = fs.constants.O_NOFOLLOW ?? 0; +const NON_BLOCK = fs.constants.O_NONBLOCK ?? 0; + +export function readPrivateRegularFile( + file: string, + options: { allowMissing?: boolean; maxBytes: number }, +): string | null { + let descriptor: number; + try { + descriptor = fs.openSync(file, fs.constants.O_RDONLY | NO_FOLLOW | NON_BLOCK); + } catch (error) { + if (options.allowMissing && (error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } + + try { + const stat = fs.fstatSync(descriptor); + if (!stat.isFile() || stat.nlink !== 1) { + throw new Error(`${file} must be a private regular file`); + } + if (stat.size > options.maxBytes) { + throw new Error(`${file} exceeds ${options.maxBytes} bytes`); + } + return fs.readFileSync(descriptor, "utf8"); + } finally { + fs.closeSync(descriptor); + } +} + +export function writePrivateRegularFile(file: string, contents: string): void { + const descriptor = fs.openSync( + file, + fs.constants.O_WRONLY | fs.constants.O_CREAT | NO_FOLLOW | NON_BLOCK, + 0o600, + ); + try { + const stat = fs.fstatSync(descriptor); + if (!stat.isFile() || stat.nlink !== 1) { + throw new Error(`${file} must be a private regular file`); + } + fs.fchmodSync(descriptor, 0o600); + fs.ftruncateSync(descriptor, 0); + // The caller validates and bounds controller state before this private, + // no-follow descriptor write. Persisting that state is intentional. + // codeql[js/http-to-file-access] + fs.writeFileSync(descriptor, contents, "utf8"); + fs.fsyncSync(descriptor); + } finally { + fs.closeSync(descriptor); + } +} From f15e4686f68550528b3387f5bc6c7d7b656b3b01 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Wed, 8 Jul 2026 00:50:49 -0700 Subject: [PATCH 4/9] fix(ci): secure risk gate workspace Signed-off-by: Carlos Villela --- .../post-merge-e2e-risk-gate-shadow.yaml | 25 ++++-- test/e2e-private-file.test.ts | 2 + .../post-merge-e2e-risk-gate-workflow.test.ts | 11 +++ test/post-merge-e2e-risk-gate.test.ts | 37 ++++++++ tools/advisors/github.mts | 11 +-- tools/e2e-advisor/post-merge-risk-gate.mts | 85 ++++++++++++++++--- tools/e2e-advisor/private-file.ts | 25 ++++-- 7 files changed, 161 insertions(+), 35 deletions(-) diff --git a/.github/workflows/post-merge-e2e-risk-gate-shadow.yaml b/.github/workflows/post-merge-e2e-risk-gate-shadow.yaml index 769928d21c7..f8a2526ac61 100644 --- a/.github/workflows/post-merge-e2e-risk-gate-shadow.yaml +++ b/.github/workflows/post-merge-e2e-risk-gate-shadow.yaml @@ -39,6 +39,15 @@ jobs: - name: Install trusted controller dependencies run: npm ci --ignore-scripts + - id: workspace + name: Create private controller workspace + shell: bash + run: | + set -euo pipefail + work_dir="$(mktemp -d "${RUNNER_TEMP}/nemoclaw-e2e-risk-gate.XXXXXX")" + chmod 700 "$work_dir" + printf 'work_dir=%s\n' "$work_dir" >> "$GITHUB_OUTPUT" + - id: start name: Build plan and dispatch exact-commit E2E continue-on-error: true @@ -49,15 +58,14 @@ jobs: --mode start --base "${{ github.event.before }}" --commit "${{ github.event.after }}" - --plan /tmp/post-merge-risk-plan.json - --state /tmp/e2e-risk-gate-state.json + --work-dir "${{ steps.workspace.outputs.work_dir }}" - name: Upload post-merge risk plan if: ${{ always() }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: post-merge-risk-plan-${{ github.event.after }} - path: /tmp/post-merge-risk-plan.json + path: ${{ steps.workspace.outputs.work_dir }}/post-merge-risk-plan.json if-no-files-found: ignore retention-days: 14 @@ -90,7 +98,9 @@ jobs: env: GH_TOKEN: ${{ github.token }} RUN_ID: ${{ steps.start.outputs.run_id }} - run: gh run download "$RUN_ID" --repo "$GITHUB_REPOSITORY" --dir /tmp/e2e-risk-evidence + run: >- + gh run download "$RUN_ID" --repo "$GITHUB_REPOSITORY" + --dir "${{ steps.workspace.outputs.work_dir }}/evidence" - id: finish name: Complete exact-commit shadow check @@ -101,8 +111,7 @@ jobs: run: >- node --experimental-strip-types tools/e2e-advisor/post-merge-risk-gate.mts --mode finish - --state /tmp/e2e-risk-gate-state.json - --evidence /tmp/e2e-risk-evidence + --work-dir "${{ steps.workspace.outputs.work_dir }}" - name: Close shadow check after completion failure if: ${{ always() && steps.start.outputs.check_id != '' && steps.start.outputs.dispatched == 'true' && steps.finish.outcome == 'failure' }} @@ -116,3 +125,7 @@ jobs: - name: Propagate shadow completion failure if: ${{ steps.finish.outcome == 'failure' }} run: exit 1 + + - name: Remove private controller workspace + if: ${{ always() && steps.workspace.outputs.work_dir != '' }} + run: rm -rf -- "${{ steps.workspace.outputs.work_dir }}" diff --git a/test/e2e-private-file.test.ts b/test/e2e-private-file.test.ts index 48ab24aea39..fe4c54c8645 100644 --- a/test/e2e-private-file.test.ts +++ b/test/e2e-private-file.test.ts @@ -24,6 +24,8 @@ describe("private E2E controller files", () => { writePrivateRegularFile(regular, "regular\n"); expect(readPrivateRegularFile(regular, { maxBytes: 64 })).toBe("regular\n"); expect(fs.statSync(regular).mode & 0o777).toBe(0o600); + writePrivateRegularFile(regular, "updated\n"); + expect(readPrivateRegularFile(regular, { maxBytes: 64 })).toBe("updated\n"); fs.writeFileSync(target, "protected\n"); fs.symlinkSync(target, symlink); diff --git a/test/post-merge-e2e-risk-gate-workflow.test.ts b/test/post-merge-e2e-risk-gate-workflow.test.ts index 5ed8cb1614d..680f4971753 100644 --- a/test/post-merge-e2e-risk-gate-workflow.test.ts +++ b/test/post-merge-e2e-risk-gate-workflow.test.ts @@ -57,28 +57,39 @@ describe("post-merge E2E risk gate shadow workflow", () => { const workflow = readYaml(SHADOW_PATH); const job = workflow.jobs.shadow; const checkout = step(job, "Checkout trusted controller"); + const workspace = step(job, "Create private controller workspace"); const start = step(job, "Build plan and dispatch exact-commit E2E"); const wait = step(job, "Wait for correlated E2E run"); + const download = step(job, "Download correlated E2E evidence"); const finish = step(job, "Complete exact-commit shadow check"); const completionFallback = step(job, "Close shadow check after completion failure"); const propagateFailure = step(job, "Propagate shadow completion failure"); + const cleanup = step(job, "Remove private controller workspace"); expect(checkout.with).toMatchObject({ ref: "${{ github.event.after }}", "fetch-depth": 0, "persist-credentials": false, }); + expect(workspace.run).toContain('mktemp -d "${RUNNER_TEMP}/nemoclaw-e2e-risk-gate.XXXXXX"'); + expect(workspace.run).toContain('chmod 700 "$work_dir"'); expect(start.run).toContain("post-merge-risk-gate.mts --mode start"); expect(start.run).toContain('--base "${{ github.event.before }}"'); expect(start.run).toContain('--commit "${{ github.event.after }}"'); + expect(start.run).toContain('--work-dir "${{ steps.workspace.outputs.work_dir }}"'); expect(wait.run).toContain("timeout --signal=TERM --kill-after=30s 105m"); + expect(download.run).toContain('--dir "${{ steps.workspace.outputs.work_dir }}/evidence"'); expect(finish.id).toBe("finish"); expect(finish.if).toContain("always()"); expect(finish["continue-on-error"]).toBe(true); expect(finish.run).toContain("post-merge-risk-gate.mts --mode finish"); + expect(finish.run).toContain('--work-dir "${{ steps.workspace.outputs.work_dir }}"'); expect(completionFallback.if).toContain("steps.finish.outcome == 'failure'"); expect(completionFallback.run).toContain("post-merge-risk-gate.mts --mode abandon"); expect(propagateFailure.if).toContain("steps.finish.outcome == 'failure'"); + expect(cleanup.if).toContain("always() && steps.workspace.outputs.work_dir != ''"); + expect(cleanup.run).toContain('rm -rf -- "${{ steps.workspace.outputs.work_dir }}"'); + expect(collectStrings(workflow).some((value) => value.includes("/tmp/"))).toBe(false); }); it("binds every E2E checkout and test signal to the exact merged commit", () => { diff --git a/test/post-merge-e2e-risk-gate.test.ts b/test/post-merge-e2e-risk-gate.test.ts index 79c347cf152..98ae9ec95ef 100644 --- a/test/post-merge-e2e-risk-gate.test.ts +++ b/test/post-merge-e2e-risk-gate.test.ts @@ -14,6 +14,7 @@ import { classifyRiskEvidence, expectedRiskSignalShards, findSignalFiles, + parseControllerCommand, type RiskGateState, validateRiskGateState, validateRiskPlan, @@ -84,6 +85,42 @@ describe("post-merge E2E risk gate", () => { ); }); + it("requires a private controller workspace and parses the abandon check id", () => { + const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-risk-controller-")); + try { + expect( + parseControllerCommand([ + "--mode", + "start", + "--base", + "b".repeat(40), + "--commit", + HEAD_SHA, + "--work-dir", + workDir, + ]), + ).toMatchObject({ + mode: "start", + planPath: path.join(workDir, "post-merge-risk-plan.json"), + statePath: path.join(workDir, "e2e-risk-gate-state.json"), + evidencePath: path.join(workDir, "evidence"), + }); + expect(parseControllerCommand(["--mode", "abandon", "--check-id", "17"])).toEqual({ + mode: "abandon", + checkId: "17", + }); + expect(() => parseControllerCommand(["--mode", "finish"])).toThrow(/--work-dir/u); + + fs.chmodSync(workDir, 0o755); + expect(() => parseControllerCommand(["--mode", "finish", "--work-dir", workDir])).toThrow( + /owned private absolute directory/u, + ); + } finally { + fs.chmodSync(workDir, 0o700); + fs.rmSync(workDir, { recursive: true, force: true }); + } + }); + it("derives changed files from an exact checked-out commit range", () => { const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-risk-git-")); const git = (...args: string[]) => diff --git a/tools/advisors/github.mts b/tools/advisors/github.mts index 762e0e7732e..7a362614f41 100644 --- a/tools/advisors/github.mts +++ b/tools/advisors/github.mts @@ -80,10 +80,9 @@ export async function githubApi( token: string, options: GitHubRequestOptions = {}, ): Promise { - // Advisor workflows intentionally send normalized artifact summaries and - // strictly validated dispatch inputs to fixed GitHub API endpoints. - // codeql[js/file-access-to-http] - const response = await fetch(`https://api.github.com/${apiPath}`, { + const apiUrl = `https://api.github.com/${apiPath}`; + // biome-ignore format: Keep the CodeQL suppression on the alert sink. + const response = await fetch(apiUrl, { // codeql[js/file-access-to-http]: fixed GitHub API endpoint with a reviewed payload. method: options.method || "GET", headers: { Accept: "application/vnd.github+json", @@ -92,9 +91,7 @@ export async function githubApi( "X-GitHub-Api-Version": "2022-11-28", ...(options.userAgent ? { "User-Agent": options.userAgent } : {}), }, - // Sending reviewed file-derived summaries is this adapter's purpose. - // codeql[js/file-access-to-http] - body: options.body === undefined ? undefined : JSON.stringify(options.body), + body: options.body === undefined ? undefined : JSON.stringify(options.body), // codeql[js/file-access-to-http]: reviewed summaries and validated controller state go only to fixed GitHub API endpoints. }); const text = await response.text(); if (!response.ok) { diff --git a/tools/e2e-advisor/post-merge-risk-gate.mts b/tools/e2e-advisor/post-merge-risk-gate.mts index d6a7c9e51de..86d2b15aee0 100755 --- a/tools/e2e-advisor/post-merge-risk-gate.mts +++ b/tools/e2e-advisor/post-merge-risk-gate.mts @@ -33,6 +33,17 @@ const DEFAULT_EVIDENCE_LIMITS = { maxSignalFiles: 12, } as const; +type ControllerPaths = { + planPath: string; + statePath: string; + evidencePath: string; +}; + +export type ControllerCommand = + | ({ mode: "start"; baseSha: string; commitSha: string } & ControllerPaths) + | ({ mode: "finish" } & ControllerPaths) + | { mode: "abandon"; checkId: string }; + type CheckConclusion = "success" | "failure" | "neutral"; type WorkflowRun = { @@ -88,6 +99,56 @@ function isRecord(value: unknown): value is Record { return !!value && typeof value === "object" && !Array.isArray(value); } +function requiredArgument(value: string | undefined, name: string): string { + if (!value) throw new Error(`--${name} is required`); + return value; +} + +export function privateControllerPaths(workDir: string): ControllerPaths { + const resolved = path.resolve(workDir); + const stat = fs.lstatSync(resolved); + const currentUid = typeof process.getuid === "function" ? process.getuid() : null; + if ( + resolved !== workDir || + !stat.isDirectory() || + stat.isSymbolicLink() || + (stat.mode & 0o077) !== 0 || + (currentUid !== null && stat.uid !== currentUid) + ) { + throw new Error("--work-dir must be an owned private absolute directory"); + } + return { + planPath: path.join(resolved, "post-merge-risk-plan.json"), + statePath: path.join(resolved, "e2e-risk-gate-state.json"), + evidencePath: path.join(resolved, "evidence"), + }; +} + +export function parseControllerCommand(argv: string[]): ControllerCommand { + const args = parseArgs(argv); + if (args.mode === "start") { + return { + mode: "start", + baseSha: requiredArgument(args.base, "base"), + commitSha: requiredArgument(args.commit, "commit"), + ...privateControllerPaths(requiredArgument(args.workDir, "work-dir")), + }; + } + if (args.mode === "finish") { + return { + mode: "finish", + ...privateControllerPaths(requiredArgument(args.workDir, "work-dir")), + }; + } + if (args.mode === "abandon") { + return { + mode: "abandon", + checkId: requiredArgument(args.checkId, "check-id"), + }; + } + throw new Error("--mode must be start, finish, or abandon"); +} + function readRegularJson(file: string, maxBytes = MAX_PLAN_BYTES): unknown { return JSON.parse(readPrivateRegularFile(file, { maxBytes })!); } @@ -733,29 +794,27 @@ async function abandon(checkId: string): Promise { } async function main(): Promise { - const args = parseArgs(process.argv.slice(2)); - const mode = args.mode; - if (mode === "start") { + const command = parseControllerCommand(process.argv.slice(2)); + if (command.mode === "start") { await start({ - baseSha: args.base || "", - commitSha: args.commit || "", - planPath: args.plan || "/tmp/post-merge-risk-plan.json", - statePath: args.state || "/tmp/e2e-risk-gate-state.json", + baseSha: command.baseSha, + commitSha: command.commitSha, + planPath: command.planPath, + statePath: command.statePath, }); return; } - if (mode === "finish") { + if (command.mode === "finish") { await finish({ - statePath: args.state || "/tmp/e2e-risk-gate-state.json", - evidencePath: args.evidence || "/tmp/e2e-risk-evidence", + statePath: command.statePath, + evidencePath: command.evidencePath, }); return; } - if (mode === "abandon") { - await abandon(args["check-id"] || ""); + if (command.mode === "abandon") { + await abandon(command.checkId); return; } - throw new Error("--mode must be start, finish, or abandon"); } if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { diff --git a/tools/e2e-advisor/private-file.ts b/tools/e2e-advisor/private-file.ts index fa6b0662acf..3190b93cd5c 100644 --- a/tools/e2e-advisor/private-file.ts +++ b/tools/e2e-advisor/private-file.ts @@ -6,6 +6,19 @@ import fs from "node:fs"; const NO_FOLLOW = fs.constants.O_NOFOLLOW ?? 0; const NON_BLOCK = fs.constants.O_NONBLOCK ?? 0; +function openPrivateFileForWrite(file: string): number { + try { + return fs.openSync( + file, + fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | NO_FOLLOW | NON_BLOCK, + 0o600, + ); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + return fs.openSync(file, fs.constants.O_WRONLY | NO_FOLLOW | NON_BLOCK); + } +} + export function readPrivateRegularFile( file: string, options: { allowMissing?: boolean; maxBytes: number }, @@ -33,11 +46,7 @@ export function readPrivateRegularFile( } export function writePrivateRegularFile(file: string, contents: string): void { - const descriptor = fs.openSync( - file, - fs.constants.O_WRONLY | fs.constants.O_CREAT | NO_FOLLOW | NON_BLOCK, - 0o600, - ); + const descriptor = openPrivateFileForWrite(file); try { const stat = fs.fstatSync(descriptor); if (!stat.isFile() || stat.nlink !== 1) { @@ -45,10 +54,8 @@ export function writePrivateRegularFile(file: string, contents: string): void { } fs.fchmodSync(descriptor, 0o600); fs.ftruncateSync(descriptor, 0); - // The caller validates and bounds controller state before this private, - // no-follow descriptor write. Persisting that state is intentional. - // codeql[js/http-to-file-access] - fs.writeFileSync(descriptor, contents, "utf8"); + // Callers validate and bound state before this exclusive/no-follow write. + fs.writeFileSync(descriptor, contents, "utf8"); // codeql[js/http-to-file-access]: intentional validated controller state persistence. fs.fsyncSync(descriptor); } finally { fs.closeSync(descriptor); From d757888152b7dddd7fa9632822646e3d33f1eb1f Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Wed, 8 Jul 2026 01:05:24 -0700 Subject: [PATCH 5/9] fix(ci): isolate risk gate network state Signed-off-by: Carlos Villela --- .../post-merge-e2e-risk-gate-shadow.yaml | 2 + .../post-merge-e2e-risk-gate-workflow.test.ts | 2 + test/post-merge-e2e-risk-gate.test.ts | 44 ++++--- tools/advisors/github.mts | 9 +- tools/e2e-advisor/post-merge-risk-gate.mts | 109 ++++++++---------- tools/e2e-advisor/private-file.ts | 3 +- 6 files changed, 89 insertions(+), 80 deletions(-) diff --git a/.github/workflows/post-merge-e2e-risk-gate-shadow.yaml b/.github/workflows/post-merge-e2e-risk-gate-shadow.yaml index f8a2526ac61..007969b19c1 100644 --- a/.github/workflows/post-merge-e2e-risk-gate-shadow.yaml +++ b/.github/workflows/post-merge-e2e-risk-gate-shadow.yaml @@ -112,6 +112,8 @@ jobs: node --experimental-strip-types tools/e2e-advisor/post-merge-risk-gate.mts --mode finish --work-dir "${{ steps.workspace.outputs.work_dir }}" + --check-id "${{ steps.start.outputs.check_id }}" + --run-id "${{ steps.start.outputs.run_id }}" - name: Close shadow check after completion failure if: ${{ always() && steps.start.outputs.check_id != '' && steps.start.outputs.dispatched == 'true' && steps.finish.outcome == 'failure' }} diff --git a/test/post-merge-e2e-risk-gate-workflow.test.ts b/test/post-merge-e2e-risk-gate-workflow.test.ts index 680f4971753..f2f49875366 100644 --- a/test/post-merge-e2e-risk-gate-workflow.test.ts +++ b/test/post-merge-e2e-risk-gate-workflow.test.ts @@ -84,6 +84,8 @@ describe("post-merge E2E risk gate shadow workflow", () => { expect(finish["continue-on-error"]).toBe(true); expect(finish.run).toContain("post-merge-risk-gate.mts --mode finish"); expect(finish.run).toContain('--work-dir "${{ steps.workspace.outputs.work_dir }}"'); + expect(finish.run).toContain('--check-id "${{ steps.start.outputs.check_id }}"'); + expect(finish.run).toContain('--run-id "${{ steps.start.outputs.run_id }}"'); expect(completionFallback.if).toContain("steps.finish.outcome == 'failure'"); expect(completionFallback.run).toContain("post-merge-risk-gate.mts --mode abandon"); expect(propagateFailure.if).toContain("steps.finish.outcome == 'failure'"); diff --git a/test/post-merge-e2e-risk-gate.test.ts b/test/post-merge-e2e-risk-gate.test.ts index 98ae9ec95ef..1284923cbe0 100644 --- a/test/post-merge-e2e-risk-gate.test.ts +++ b/test/post-merge-e2e-risk-gate.test.ts @@ -28,7 +28,6 @@ const ALLOWED = new Set(["onboard-repair", "onboard-resume"]); function state(): RiskGateState { return { version: 1, - repository: "NVIDIA/NemoClaw", commitSha: HEAD_SHA, planHash: buildRiskPlan({ headSha: HEAD_SHA, changedFiles: ["src/lib/onboard.ts"] }).planHash, correlationId: "12345678-1234-4123-8123-123456789abc", @@ -38,9 +37,6 @@ function state(): RiskGateState { "onboard-resume": ["default"], }, requiresManualExpansion: false, - checkRunId: 1, - childRunId: 2, - childRunUrl: "https://github.com/NVIDIA/NemoClaw/actions/runs/2", }; } @@ -107,8 +103,23 @@ describe("post-merge E2E risk gate", () => { }); expect(parseControllerCommand(["--mode", "abandon", "--check-id", "17"])).toEqual({ mode: "abandon", - checkId: "17", + checkRunId: 17, }); + expect( + parseControllerCommand([ + "--mode", + "finish", + "--work-dir", + workDir, + "--check-id", + "17", + "--run-id", + "23", + ]), + ).toMatchObject({ mode: "finish", checkRunId: 17, childRunId: 23 }); + expect(() => + parseControllerCommand(["--mode", "abandon", "--check-id", "9007199254740992"]), + ).toThrow(/safe integer range/u); expect(() => parseControllerCommand(["--mode", "finish"])).toThrow(/--work-dir/u); fs.chmodSync(workDir, 0o755); @@ -163,19 +174,15 @@ describe("post-merge E2E risk gate", () => { expect(() => validateRiskPlan(plan, new Set())).toThrow(/unknown E2E job/u); }); - it("accepts only bounded gate state for this repository and exact child run", () => { + it("accepts only bounded gate state for the exact commit and evidence policy", () => { const gate = state(); - expect(validateRiskGateState(gate, gate.repository)).toEqual(gate); - expect(() => - validateRiskGateState({ ...gate, repository: "other/repository" }, gate.repository), - ).toThrow(/repository/u); - expect(() => - validateRiskGateState({ ...gate, childRunUrl: "https://example.com/run" }, gate.repository), - ).toThrow(/URL/u); - expect(() => - validateRiskGateState({ ...gate, expectedJobs: ["../unsafe"] }, gate.repository), - ).toThrow(/expected jobs/u); + expect(validateRiskGateState(gate)).toEqual(gate); + expect(() => validateRiskGateState({ ...gate, commitSha: "unsafe" })).toThrow(/commit SHA/u); + expect(() => validateRiskGateState({ ...gate, expectedJobs: ["../unsafe"] })).toThrow( + /expected jobs/u, + ); + expect(() => validateRiskGateState({ ...gate, expectedShards: {} })).toThrow(/shard jobs/u); }); it("bounds downloaded risk-evidence traversal by entries, depth, and signals", () => { @@ -244,6 +251,7 @@ describe("post-merge E2E risk gate", () => { }); expect(verdict.conclusion).toBe("success"); + expect(verdict.summary).not.toContain("onboard-repair"); }); it.each([ @@ -277,6 +285,7 @@ describe("post-merge E2E risk gate", () => { }); expect(verdict.conclusion).toBe("neutral"); + expect(verdict.summary).not.toContain("onboard-repair"); }); it("reports a product workflow failure as failure", () => { @@ -289,6 +298,7 @@ describe("post-merge E2E risk gate", () => { }); expect(verdict.conclusion).toBe("failure"); + expect(verdict.summary).not.toContain("onboard-repair"); }); it("reports failed exact-SHA test evidence as failure even when the workflow is green", () => { @@ -301,6 +311,7 @@ describe("post-merge E2E risk gate", () => { }); expect(verdict.conclusion).toBe("failure"); + expect(verdict.summary).not.toContain("onboard-repair"); }); it("requires every expected matrix shard to pass", () => { @@ -324,5 +335,6 @@ describe("post-merge E2E risk gate", () => { expect(complete.conclusion).toBe("success"); expect(missingShard.conclusion).toBe("neutral"); + expect(missingShard.summary).not.toContain("security-posture"); }); }); diff --git a/tools/advisors/github.mts b/tools/advisors/github.mts index 7a362614f41..1b786ce1462 100644 --- a/tools/advisors/github.mts +++ b/tools/advisors/github.mts @@ -80,9 +80,10 @@ export async function githubApi( token: string, options: GitHubRequestOptions = {}, ): Promise { - const apiUrl = `https://api.github.com/${apiPath}`; - // biome-ignore format: Keep the CodeQL suppression on the alert sink. - const response = await fetch(apiUrl, { // codeql[js/file-access-to-http]: fixed GitHub API endpoint with a reviewed payload. + // lgtm[js/file-access-to-http] Advisor workflows intentionally send normalized + // artifact summaries and strictly validated dispatch inputs to GitHub APIs. + // Callers construct apiPath from fixed workflow/comment endpoints, not PR text. + const response = await fetch(`https://api.github.com/${apiPath}`, { method: options.method || "GET", headers: { Accept: "application/vnd.github+json", @@ -91,7 +92,7 @@ export async function githubApi( "X-GitHub-Api-Version": "2022-11-28", ...(options.userAgent ? { "User-Agent": options.userAgent } : {}), }, - body: options.body === undefined ? undefined : JSON.stringify(options.body), // codeql[js/file-access-to-http]: reviewed summaries and validated controller state go only to fixed GitHub API endpoints. + body: options.body === undefined ? undefined : JSON.stringify(options.body), }); const text = await response.text(); if (!response.ok) { diff --git a/tools/e2e-advisor/post-merge-risk-gate.mts b/tools/e2e-advisor/post-merge-risk-gate.mts index 86d2b15aee0..a062fa15536 100755 --- a/tools/e2e-advisor/post-merge-risk-gate.mts +++ b/tools/e2e-advisor/post-merge-risk-gate.mts @@ -41,8 +41,8 @@ type ControllerPaths = { export type ControllerCommand = | ({ mode: "start"; baseSha: string; commitSha: string } & ControllerPaths) - | ({ mode: "finish" } & ControllerPaths) - | { mode: "abandon"; checkId: string }; + | ({ mode: "finish"; checkRunId: number; childRunId: number } & ControllerPaths) + | { mode: "abandon"; checkRunId: number }; type CheckConclusion = "success" | "failure" | "neutral"; @@ -62,16 +62,12 @@ type CheckRun = { id: number }; export type RiskGateState = { version: 1; - repository: string; commitSha: string; planHash: string; correlationId: string; expectedJobs: string[]; expectedShards: Record; requiresManualExpansion: boolean; - checkRunId: number; - childRunId: number; - childRunUrl: string; }; export type RiskEvidenceVerdict = { @@ -104,6 +100,13 @@ function requiredArgument(value: string | undefined, name: string): string { return value; } +function parsePositiveId(value: string, name: string): number { + if (!/^[1-9][0-9]*$/u.test(value)) throw new Error(`${name} must be a positive integer`); + const parsed = Number(value); + if (!Number.isSafeInteger(parsed)) throw new Error(`${name} exceeds the safe integer range`); + return parsed; +} + export function privateControllerPaths(workDir: string): ControllerPaths { const resolved = path.resolve(workDir); const stat = fs.lstatSync(resolved); @@ -138,12 +141,14 @@ export function parseControllerCommand(argv: string[]): ControllerCommand { return { mode: "finish", ...privateControllerPaths(requiredArgument(args.workDir, "work-dir")), + checkRunId: parsePositiveId(requiredArgument(args.checkId, "check-id"), "--check-id"), + childRunId: parsePositiveId(requiredArgument(args.runId, "run-id"), "--run-id"), }; } if (args.mode === "abandon") { return { mode: "abandon", - checkId: requiredArgument(args.checkId, "check-id"), + checkRunId: parsePositiveId(requiredArgument(args.checkId, "check-id"), "--check-id"), }; } throw new Error("--mode must be start, finish, or abandon"); @@ -153,14 +158,8 @@ function readRegularJson(file: string, maxBytes = MAX_PLAN_BYTES): unknown { return JSON.parse(readPrivateRegularFile(file, { maxBytes })!); } -export function validateRiskGateState(value: unknown, expectedRepository: string): RiskGateState { +export function validateRiskGateState(value: unknown): RiskGateState { if (!isRecord(value) || value.version !== 1) throw new Error("invalid risk-gate state version"); - if ( - value.repository !== expectedRepository || - !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(expectedRepository) - ) { - throw new Error("risk-gate state repository does not match this workflow"); - } if (typeof value.commitSha !== "string" || !SHA_PATTERN.test(value.commitSha)) { throw new Error("risk-gate state commit SHA is invalid"); } @@ -198,14 +197,6 @@ export function validateRiskGateState(value: unknown, expectedRepository: string if (typeof value.requiresManualExpansion !== "boolean") { throw new Error("risk-gate manual-expansion state is invalid"); } - if (!Number.isSafeInteger(value.checkRunId) || Number(value.checkRunId) < 1) { - throw new Error("risk-gate check id is invalid"); - } - if (!Number.isSafeInteger(value.childRunId) || Number(value.childRunId) < 1) { - throw new Error("risk-gate child run id is invalid"); - } - const expectedUrl = `https://github.com/${expectedRepository}/actions/runs/${value.childRunId}`; - if (value.childRunUrl !== expectedUrl) throw new Error("risk-gate child run URL is invalid"); return value as RiskGateState; } @@ -288,14 +279,14 @@ export function classifyRiskEvidence(options: { return { conclusion: "failure", title: "Selected E2E workflow failed", - summary: `The correlated workflow concluded ${options.workflowConclusion}.`, + summary: "The correlated workflow reported a failing terminal conclusion.", }; } if (options.workflowConclusion !== "success") { return { conclusion: "neutral", title: "Selected E2E workflow produced no complete signal", - summary: `The correlated workflow concluded ${options.workflowConclusion ?? "without a conclusion"}.`, + summary: "The correlated workflow did not report a successful terminal conclusion.", }; } const byJobShard = new Map(); @@ -309,7 +300,7 @@ export function classifyRiskEvidence(options: { return { conclusion: "neutral", title: "Selected E2E jobs produced ambiguous evidence", - summary: `Multiple risk signals were uploaded for: ${[...duplicates].sort().join(", ")}.`, + summary: "More than one signal was uploaded for at least one expected job shard.", }; } const expectedEvidence = options.expectedJobs.flatMap((job) => @@ -322,7 +313,7 @@ export function classifyRiskEvidence(options: { return { conclusion: "neutral", title: "Selected E2E jobs lack an evidence policy", - summary: `No trusted shard policy was found for: ${jobsWithoutShardPolicy.join(", ")}.`, + summary: "At least one selected job had no trusted shard policy.", }; } const missing = expectedEvidence.filter((key) => !byJobShard.has(key)); @@ -333,7 +324,7 @@ export function classifyRiskEvidence(options: { return { conclusion: "neutral", title: "Selected E2E jobs are missing test evidence", - summary: `No risk signal was uploaded for: ${missing.join(", ")}.`, + summary: "At least one expected job shard did not upload a bound risk signal.", }; } const failed = expectedEvidence.filter((key) => { @@ -344,7 +335,7 @@ export function classifyRiskEvidence(options: { return { conclusion: "failure", title: "Selected E2E jobs reported test failures", - summary: `Failed exact-SHA evidence was reported for: ${failed.join(", ")}.`, + summary: "At least one exact-SHA job shard reported a test failure or unhandled error.", }; } const partial = expectedEvidence.filter((key) => { @@ -357,7 +348,7 @@ export function classifyRiskEvidence(options: { return { conclusion: "neutral", title: "Selected E2E jobs produced partial or skipped evidence", - summary: `The following jobs did not produce an unskipped pass: ${partial.join(", ")}.`, + summary: "At least one expected job shard did not produce a complete, unskipped pass.", }; } if (options.requiresManualExpansion) { @@ -371,7 +362,7 @@ export function classifyRiskEvidence(options: { return { conclusion: "success", title: "All risk-selected E2E jobs passed", - summary: `${expectedEvidence.length} exact-SHA job shard(s) produced complete, unskipped evidence.`, + summary: "Every expected exact-SHA job shard produced complete, unskipped evidence.", }; } @@ -422,12 +413,12 @@ async function createCheck( } async function completeCheck( - state: Pick, + context: { repository: string; checkRunId: number }, token: string, verdict: RiskEvidenceVerdict, detailsUrl?: string, ): Promise { - await githubApi(`repos/${state.repository}/check-runs/${state.checkRunId}`, token, { + await githubApi(`repos/${context.repository}/check-runs/${context.checkRunId}`, token, { method: "PATCH", body: { status: "completed", @@ -441,14 +432,14 @@ async function completeCheck( } async function completeNeutralAfterControllerError( - state: Pick, + context: { repository: string; checkRunId: number }, token: string, title: string, detailsUrl?: string, ): Promise { try { await completeCheck( - state, + context, token, { conclusion: "neutral", @@ -647,20 +638,16 @@ async function start(options: { const child = await findCorrelatedRun(repository, token, correlationId, dispatchedAt); const state: RiskGateState = { version: 1, - repository, commitSha: options.commitSha, planHash: plan.planHash, correlationId, expectedJobs: plan.automaticJobs, expectedShards, requiresManualExpansion: plan.requiresManualExpansion, - checkRunId, - childRunId: child.id, - childRunUrl: child.html_url, }; writePrivateRegularFile(options.statePath, `${JSON.stringify(state, null, 2)}\n`); - appendOutput("dispatched", "true"); appendOutput("run_id", String(child.id)); + appendOutput("dispatched", "true"); } catch (error) { const finalized = await completeNeutralAfterControllerError( { repository, checkRunId }, @@ -727,25 +714,33 @@ export function findSignalFiles( return files.sort((left, right) => left.localeCompare(right)); } -async function finish(options: { statePath: string; evidencePath: string }): Promise { +async function finish(options: { + statePath: string; + evidencePath: string; + checkRunId: number; + childRunId: number; +}): Promise { const token = process.env.GITHUB_TOKEN ?? ""; const repository = process.env.GITHUB_REPOSITORY ?? ""; - const state = validateRiskGateState(readRegularJson(options.statePath), repository); - if (!token) { - throw new Error("valid gate state and GITHUB_TOKEN are required"); + if (!token || !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository)) { + throw new Error("GITHUB_TOKEN and a safe GITHUB_REPOSITORY are required"); } + const { checkRunId, childRunId } = options; + const childRunUrl = `https://github.com/${repository}/actions/runs/${childRunId}`; + const context = { repository, checkRunId }; + const state = validateRiskGateState(readRegularJson(options.statePath)); try { const child = await githubApi( - `repos/${state.repository}/actions/runs/${state.childRunId}`, + `repos/${repository}/actions/runs/${childRunId}`, token, { userAgent: "nemoclaw-e2e-risk-gate" }, ); if ( - child.id !== state.childRunId || + child.id !== childRunId || child.name !== "E2E" || child.event !== "workflow_dispatch" || !SHA_PATTERN.test(child.head_sha) || - child.html_url !== state.childRunUrl || + child.html_url !== childRunUrl || child.display_title !== `E2E risk ${state.correlationId}` ) { throw new Error("correlated E2E workflow identity changed"); @@ -763,29 +758,25 @@ async function finish(options: { statePath: string; evidencePath: string }): Pro signals, requiresManualExpansion: state.requiresManualExpansion, }); - await completeCheck(state, token, verdict, state.childRunUrl); + await completeCheck(context, token, verdict, childRunUrl); } catch (error) { await completeNeutralAfterControllerError( - state, + context, token, "Risk-selected E2E evidence could not be verified", - state.childRunUrl, + childRunUrl, ); throw error; } } -async function abandon(checkId: string): Promise { +async function abandon(checkRunId: number): Promise { const token = process.env.GITHUB_TOKEN ?? ""; const repository = process.env.GITHUB_REPOSITORY ?? ""; - if ( - !token || - !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository) || - !/^[1-9][0-9]*$/u.test(checkId) - ) { - throw new Error("GITHUB_TOKEN, a safe GITHUB_REPOSITORY, and numeric --check-id are required"); + if (!token || !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository)) { + throw new Error("GITHUB_TOKEN and a safe GITHUB_REPOSITORY are required"); } - await completeCheck({ repository, checkRunId: Number(checkId) }, token, { + await completeCheck({ repository, checkRunId }, token, { conclusion: "neutral", title: "Risk-selected E2E controller stopped early", summary: @@ -808,11 +799,13 @@ async function main(): Promise { await finish({ statePath: command.statePath, evidencePath: command.evidencePath, + checkRunId: command.checkRunId, + childRunId: command.childRunId, }); return; } if (command.mode === "abandon") { - await abandon(command.checkId); + await abandon(command.checkRunId); return; } } diff --git a/tools/e2e-advisor/private-file.ts b/tools/e2e-advisor/private-file.ts index 3190b93cd5c..ca281d92e5f 100644 --- a/tools/e2e-advisor/private-file.ts +++ b/tools/e2e-advisor/private-file.ts @@ -54,8 +54,7 @@ export function writePrivateRegularFile(file: string, contents: string): void { } fs.fchmodSync(descriptor, 0o600); fs.ftruncateSync(descriptor, 0); - // Callers validate and bound state before this exclusive/no-follow write. - fs.writeFileSync(descriptor, contents, "utf8"); // codeql[js/http-to-file-access]: intentional validated controller state persistence. + fs.writeFileSync(descriptor, contents, "utf8"); fs.fsyncSync(descriptor); } finally { fs.closeSync(descriptor); From 197c90f11a3e0f16de87a4756a41eb65628c7316 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Wed, 8 Jul 2026 01:12:46 -0700 Subject: [PATCH 6/9] fix(ci): reserve exact checkout for risk shadow Signed-off-by: Carlos Villela --- .github/workflows/e2e.yaml | 12 ++++-------- test/post-merge-e2e-risk-gate-workflow.test.ts | 3 +++ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index c7d03ab75e3..2a6a1365adf 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -92,17 +92,13 @@ jobs: run: | set -euo pipefail [[ "$WORKFLOW_EVENT" == "workflow_dispatch" && "$WORKFLOW_REF" == "refs/heads/main" ]] || { echo "::error::exact-commit runs require a workflow_dispatch from main"; exit 1; } + [[ "$RISK_SHADOW" == "true" ]] || { echo "::error::exact-commit inputs require risk_shadow=true"; exit 1; } [[ "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ ]] || { echo "::error::checkout_sha must be a lowercase 40-character SHA"; exit 1; } [[ "$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]] || { echo "::error::checked-out HEAD does not match checkout_sha"; exit 1; } git merge-base --is-ancestor "$CHECKOUT_SHA" origin/main || { echo "::error::checkout_sha must already be reachable from main"; exit 1; } - if [[ "$RISK_SHADOW" == "true" ]]; then - [[ "$PLAN_HASH" =~ ^[a-f0-9]{64}$ ]] || { echo "::error::risk_plan_hash must be a lowercase SHA-256"; exit 1; } - [[ "$RISK_CORRELATION" =~ ^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$ ]] || { echo "::error::risk_correlation must be a lowercase UUIDv4"; exit 1; } - [[ -n "$JOBS" && -z "$TARGETS" ]] || { echo "::error::shadow risk runs require selective jobs and forbid targets/fan-out"; exit 1; } - elif [[ -n "$PLAN_HASH" || -n "$RISK_CORRELATION" ]]; then - echo "::error::risk correlation metadata requires risk_shadow=true" - exit 1 - fi + [[ "$PLAN_HASH" =~ ^[a-f0-9]{64}$ ]] || { echo "::error::risk_plan_hash must be a lowercase SHA-256"; exit 1; } + [[ "$RISK_CORRELATION" =~ ^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$ ]] || { echo "::error::risk_correlation must be a lowercase UUIDv4"; exit 1; } + [[ -n "$JOBS" && -z "$TARGETS" ]] || { echo "::error::shadow risk runs require selective jobs and forbid targets/fan-out"; exit 1; } - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@50281ee84c4a6fc759da95ea28fc0b7d9c378a28 diff --git a/test/post-merge-e2e-risk-gate-workflow.test.ts b/test/post-merge-e2e-risk-gate-workflow.test.ts index f2f49875366..d514501b608 100644 --- a/test/post-merge-e2e-risk-gate-workflow.test.ts +++ b/test/post-merge-e2e-risk-gate-workflow.test.ts @@ -180,6 +180,9 @@ describe("post-merge E2E risk gate shadow workflow", () => { expect(validateIndex).toBeGreaterThan(0); expect(validateIndex).toBeLessThan(prepareIndex); + expect(validate?.if).toContain("inputs.checkout_sha != ''"); + expect(validate?.run).toContain('[[ "$RISK_SHADOW" == "true" ]]'); + expect(validate?.run).toContain("exact-commit inputs require risk_shadow=true"); expect(validate?.run).toContain("checkout_sha must be a lowercase 40-character SHA"); expect(validate?.run).toContain('"$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA"'); expect(validate?.run).toContain('git merge-base --is-ancestor "$CHECKOUT_SHA" origin/main'); From e7c5f5f119a5716c5b637898c870f10d270deaa5 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Wed, 8 Jul 2026 01:26:17 -0700 Subject: [PATCH 7/9] fix(ci): bind risk shadow to current main Signed-off-by: Carlos Villela --- .github/workflows/e2e.yaml | 4 +++- test/e2e/README.md | 9 ++++++--- test/post-merge-e2e-risk-gate-workflow.test.ts | 3 +++ tools/e2e-advisor/README.md | 9 ++++++--- 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 2a6a1365adf..a3edf40c1ee 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -30,7 +30,7 @@ on: default: false type: boolean checkout_sha: - description: Exact commit to test while keeping the workflow definition on the dispatched ref. + description: Current main commit selected by the trusted post-merge shadow controller. required: false default: "" type: string @@ -89,11 +89,13 @@ jobs: TARGETS: ${{ inputs.targets }} WORKFLOW_EVENT: ${{ github.event_name }} WORKFLOW_REF: ${{ github.ref }} + WORKFLOW_SHA: ${{ github.sha }} run: | set -euo pipefail [[ "$WORKFLOW_EVENT" == "workflow_dispatch" && "$WORKFLOW_REF" == "refs/heads/main" ]] || { echo "::error::exact-commit runs require a workflow_dispatch from main"; exit 1; } [[ "$RISK_SHADOW" == "true" ]] || { echo "::error::exact-commit inputs require risk_shadow=true"; exit 1; } [[ "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ ]] || { echo "::error::checkout_sha must be a lowercase 40-character SHA"; exit 1; } + [[ "$CHECKOUT_SHA" == "$WORKFLOW_SHA" ]] || { echo "::error::checkout_sha must equal the current main workflow commit"; exit 1; } [[ "$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]] || { echo "::error::checked-out HEAD does not match checkout_sha"; exit 1; } git merge-base --is-ancestor "$CHECKOUT_SHA" origin/main || { echo "::error::checkout_sha must already be reachable from main"; exit 1; } [[ "$PLAN_HASH" =~ ^[a-f0-9]{64}$ ]] || { echo "::error::risk_plan_hash must be a lowercase SHA-256"; exit 1; } diff --git a/test/e2e/README.md b/test/e2e/README.md index 7847e422253..dc7fab6b67c 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -67,9 +67,12 @@ The workflow definition stays on `main`, while every E2E checkout uses the exact merged commit supplied through `checkout_sha`. Before E2E preparation or selected jobs can use repository secrets, -`e2e.yaml` verifies the requested SHA, confirms that checked-out `HEAD` matches -it, proves that the commit is reachable from `main`, and accepts only selective -job dispatch with valid plan and correlation metadata. The shadow-only Vitest +`e2e.yaml` verifies that the requested SHA equals the workflow's own current +`main` commit, confirms that checked-out `HEAD` matches it, proves reachability, +and accepts only selective job dispatch with valid plan and correlation +metadata. If `main` advances before an older controller dispatches, that child +fails closed and the controller records failure instead of running historical +code with current secrets. The shadow-only Vitest reporter then writes a `risk-signal.json` for each selected job and matrix shard. Each signal binds the observed checkout SHA, expected SHA, plan hash, correlation ID, and pass, failure, skip, pending, and unhandled-error counts. diff --git a/test/post-merge-e2e-risk-gate-workflow.test.ts b/test/post-merge-e2e-risk-gate-workflow.test.ts index d514501b608..ab038423c71 100644 --- a/test/post-merge-e2e-risk-gate-workflow.test.ts +++ b/test/post-merge-e2e-risk-gate-workflow.test.ts @@ -181,9 +181,12 @@ describe("post-merge E2E risk gate shadow workflow", () => { expect(validateIndex).toBeGreaterThan(0); expect(validateIndex).toBeLessThan(prepareIndex); expect(validate?.if).toContain("inputs.checkout_sha != ''"); + expect(validate?.env?.WORKFLOW_SHA).toBe("${{ github.sha }}"); expect(validate?.run).toContain('[[ "$RISK_SHADOW" == "true" ]]'); expect(validate?.run).toContain("exact-commit inputs require risk_shadow=true"); expect(validate?.run).toContain("checkout_sha must be a lowercase 40-character SHA"); + expect(validate?.run).toContain('[[ "$CHECKOUT_SHA" == "$WORKFLOW_SHA" ]]'); + expect(validate?.run).toContain("checkout_sha must equal the current main workflow commit"); expect(validate?.run).toContain('"$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA"'); expect(validate?.run).toContain('git merge-base --is-ancestor "$CHECKOUT_SHA" origin/main'); expect(validate?.run).toContain("forbid targets/fan-out"); diff --git a/tools/e2e-advisor/README.md b/tools/e2e-advisor/README.md index f32f17258a5..88fe1151e94 100644 --- a/tools/e2e-advisor/README.md +++ b/tools/e2e-advisor/README.md @@ -47,9 +47,12 @@ builds a new plan from the exact `github.event.before` and `github.event.after` and dispatches at most three `automaticJobs` to `e2e.yaml` against the merged commit. -The child workflow validates the exact checkout SHA, its reachability from -`main`, selective-job inputs, plan hash, and correlation ID before E2E -preparation or secret-bearing jobs can run. The shadow-only Vitest reporter +The child workflow validates that the exact checkout SHA equals the workflow's +own current `main` commit, verifies its reachability, and checks selective-job +inputs, plan hash, and correlation ID before E2E preparation or secret-bearing +jobs can run. If `main` advances before an older controller dispatches, that +child fails closed and the controller reports failure rather than executing the +older commit with current secrets. The shadow-only Vitest reporter records the observed checkout SHA and pass, failure, skip, pending, and unhandled-error counts for each job and matrix shard. The controller accepts only signals bound to the expected SHA, plan hash, correlation ID, job, and From 945808241925035254a62cc43a331f35d724bf7f Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Wed, 8 Jul 2026 01:44:49 -0700 Subject: [PATCH 8/9] fix(ci): bind risk gate to dispatched run Signed-off-by: Carlos Villela --- test/e2e/README.md | 4 +- test/e2e/docs/README.md | 10 +- .../post-merge-e2e-risk-gate-workflow.test.ts | 2 + test/post-merge-e2e-risk-gate.test.ts | 107 +++++++++++++++- tools/e2e-advisor/README.md | 4 +- tools/e2e-advisor/post-merge-risk-gate.mts | 117 +++++++++++------- 6 files changed, 193 insertions(+), 51 deletions(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index dc7fab6b67c..c7add0306e4 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -64,7 +64,9 @@ shadow controller. It builds the deterministic risk plan from the exact matches the pushed commit. If the plan matches runtime regression families, the controller dispatches at most three `automaticJobs` through `e2e.yaml`. The workflow definition stays on `main`, while every E2E checkout uses the -exact merged commit supplied through `checkout_sha`. +exact merged commit supplied through `checkout_sha`. GitHub returns the +dispatched workflow's run ID directly, and the controller uses that ID as the +sole child-run selector for waiting, evidence download, and completion. Before E2E preparation or selected jobs can use repository secrets, `e2e.yaml` verifies that the requested SHA equals the workflow's own current diff --git a/test/e2e/docs/README.md b/test/e2e/docs/README.md index 4e086fd19c2..8e2893e8728 100644 --- a/test/e2e/docs/README.md +++ b/test/e2e/docs/README.md @@ -121,10 +121,12 @@ test/e2e/ These per-target timing summaries are artifact evidence only. The Slack and GitHub scorecard timing comparison remains scoped to the dedicated `cloud-onboard` artifact. - Exact-commit shadow dispatches validate that the requested checkout is - reachable from `main` before preparation, attach - `test/e2e/risk-signal-reporter.ts` to live Vitest invocations, and suppress - PR reporting and scorecards. + Exact-commit shadow dispatches require the requested checkout to equal the + workflow's current `main` commit and verify its reachability before + preparation. The controller uses GitHub's returned workflow-dispatch run ID + as the sole child-run selector for waiting, evidence download, and + completion, attaches `test/e2e/risk-signal-reporter.ts` to live Vitest + invocations, and suppresses PR reporting and scorecards. - `.github/workflows/e2e-branch-validation.yaml`, `macos-e2e.yaml`, `wsl-e2e.yaml`, `ollama-proxy-e2e.yaml`, and `regression-e2e.yaml` call focused E2E targets directly for their E2E coverage. diff --git a/test/post-merge-e2e-risk-gate-workflow.test.ts b/test/post-merge-e2e-risk-gate-workflow.test.ts index ab038423c71..828604d610d 100644 --- a/test/post-merge-e2e-risk-gate-workflow.test.ts +++ b/test/post-merge-e2e-risk-gate-workflow.test.ts @@ -78,7 +78,9 @@ describe("post-merge E2E risk gate shadow workflow", () => { expect(start.run).toContain('--commit "${{ github.event.after }}"'); expect(start.run).toContain('--work-dir "${{ steps.workspace.outputs.work_dir }}"'); expect(wait.run).toContain("timeout --signal=TERM --kill-after=30s 105m"); + expect(wait.env?.RUN_ID).toBe("${{ steps.start.outputs.run_id }}"); expect(download.run).toContain('--dir "${{ steps.workspace.outputs.work_dir }}/evidence"'); + expect(download.env?.RUN_ID).toBe("${{ steps.start.outputs.run_id }}"); expect(finish.id).toBe("finish"); expect(finish.if).toContain("always()"); expect(finish["continue-on-error"]).toBe(true); diff --git a/test/post-merge-e2e-risk-gate.test.ts b/test/post-merge-e2e-risk-gate.test.ts index 1284923cbe0..1e77d5ac605 100644 --- a/test/post-merge-e2e-risk-gate.test.ts +++ b/test/post-merge-e2e-risk-gate.test.ts @@ -6,25 +6,33 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { buildRiskPlan, RISK_RULES } from "../tools/advisors/risk-plan.mts"; import { assertTrustedMainPush, changedFilesBetween, classifyRiskEvidence, + dispatchRiskWorkflow, expectedRiskSignalShards, findSignalFiles, + finishRiskGate, parseControllerCommand, type RiskGateState, validateRiskGateState, validateRiskPlan, validateSignal, + validateWorkflowDispatchDetails, } from "../tools/e2e-advisor/post-merge-risk-gate.mts"; import type { E2eRiskSignal } from "../tools/e2e-advisor/risk-signal.ts"; const HEAD_SHA = "a".repeat(40); const ALLOWED = new Set(["onboard-repair", "onboard-resume"]); +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); +}); + function state(): RiskGateState { return { version: 1, @@ -185,6 +193,103 @@ describe("post-merge E2E risk gate", () => { expect(() => validateRiskGateState({ ...gate, expectedShards: {} })).toThrow(/shard jobs/u); }); + it("uses the workflow run identity returned by the exact dispatch request", async () => { + const gate = state(); + const fetchMock = vi.spyOn(globalThis, "fetch").mockResolvedValue({ + ok: true, + text: async () => + JSON.stringify({ + workflow_run_id: 23, + run_url: "https://api.github.com/repos/NVIDIA/NemoClaw/actions/runs/23", + html_url: "https://github.com/NVIDIA/NemoClaw/actions/runs/23", + }), + } as Response); + + const runId = await dispatchRiskWorkflow({ + repository: "NVIDIA/NemoClaw", + token: "token", + jobs: ["onboard-repair"], + commitSha: gate.commitSha, + planHash: gate.planHash, + correlationId: gate.correlationId, + }); + + expect(runId).toBe(23); + expect(fetchMock).toHaveBeenCalledOnce(); + expect(String(fetchMock.mock.calls[0]?.[0])).toBe( + "https://api.github.com/repos/NVIDIA/NemoClaw/actions/workflows/e2e.yaml/dispatches", + ); + expect(fetchMock.mock.calls[0]?.[1]?.method).toBe("POST"); + expect(JSON.parse(String(fetchMock.mock.calls[0]?.[1]?.body))).toMatchObject({ + ref: "main", + return_run_details: true, + inputs: { + jobs: "onboard-repair", + checkout_sha: gate.commitSha, + risk_plan_hash: gate.planHash, + risk_correlation: gate.correlationId, + risk_shadow: "true", + }, + }); + expect(() => + validateWorkflowDispatchDetails( + { + workflow_run_id: 23, + run_url: "https://api.github.com/repos/NVIDIA/NemoClaw/actions/runs/24", + html_url: "https://github.com/NVIDIA/NemoClaw/actions/runs/23", + }, + "NVIDIA/NemoClaw", + ), + ).toThrow(/mismatched workflow dispatch URLs/u); + }); + + it("finish reports the directly dispatched child failure when main advances", async () => { + const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-risk-finish-")); + const statePath = path.join(workDir, "e2e-risk-gate-state.json"); + const gate = state(); + const childRunId = 23; + vi.stubEnv("GITHUB_TOKEN", "token"); + vi.stubEnv("GITHUB_REPOSITORY", "NVIDIA/NemoClaw"); + fs.writeFileSync(statePath, `${JSON.stringify(gate)}\n`, { mode: 0o600 }); + const fetchMock = vi + .spyOn(globalThis, "fetch") + .mockResolvedValueOnce({ + ok: true, + text: async () => + JSON.stringify({ + id: childRunId, + name: "E2E", + event: "workflow_dispatch", + head_sha: "b".repeat(40), + status: "completed", + conclusion: "failure", + created_at: "2026-07-08T00:00:00.000Z", + display_title: `E2E risk ${gate.correlationId}`, + html_url: `https://github.com/NVIDIA/NemoClaw/actions/runs/${childRunId}`, + }), + } as Response) + .mockResolvedValueOnce({ ok: true, text: async () => "{}" } as Response); + + try { + await finishRiskGate({ + statePath, + evidencePath: path.join(workDir, "evidence"), + checkRunId: 17, + childRunId, + }); + + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(String(fetchMock.mock.calls[1]?.[0])).toContain("check-runs/17"); + expect(JSON.parse(String(fetchMock.mock.calls[1]?.[1]?.body))).toMatchObject({ + status: "completed", + conclusion: "failure", + output: { title: "Selected E2E workflow failed" }, + }); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + }); + it("bounds downloaded risk-evidence traversal by entries, depth, and signals", () => { const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-risk-evidence-")); const rootLink = `${directory}-link`; diff --git a/tools/e2e-advisor/README.md b/tools/e2e-advisor/README.md index 88fe1151e94..b1e5d17d938 100644 --- a/tools/e2e-advisor/README.md +++ b/tools/e2e-advisor/README.md @@ -45,7 +45,9 @@ does not consume model output or the advisor artifact. Instead, `tools/e2e-advisor/post-merge-risk-gate.mts` confirms the trusted controller checkout, builds a new plan from the exact `github.event.before` and `github.event.after` range, and dispatches at most three `automaticJobs` to `e2e.yaml` against the merged -commit. +commit. The controller opts into GitHub's workflow-dispatch run details and +uses the returned run ID as the sole child-run selector, so a lookalike run +cannot win a polling race for the same correlation ID. The child workflow validates that the exact checkout SHA equals the workflow's own current `main` commit, verifies its reachability, and checks selective-job diff --git a/tools/e2e-advisor/post-merge-risk-gate.mts b/tools/e2e-advisor/post-merge-risk-gate.mts index a062fa15536..fb19be80a5a 100755 --- a/tools/e2e-advisor/post-merge-risk-gate.mts +++ b/tools/e2e-advisor/post-merge-risk-gate.mts @@ -60,6 +60,12 @@ type WorkflowRun = { type CheckRun = { id: number }; +type WorkflowDispatchDetails = { + workflow_run_id: number; + run_url: string; + html_url: string; +}; + export type RiskGateState = { version: 1; commitSha: string; @@ -534,34 +540,65 @@ export function expectedRiskSignalShards( ); } -async function findCorrelatedRun( +export function validateWorkflowDispatchDetails( + value: unknown, repository: string, - token: string, - correlationId: string, - dispatchedAt: number, -): Promise { - const expectedTitle = `E2E risk ${correlationId}`; - for (let attempt = 0; attempt < 60; attempt += 1) { - const payload = await githubApi<{ workflow_runs: WorkflowRun[] }>( - `repos/${repository}/actions/workflows/${E2E_WORKFLOW}/runs?event=workflow_dispatch&branch=main&per_page=50`, - token, - { userAgent: "nemoclaw-e2e-risk-gate" }, - ); - const match = payload.workflow_runs.find( - (run) => - run.display_title === expectedTitle && - run.name === "E2E" && - run.event === "workflow_dispatch" && - SHA_PATTERN.test(run.head_sha) && - Number.isSafeInteger(run.id) && - run.id > 0 && - run.html_url === `https://github.com/${repository}/actions/runs/${run.id}` && - Date.parse(run.created_at) >= dispatchedAt - 5000, - ); - if (match) return match; - await new Promise((resolve) => setTimeout(resolve, 5000)); +): WorkflowDispatchDetails { + if (!isRecord(value)) throw new Error("GitHub returned invalid workflow dispatch details"); + const runId = value.workflow_run_id; + if (!Number.isSafeInteger(runId) || (runId as number) < 1) { + throw new Error("GitHub returned an invalid dispatched workflow run id"); + } + const expectedApiUrl = `https://api.github.com/repos/${repository}/actions/runs/${runId}`; + const expectedHtmlUrl = `https://github.com/${repository}/actions/runs/${runId}`; + if (value.run_url !== expectedApiUrl || value.html_url !== expectedHtmlUrl) { + throw new Error("GitHub returned mismatched workflow dispatch URLs"); + } + return value as WorkflowDispatchDetails; +} + +export async function dispatchRiskWorkflow(options: { + repository: string; + token: string; + jobs: readonly string[]; + commitSha: string; + planHash: string; + correlationId: string; +}): Promise { + if ( + !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(options.repository) || + !options.token || + options.jobs.length < 1 || + options.jobs.length > 3 || + new Set(options.jobs).size !== options.jobs.length || + options.jobs.some((job) => !JOB_PATTERN.test(job)) || + !SHA_PATTERN.test(options.commitSha) || + !HASH_PATTERN.test(options.planHash) || + !CORRELATION_PATTERN.test(options.correlationId) + ) { + throw new Error("risk workflow dispatch inputs are invalid"); } - throw new Error("timed out locating the correlated E2E workflow run"); + const details = await githubApi( + `repos/${options.repository}/actions/workflows/${E2E_WORKFLOW}/dispatches`, + options.token, + { + method: "POST", + body: { + ref: "main", + inputs: { + jobs: options.jobs.join(","), + checkout_sha: options.commitSha, + risk_plan_hash: options.planHash, + risk_correlation: options.correlationId, + risk_shadow: "true", + }, + // GitHub REST 2022-11-28 otherwise returns no run identity. + return_run_details: true, + }, + userAgent: "nemoclaw-e2e-risk-gate", + }, + ); + return validateWorkflowDispatchDetails(details, options.repository).workflow_run_id; } async function start(options: { @@ -620,22 +657,14 @@ async function start(options: { if (!CORRELATION_PATTERN.test(correlationId)) { throw new Error("generated correlation id is invalid"); } - const dispatchedAt = Date.now(); - await githubApi(`repos/${repository}/actions/workflows/${E2E_WORKFLOW}/dispatches`, token, { - method: "POST", - body: { - ref: "main", - inputs: { - jobs: plan.automaticJobs.join(","), - checkout_sha: options.commitSha, - risk_plan_hash: plan.planHash, - risk_correlation: correlationId, - risk_shadow: "true", - }, - }, - userAgent: "nemoclaw-e2e-risk-gate", + const childRunId = await dispatchRiskWorkflow({ + repository, + token, + jobs: plan.automaticJobs, + commitSha: options.commitSha, + planHash: plan.planHash, + correlationId, }); - const child = await findCorrelatedRun(repository, token, correlationId, dispatchedAt); const state: RiskGateState = { version: 1, commitSha: options.commitSha, @@ -646,7 +675,7 @@ async function start(options: { requiresManualExpansion: plan.requiresManualExpansion, }; writePrivateRegularFile(options.statePath, `${JSON.stringify(state, null, 2)}\n`); - appendOutput("run_id", String(child.id)); + appendOutput("run_id", String(childRunId)); appendOutput("dispatched", "true"); } catch (error) { const finalized = await completeNeutralAfterControllerError( @@ -714,7 +743,7 @@ export function findSignalFiles( return files.sort((left, right) => left.localeCompare(right)); } -async function finish(options: { +export async function finishRiskGate(options: { statePath: string; evidencePath: string; checkRunId: number; @@ -796,7 +825,7 @@ async function main(): Promise { return; } if (command.mode === "finish") { - await finish({ + await finishRiskGate({ statePath: command.statePath, evidencePath: command.evidencePath, checkRunId: command.checkRunId, From b5552e68b425de52507fb4b50fa6e35a7c2fd0c9 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Wed, 8 Jul 2026 15:27:23 -0700 Subject: [PATCH 9/9] fix(ci): bind shadow state across evidence download Signed-off-by: Carlos Villela --- .../post-merge-e2e-risk-gate-shadow.yaml | 1 + .../post-merge-e2e-risk-gate-workflow.test.ts | 1 + test/post-merge-e2e-risk-gate.test.ts | 67 ++++++++++++++++++- tools/e2e-advisor/README.md | 6 ++ tools/e2e-advisor/post-merge-risk-gate.mts | 44 ++++++++++-- 5 files changed, 110 insertions(+), 9 deletions(-) diff --git a/.github/workflows/post-merge-e2e-risk-gate-shadow.yaml b/.github/workflows/post-merge-e2e-risk-gate-shadow.yaml index 007969b19c1..ef6667de248 100644 --- a/.github/workflows/post-merge-e2e-risk-gate-shadow.yaml +++ b/.github/workflows/post-merge-e2e-risk-gate-shadow.yaml @@ -112,6 +112,7 @@ jobs: node --experimental-strip-types tools/e2e-advisor/post-merge-risk-gate.mts --mode finish --work-dir "${{ steps.workspace.outputs.work_dir }}" + --state-hash "${{ steps.start.outputs.state_hash }}" --check-id "${{ steps.start.outputs.check_id }}" --run-id "${{ steps.start.outputs.run_id }}" diff --git a/test/post-merge-e2e-risk-gate-workflow.test.ts b/test/post-merge-e2e-risk-gate-workflow.test.ts index 061db5c1f71..b98c9bb9c34 100644 --- a/test/post-merge-e2e-risk-gate-workflow.test.ts +++ b/test/post-merge-e2e-risk-gate-workflow.test.ts @@ -86,6 +86,7 @@ describe("post-merge E2E risk gate shadow workflow", () => { expect(finish["continue-on-error"]).toBe(true); expect(finish.run).toContain("post-merge-risk-gate.mts --mode finish"); expect(finish.run).toContain('--work-dir "${{ steps.workspace.outputs.work_dir }}"'); + expect(finish.run).toContain('--state-hash "${{ steps.start.outputs.state_hash }}"'); expect(finish.run).toContain('--check-id "${{ steps.start.outputs.check_id }}"'); expect(finish.run).toContain('--run-id "${{ steps.start.outputs.run_id }}"'); expect(completionFallback.if).toContain("steps.finish.outcome == 'failure'"); diff --git a/test/post-merge-e2e-risk-gate.test.ts b/test/post-merge-e2e-risk-gate.test.ts index bdd2f8bb5d7..4b3d2126d24 100644 --- a/test/post-merge-e2e-risk-gate.test.ts +++ b/test/post-merge-e2e-risk-gate.test.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -48,6 +49,10 @@ function state(): RiskGateState { }; } +function sha256(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} + function signal(jobId: string, overrides: Partial = {}): E2eRiskSignal { const gate = state(); return { @@ -119,15 +124,36 @@ describe("post-merge E2E risk gate", () => { "finish", "--work-dir", workDir, + "--state-hash", + "b".repeat(64), "--check-id", "17", "--run-id", "23", ]), - ).toMatchObject({ mode: "finish", checkRunId: 17, childRunId: 23 }); + ).toMatchObject({ + mode: "finish", + checkRunId: 17, + childRunId: 23, + stateHash: "b".repeat(64), + }); expect(() => parseControllerCommand(["--mode", "abandon", "--check-id", "9007199254740992"]), ).toThrow(/safe integer range/u); + expect(() => + parseControllerCommand([ + "--mode", + "finish", + "--work-dir", + workDir, + "--state-hash", + "unsafe", + "--check-id", + "17", + "--run-id", + "23", + ]), + ).toThrow(/state-hash must be a lowercase SHA-256 hash/u); expect(() => parseControllerCommand(["--mode", "finish"])).toThrow(/--work-dir/u); fs.chmodSync(workDir, 0o755); @@ -247,10 +273,11 @@ describe("post-merge E2E risk gate", () => { const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-risk-finish-")); const statePath = path.join(workDir, "e2e-risk-gate-state.json"); const gate = state(); + const serializedState = `${JSON.stringify(gate)}\n`; const childRunId = 23; vi.stubEnv("GITHUB_TOKEN", "token"); vi.stubEnv("GITHUB_REPOSITORY", "NVIDIA/NemoClaw"); - fs.writeFileSync(statePath, `${JSON.stringify(gate)}\n`, { mode: 0o600 }); + fs.writeFileSync(statePath, serializedState, { mode: 0o600 }); const fetchMock = vi .spyOn(globalThis, "fetch") .mockResolvedValueOnce({ @@ -273,6 +300,7 @@ describe("post-merge E2E risk gate", () => { try { await finishRiskGate({ statePath, + stateHash: sha256(serializedState), evidencePath: path.join(workDir, "evidence"), checkRunId: 17, childRunId, @@ -290,6 +318,41 @@ describe("post-merge E2E risk gate", () => { } }); + it("rejects changed controller state before classifying downloaded evidence", async () => { + const workDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-risk-state-")); + const statePath = path.join(workDir, "e2e-risk-gate-state.json"); + const originalState = `${JSON.stringify(state())}\n`; + const changedState = `${JSON.stringify({ ...state(), requiresManualExpansion: true })}\n`; + vi.stubEnv("GITHUB_TOKEN", "token"); + vi.stubEnv("GITHUB_REPOSITORY", "NVIDIA/NemoClaw"); + fs.writeFileSync(statePath, changedState, { mode: 0o600 }); + const fetchMock = vi + .spyOn(globalThis, "fetch") + .mockResolvedValue({ ok: true, text: async () => "{}" } as Response); + + try { + await expect( + finishRiskGate({ + statePath, + stateHash: sha256(originalState), + evidencePath: path.join(workDir, "evidence"), + checkRunId: 17, + childRunId: 23, + }), + ).rejects.toThrow(/controller state changed after E2E dispatch/u); + + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(String(fetchMock.mock.calls[0]?.[0])).toContain("check-runs/17"); + expect(JSON.parse(String(fetchMock.mock.calls[0]?.[1]?.body))).toMatchObject({ + status: "completed", + conclusion: "neutral", + output: { title: "Risk-selected E2E evidence could not be verified" }, + }); + } finally { + fs.rmSync(workDir, { recursive: true, force: true }); + } + }); + it("bounds downloaded risk-evidence traversal by entries, depth, and signals", () => { const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-risk-evidence-")); const rootLink = `${directory}-link`; diff --git a/tools/e2e-advisor/README.md b/tools/e2e-advisor/README.md index 8133e6365d0..b200c8c1032 100644 --- a/tools/e2e-advisor/README.md +++ b/tools/e2e-advisor/README.md @@ -60,6 +60,12 @@ unhandled-error counts for each job and matrix shard. The controller accepts only signals bound to the expected SHA, plan hash, correlation ID, job, and shard. +The start step records a SHA-256 digest of its private controller state in a +GitHub step output. After child artifacts are downloaded, the finish step reads +that state once and verifies the digest before parsing it or classifying any +evidence, so downloaded files cannot change the dispatch state used for the +check result. + The controller writes a check on the merged commit without posting a PR comment or running the scheduled/manual scorecard. Complete unskipped evidence reports success, selected E2E workflow or test failures for the merged commit report diff --git a/tools/e2e-advisor/post-merge-risk-gate.mts b/tools/e2e-advisor/post-merge-risk-gate.mts index 9a754596620..b9d619904d0 100755 --- a/tools/e2e-advisor/post-merge-risk-gate.mts +++ b/tools/e2e-advisor/post-merge-risk-gate.mts @@ -4,7 +4,7 @@ // SPDX-License-Identifier: Apache-2.0 import { execFileSync } from "node:child_process"; -import { randomUUID } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import fs from "node:fs"; import path from "node:path"; import { pathToFileURL } from "node:url"; @@ -41,7 +41,12 @@ type ControllerPaths = { export type ControllerCommand = | ({ mode: "start"; baseSha: string; commitSha: string } & ControllerPaths) - | ({ mode: "finish"; checkRunId: number; childRunId: number } & ControllerPaths) + | ({ + mode: "finish"; + checkRunId: number; + childRunId: number; + stateHash: string; + } & ControllerPaths) | { mode: "abandon"; checkRunId: number }; type CheckConclusion = "success" | "failure" | "neutral"; @@ -113,6 +118,16 @@ function parsePositiveId(value: string, name: string): number { return parsed; } +function parseHash(value: string | undefined, name: string): string { + const parsed = requiredArgument(value, name); + if (!HASH_PATTERN.test(parsed)) throw new Error(`--${name} must be a lowercase SHA-256 hash`); + return parsed; +} + +function sha256(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} + export function privateControllerPaths(workDir: string): ControllerPaths { const resolved = path.resolve(workDir); const stat = fs.lstatSync(resolved); @@ -149,6 +164,7 @@ export function parseControllerCommand(argv: string[]): ControllerCommand { ...privateControllerPaths(requiredArgument(args.workDir, "work-dir")), checkRunId: parsePositiveId(requiredArgument(args.checkId, "check-id"), "--check-id"), childRunId: parsePositiveId(requiredArgument(args.runId, "run-id"), "--run-id"), + stateHash: parseHash(args.stateHash, "state-hash"), }; } if (args.mode === "abandon") { @@ -375,10 +391,12 @@ export function classifyRiskEvidence(options: { function appendOutput(name: string, value: string): void { const output = process.env.GITHUB_OUTPUT; if (!output) return; - if (!/^(?:check_id|dispatched|finalized|run_id)$/u.test(name)) { + if (!/^(?:check_id|dispatched|finalized|run_id|state_hash)$/u.test(name)) { throw new Error("invalid controller output name"); } - if (!/^(?:true|false|[1-9][0-9]*)$/u.test(value)) { + const validValue = + name === "state_hash" ? HASH_PATTERN.test(value) : /^(?:true|false|[1-9][0-9]*)$/u.test(value); + if (!validValue) { throw new Error("invalid controller output value"); } const descriptor = fs.openSync( @@ -388,7 +406,8 @@ function appendOutput(name: string, value: string): void { try { if (!fs.fstatSync(descriptor).isFile()) throw new Error("GITHUB_OUTPUT must be a regular file"); // GitHub supplies this output file; values are restricted above to fixed - // booleans or positive decimal IDs before the descriptor write. + // booleans, positive decimal IDs, or a lowercase SHA-256 digest before the + // descriptor write. // codeql[js/http-to-file-access] fs.writeFileSync(descriptor, `${name}=${value}\n`, "utf8"); } finally { @@ -674,7 +693,9 @@ async function start(options: { expectedShards, requiresManualExpansion: plan.requiresManualExpansion, }; - writePrivateRegularFile(options.statePath, `${JSON.stringify(state, null, 2)}\n`); + const serializedState = `${JSON.stringify(state, null, 2)}\n`; + writePrivateRegularFile(options.statePath, serializedState); + appendOutput("state_hash", sha256(serializedState)); appendOutput("run_id", String(childRunId)); appendOutput("dispatched", "true"); } catch (error) { @@ -745,6 +766,7 @@ export function findSignalFiles( export async function finishRiskGate(options: { statePath: string; + stateHash: string; evidencePath: string; checkRunId: number; childRunId: number; @@ -757,8 +779,15 @@ export async function finishRiskGate(options: { const { checkRunId, childRunId } = options; const childRunUrl = `https://github.com/${repository}/actions/runs/${childRunId}`; const context = { repository, checkRunId }; - const state = validateRiskGateState(readRegularJson(options.statePath)); try { + if (!HASH_PATTERN.test(options.stateHash)) throw new Error("controller state hash is invalid"); + const serializedState = readPrivateRegularFile(options.statePath, { + maxBytes: MAX_PLAN_BYTES, + })!; + if (sha256(serializedState) !== options.stateHash) { + throw new Error("controller state changed after E2E dispatch"); + } + const state = validateRiskGateState(JSON.parse(serializedState)); const child = await githubApi( `repos/${repository}/actions/runs/${childRunId}`, token, @@ -827,6 +856,7 @@ async function main(): Promise { if (command.mode === "finish") { await finishRiskGate({ statePath: command.statePath, + stateHash: command.stateHash, evidencePath: command.evidencePath, checkRunId: command.checkRunId, childRunId: command.childRunId,