diff --git a/.agents/skills/nemoclaw-maintainer-e2e/references/manual-pr.md b/.agents/skills/nemoclaw-maintainer-e2e/references/manual-pr.md index 2a6ff9fe8c..1080df6f89 100644 --- a/.agents/skills/nemoclaw-maintainer-e2e/references/manual-pr.md +++ b/.agents/skills/nemoclaw-maintainer-e2e/references/manual-pr.md @@ -12,9 +12,16 @@ check. Before dispatch, read [Push and Manual PR E2E](../../../../test/e2e/README.md#push-and-manual-pr-e2e) for the selected jobs' credential locations, access, lifetimes, and removal or cleanup boundaries. -Depending on its selection, a manual PR run can expose long-lived inference, Brave Search, and -messaging credentials to candidate-controlled jobs. Some jobs also expose a read-only job-scoped -`GITHUB_TOKEN`, messaging identifiers, or external resources. +An empty-selector NVIDIA-owned PR run can expose these values to candidate-controlled jobs: + +- long-lived NVIDIA inference and Brave Search API keys; +- Docker Hub credentials through the job's temporary Docker configuration; +- long-lived Telegram, Discord, and Slack credentials; +- a read-only job-scoped `GITHUB_TOKEN` in jobs that need repository access; and +- messaging account and channel identifiers. + +The workflow does not revoke long-lived credentials or erase identifiers copied by candidate code. +GitHub invalidates the job-scoped token after the job ends. Before dispatch, review the complete candidate diff. After a failure: @@ -22,8 +29,10 @@ Before dispatch, review the complete candidate diff. After a failure: - remove resources that cleanup left behind; and - rotate or revoke exposed credentials when necessary. -`Exact staging Brev Launchable` is not available to manual PR runs. The checked-in workflow jobs -define narrower trusted-host boundaries for protected managed-image and native-runtime qualification. +`Exact staging Brev Launchable` is available only when the source is an NVIDIA-owned branch in +`NVIDIA/NemoClaw`. Its trusted host receives the Brev API key and image-dispatch token. The guest +receives the NVIDIA inference API key. The protected managed-image and native-runtime qualification +jobs define narrower trusted-host boundaries in the workflow. ## Resolve and Authorize the Revision @@ -34,39 +43,38 @@ git fetch --prune origin main WORKFLOW_SHA="$(git rev-parse origin/main)" PR_JSON="$(gh api "repos/NVIDIA/NemoClaw/pulls/${PR_NUMBER}")" test "$(jq -r .state <<<"$PR_JSON")" = open +test "$(jq -r .base.repo.full_name <<<"$PR_JSON")" = NVIDIA/NemoClaw +test "$(jq -r .base.ref <<<"$PR_JSON")" = main HEAD_SHA="$(jq -r .head.sha <<<"$PR_JSON")" BASE_SHA="$(jq -r .base.sha <<<"$PR_JSON")" HEAD_REPOSITORY="$(jq -r .head.repo.full_name <<<"$PR_JSON")" +HEAD_OWNER="$(jq -r .head.repo.owner.login <<<"$PR_JSON")" +HEAD_OWNER_TYPE="$(jq -r .head.repo.owner.type <<<"$PR_JSON")" [[ "$HEAD_SHA" =~ ^[0-9a-f]{40}$ ]] [[ "$BASE_SHA" =~ ^[0-9a-f]{40}$ ]] [[ "$WORKFLOW_SHA" =~ ^[0-9a-f]{40}$ ]] ``` -Require a review reason containing 10 to 500 printable characters. Choose one allowed selection: +Choose the selection from the source owner: -- Empty jobs and targets: the trusted default PR selection. -- `jobs=inference-routing`. -- `jobs=managed-image-protected-runtime`. The candidate must contain `ci/protected-managed-image-multiarch-activation-v1.json` and `ci/protected-managed-image-runtime-activation-v1.json`. -- `jobs=native-runtime-qualification-producer` for a same-repository PR and first workflow attempt. The workflow SHA and PR base SHA must match. Set `NATIVE_RUNTIME_EPHEMERAL_RUNNER_POOL=enabled` and `NATIVE_RUNTIME_ARM64_GPU_RUNNER_LABEL` to the reviewed runner label. The candidate must contain `test/e2e/live/native-runtime-qualification-case.test.ts`. -- `targets=jetson-nvmap-gpu` with `allow_jetson_dispatch=true`, only after the operator confirms that - the service is available and compatible with HTTP contract version `1.0.0` and that - `JETSON_DISPATCH_URL` is set to the verified HTTPS origin. See - [Jetson Dispatch Controller](../../../../test/e2e/docs/jetson-dispatch.md). +- An NVIDIA-owned PR can use the full candidate plan and credential profiles. Empty selectors run + every default-enabled E2E. Any supported job or target selector is allowed. +- An external PR keeps the credential-free controller selection. Empty selectors run the trusted + default PR selection. The controller also permits `jobs=inference-routing`, + `jobs=managed-image-protected-runtime`, `jobs=native-runtime-qualification-producer`, or the + documented credential-free target selectors. -Do not rerun the same native-runtime workflow attempt. +Jetson and Launchable runs require a branch in `NVIDIA/NemoClaw`. Jetson also requires +`allow_jetson_dispatch=true` and the reviewed service configuration in +[Jetson Dispatch Controller](../../../../test/e2e/docs/jetson-dispatch.md). -Set one selector and its required flag, or leave all of them empty: +Set the requested selectors and flags, or leave them empty: ```bash E2E_JOBS="${E2E_JOBS:-}" E2E_TARGETS="${E2E_TARGETS:-}" ALLOW_JETSON_DISPATCH="${ALLOW_JETSON_DISPATCH:-false}" -case "${E2E_JOBS}:${E2E_TARGETS}:${ALLOW_JETSON_DISPATCH}" in - ::false | inference-routing::false | managed-image-protected-runtime::false | native-runtime-qualification-producer::false | :jetson-nvmap-gpu:true) ;; - *) echo "Unsupported manual PR E2E selection" >&2; exit 1 ;; -esac - -REVIEW_REASON='Reviewed the latest PR commit and selected E2E boundary.' +INCLUDE_STAGING_BREV_LAUNCHABLE="${INCLUDE_STAGING_BREV_LAUNCHABLE:-false}" CORRELATION_ID="$(python3 -c 'import uuid; print(uuid.uuid4())')" gh workflow run .github/workflows/e2e.yaml \ --repo NVIDIA/NemoClaw \ @@ -74,7 +82,7 @@ gh workflow run .github/workflows/e2e.yaml \ -f "targets=${E2E_TARGETS}" \ -f "jobs=${E2E_JOBS}" \ -f inference_mode=mock \ - -f include_staging_brev_launchable=false \ + -f "include_staging_brev_launchable=${INCLUDE_STAGING_BREV_LAUNCHABLE}" \ -f "allow_jetson_dispatch=${ALLOW_JETSON_DISPATCH}" \ -f allow_dgx_spark_runner_queue=false \ -f "pr_number=${PR_NUMBER}" \ @@ -82,24 +90,23 @@ gh workflow run .github/workflows/e2e.yaml \ -f "checkout_repository=${HEAD_REPOSITORY}" \ -f "base_sha=${BASE_SHA}" \ -f "workflow_sha=${WORKFLOW_SHA}" \ - -f "review_reason=${REVIEW_REASON}" \ -f "correlation_id=${CORRELATION_ID}" ``` -The trusted pre-checkout step requires the actor to have repository `maintain` or `admin` permission -at dispatch time. It validates: +GitHub's permission to dispatch the workflow authorizes the actor. The workflow does not repeat that +repository-role check. The trusted pre-checkout step validates: -- the actor; - the open PR; -- the source repository; +- the target repository and branch; +- the source repository and owner; - the latest PR commit SHA; - the base SHA; -- the workflow SHA; -- the review reason; and -- the selector combination. +- the workflow SHA; and +- whether the source can receive the selected jobs and credentials. It then records and uploads the immutable `nemoclaw-e2e-dispatch-v2` receipt before candidate -execution. A second validation after checkout rejects changed PR identity. +execution. The matrix planner and its dependencies come from the trusted workflow commit. A second +validation after checkout rejects changed PR identity or ownership. ## Find and Verify the Run @@ -122,7 +129,7 @@ RUN_ID="$(jq -r '.[0].databaseId' <<<"$MATCHES")" gh run watch "$RUN_ID" --repo NVIDIA/NemoClaw --exit-status RUN_JSON="$(gh api "repos/NVIDIA/NemoClaw/actions/runs/${RUN_ID}")" jq -e --arg sha "$WORKFLOW_SHA" ' - .run_attempt == 1 and .head_sha == $sha and + .run_attempt >= 1 and .head_sha == $sha and .status == "completed" and .conclusion == "success" ' <<<"$RUN_JSON" >/dev/null CURRENT_PR="$(gh api "repos/NVIDIA/NemoClaw/pulls/${PR_NUMBER}")" @@ -130,6 +137,8 @@ test "$(jq -r .state <<<"$CURRENT_PR")" = open test "$(jq -r .head.sha <<<"$CURRENT_PR")" = "$HEAD_SHA" test "$(jq -r .base.sha <<<"$CURRENT_PR")" = "$BASE_SHA" test "$(jq -r .head.repo.full_name <<<"$CURRENT_PR")" = "$HEAD_REPOSITORY" +test "$(jq -r .head.repo.owner.login <<<"$CURRENT_PR")" = "$HEAD_OWNER" +test "$(jq -r .head.repo.owner.type <<<"$CURRENT_PR")" = "$HEAD_OWNER_TYPE" ``` If the run is not visible after bounded polling, do not dispatch again. Inspect GitHub Actions for @@ -139,6 +148,7 @@ Return: - the PR number; - the source repository; +- the source repository owner; - the latest PR commit SHA; - the base SHA; - the workflow SHA; @@ -147,4 +157,4 @@ Return: - the workflow URL; and - the result. -A changed source repository, latest PR commit SHA, or base SHA invalidates the run claim. +A changed source repository, owner, latest PR commit SHA, or base SHA invalidates the run claim. diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 6a5f444787..dbca2faa47 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -63,11 +63,6 @@ on: required: false default: "" type: string - review_reason: - description: Optional review reason containing 10 to 500 printable characters. - required: false - default: "" - type: string base_sha: description: Optional lowercase PR base SHA for manual exact-revision E2E. required: false @@ -92,7 +87,7 @@ permissions: concurrency: group: e2e-${{ github.ref }}-${{ inputs.checkout_sha != '' && format('pr-{0}', inputs.pr_number) || (inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '' && format('full-{0}', github.run_id)) || inputs.targets || 'supported' }}-${{ inputs.checkout_sha != '' && 'manual-pr' || inputs.jobs || 'all-jobs' }} - cancel-in-progress: ${{ inputs.checkout_sha != '' && !inputs.allow_jetson_dispatch }} + cancel-in-progress: ${{ inputs.checkout_sha != '' && !inputs.allow_jetson_dispatch && !contains(format(',{0},', inputs.jobs), ',staging-brev-launchable,') && !inputs.include_staging_brev_launchable }} env: NEMOCLAW_E2E_EXPECTED_SHA: ${{ inputs.checkout_sha }} @@ -194,41 +189,6 @@ jobs: catalogue_brave_nvidia_inference_matrix: ${{ steps.matrix.outputs.catalogue_brave_nvidia_inference_matrix }} runner_routing: ${{ steps.runner_routing.outputs.runner_routing }} steps: - - id: controller_matrix - name: Build trusted controller target matrix - if: ${{ inputs.checkout_sha != '' }} - env: - JOBS: ${{ inputs.jobs }} - TARGETS: ${{ inputs.targets }} - shell: bash - run: | - set -euo pipefail - test_matrix='[]' - case "${JOBS}:${TARGETS}" in - :) - matrix='[{"id":"ubuntu-policy-custom-missing-presets-negative","runner":"ubuntu-latest"},{"id":"ubuntu-repo-cloud-langchain-deepagents-code","runner":"ubuntu-latest"},{"id":"ubuntu-repo-cloud-openclaw","runner":"ubuntu-latest"},{"id":"ubuntu-repo-docker-post-reboot-recovery","runner":"ubuntu-latest"}]' - test_matrix='[{"id":"onboard-managed-image-buildless-e2e","file":"test/onboard-managed-image-buildless-e2e.test.ts","project":"integration"},{"id":"vllm-docker-storage","file":"test/vllm-docker-storage.test.ts","project":"integration"}]' - ;; - inference-routing: | managed-image-protected-runtime: | native-runtime-qualification-producer: | :jetson-nvmap-gpu) - matrix='[]' - ;; - :ubuntu-repo-cloud-langchain-deepagents-code) - matrix='[{"id":"ubuntu-repo-cloud-langchain-deepagents-code","runner":"ubuntu-latest","label":"ubuntu-repo-cloud-langchain-deepagents-code"}]' - ;; - :ubuntu-repo-docker-post-reboot-recovery) - matrix='[{"id":"ubuntu-repo-docker-post-reboot-recovery","runner":"ubuntu-latest","label":"ubuntu-repo-docker-post-reboot-recovery"}]' - ;; - :ubuntu-repo-cloud-langchain-deepagents-code,ubuntu-repo-docker-post-reboot-recovery) - matrix='[{"id":"ubuntu-repo-cloud-langchain-deepagents-code","runner":"ubuntu-latest","label":"ubuntu-repo-cloud-langchain-deepagents-code"},{"id":"ubuntu-repo-docker-post-reboot-recovery","runner":"ubuntu-latest","label":"ubuntu-repo-docker-post-reboot-recovery"}]' - ;; - *) - echo "::error::PR E2E target is not approved by the trusted controller" >&2 - exit 1 - ;; - esac - printf 'matrix=%s\n' "${matrix}" >> "${GITHUB_OUTPUT}" - printf 'test_matrix=%s\n' "${test_matrix}" >> "${GITHUB_OUTPUT}" - - id: runner_routing name: Build trusted larger-runner routing env: @@ -250,24 +210,19 @@ jobs: runner_routing="$(jq -cn --arg standard "ubuntu-latest" --arg larger "${larger_runner}" '{"channels-stop-start-hermes":$larger,"common-egress-agent":$larger,"hermes-discord":$larger,"hermes-e2e":$larger,"hermes-inference-switch":$larger,"hermes-shields-config":$larger,"mcp-bridge-deepagents":$larger,"mcp-bridge-hermes":$larger,"mcp-bridge-openclaw":$standard,"rebuild-hermes":$larger,"rebuild-hermes-stale-base":$larger,"security-posture-hermes":$larger}')" printf 'runner_routing=%s\n' "${runner_routing}" >> "${GITHUB_OUTPUT}" - - name: Authenticate manual PR dispatch - if: ${{ inputs.checkout_sha != '' }} + - id: candidate_authorization + name: Authenticate manual PR dispatch + if: ${{ inputs.pr_number != '' || inputs.checkout_sha != '' || inputs.checkout_repository != '' || inputs.base_sha != '' || inputs.workflow_sha != '' }} env: - ACTOR: ${{ github.actor }} - ALLOW_JETSON_DISPATCH: ${{ inputs.allow_jetson_dispatch && 'true' || 'false' }} BASE_SHA: ${{ inputs.base_sha }} CHECKOUT_REPOSITORY: ${{ inputs.checkout_repository }} CHECKOUT_SHA: ${{ inputs.checkout_sha }} GITHUB_TOKEN: ${{ github.token }} EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }} - INCLUDE_LAUNCHABLE: ${{ inputs.include_staging_brev_launchable }} + INCLUDE_LAUNCHABLE: ${{ inputs.include_staging_brev_launchable && 'true' || 'false' }} JOBS: ${{ inputs.jobs }} PR_NUMBER: ${{ inputs.pr_number }} - REVIEW_REASON: ${{ inputs.review_reason }} - RUN_ATTEMPT: ${{ github.run_attempt }} - TARGETS: ${{ inputs.targets }} - TRIGGERING_ACTOR: ${{ github.triggering_actor }} WORKFLOW_EVENT: ${{ github.event_name }} WORKFLOW_REF: ${{ github.ref }} WORKFLOW_SHA: ${{ github.workflow_sha }} @@ -276,115 +231,72 @@ jobs: run: | set -euo pipefail - read_collaborator_permission() { - local maintainer="$1" - local attempt curl_exit failure http_status permission_file - permission_file="$(mktemp "${RUNNER_TEMP:-/tmp}/nemoclaw-collaborator-permission.XXXXXX")" - - for attempt in 1 2 3; do - : >"$permission_file" - if http_status="$(curl --silent --proto '=https' --connect-timeout 10 --max-time 30 \ - --output "$permission_file" --write-out "%{http_code}" \ - --header "Authorization: Bearer ${GITHUB_TOKEN}" \ - --header "Accept: application/vnd.github+json" \ - --header "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/${GITHUB_REPOSITORY}/collaborators/${maintainer}/permission" \ - 2>/dev/null)"; then - if [[ "$http_status" =~ ^2[0-9]{2}$ ]]; then - if jq -e 'type == "object" and (.user.login | type == "string") and (.role_name | type == "string")' "$permission_file" >/dev/null 2>&1; then - if (( attempt > 1 )); then - echo "::notice::Collaborator permission read passed after retry on attempt ${attempt}/3" >&2 - fi - cat "$permission_file" - rm -f "$permission_file" - return 0 - fi - echo "::error::Collaborator permission read attempt ${attempt}/3 failed: malformed response" >&2 - rm -f "$permission_file" - return 1 - fi - if [[ "$http_status" =~ ^[0-9]{3}$ ]]; then - failure="HTTP ${http_status}" - case "$http_status" in - 408 | 429 | 5??) ;; - *) echo "::error::Collaborator permission read attempt ${attempt}/3 failed: ${failure}" >&2; rm -f "$permission_file"; return 1 ;; - esac - else - echo "::error::Collaborator permission read attempt ${attempt}/3 failed: invalid HTTP status" >&2 - rm -f "$permission_file" - return 1 - fi - else - curl_exit=$? - case "$curl_exit" in - 5 | 6 | 7 | 16 | 18 | 28 | 35 | 52 | 55 | 56 | 92 | 95 | 96) failure="transport" ;; - *) echo "::error::Collaborator permission read attempt ${attempt}/3 failed: curl exit ${curl_exit}" >&2; rm -f "$permission_file"; return 1 ;; - esac - fi - - if (( attempt == 3 )); then - echo "::error::Collaborator permission read exhausted after attempt ${attempt}/3: ${failure}" >&2 - rm -f "$permission_file" - return 1 - fi - echo "::warning::Collaborator permission read attempt ${attempt}/3 failed: ${failure}; retrying" >&2 - sleep "$attempt" - done - } - - require_maintainer() { - local maintainer="$1" - [[ "$maintainer" =~ ^[A-Za-z0-9-]{1,39}$ && "$maintainer" != -* && "$maintainer" != *- ]] || { - echo "::error::Manual PR E2E actor is invalid" >&2 - exit 1 - } - local permission_json - permission_json="$(read_collaborator_permission "$maintainer")" - if [[ "$(jq -r '.user.login // ""' <<< "$permission_json" | tr '[:upper:]' '[:lower:]')" != "$(tr '[:upper:]' '[:lower:]' <<< "$maintainer")" ]]; then - echo "::error::Manual PR E2E permission response did not match the actor" >&2 - exit 1 - fi - case "$(jq -r '.role_name // ""' <<< "$permission_json")" in - maintain | admin) ;; - *) echo "::error::Manual PR E2E requires a repository maintainer or administrator" >&2; exit 1 ;; - esac - } - [[ "$WORKFLOW_EVENT" == "workflow_dispatch" && "$WORKFLOW_REF" == "refs/heads/main" ]] || { echo "::error::Manual PR E2E must be dispatched from main" >&2 exit 1 } - [[ "$RUN_ATTEMPT" == "1" ]] || { echo "::error::Manual PR E2E cannot be rerun" >&2; exit 1; } - case "${JOBS}:${TARGETS}:${INCLUDE_LAUNCHABLE}:${ALLOW_JETSON_DISPATCH}" in - ::false:false | inference-routing::false:false | managed-image-protected-runtime::false:false | native-runtime-qualification-producer::false:false | :jetson-nvmap-gpu:false:true) ;; - *) - echo "::error::Manual PR E2E accepts only empty selectors, inference-routing, managed-image-protected-runtime, native-runtime-qualification-producer, or jetson-nvmap-gpu with its dispatch flag" >&2 - exit 1 - ;; - esac [[ "$PR_NUMBER" =~ ^[1-9][0-9]*$ ]] || { echo "::error::pr_number must be a positive integer" >&2; exit 1; } [[ "$CHECKOUT_REPOSITORY" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || { echo "::error::checkout_repository must be an owner/repository name" >&2; exit 1; } [[ "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ ]] || { echo "::error::checkout_sha must be a lowercase 40-character SHA" >&2; exit 1; } [[ "$BASE_SHA" =~ ^[a-f0-9]{40}$ ]] || { echo "::error::base_sha must be a lowercase 40-character SHA" >&2; exit 1; } - [[ "$REVIEW_REASON" =~ ^[[:print:]]+$ ]] && - (( ${#REVIEW_REASON} >= 10 && ${#REVIEW_REASON} <= 500 )) || - { echo "::error::review_reason must contain 10 to 500 printable characters" >&2; exit 1; } [[ "$EXPECTED_WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ && "$EXPECTED_WORKFLOW_SHA" == "$WORKFLOW_SHA" ]] || { echo "::error::workflow_sha must match the trusted main workflow SHA" >&2; exit 1; } - require_maintainer "$ACTOR" - if [[ "$(printf '%s' "$TRIGGERING_ACTOR" | tr '[:upper:]' '[:lower:]')" != "$(printf '%s' "$ACTOR" | tr '[:upper:]' '[:lower:]')" ]]; then - require_maintainer "$TRIGGERING_ACTOR" - fi - pull_json="$(curl --fail --silent --show-error --proto '=https' \ --header "Authorization: Bearer ${GITHUB_TOKEN}" \ --header "Accept: application/vnd.github+json" \ --header "X-GitHub-Api-Version: 2022-11-28" \ "https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}")" [[ "$(jq -r '.state' <<< "$pull_json")" == "open" ]] || { echo "::error::pull request must be open" >&2; exit 1; } + [[ "$(jq -r '.base.repo.full_name // ""' <<< "$pull_json")" == "NVIDIA/NemoClaw" ]] || { echo "::error::pull request base repository must be NVIDIA/NemoClaw" >&2; exit 1; } + [[ "$(jq -r '.base.ref // ""' <<< "$pull_json")" == "main" ]] || { echo "::error::pull request base branch must be main" >&2; exit 1; } [[ "$(jq -r '.head.repo.full_name // ""' <<< "$pull_json")" == "$CHECKOUT_REPOSITORY" ]] || { echo "::error::checkout_repository must match the PR source repository" >&2; exit 1; } [[ "$(jq -r '.head.sha' <<< "$pull_json")" == "$CHECKOUT_SHA" ]] || { echo "::error::checkout_sha must match the latest PR commit SHA" >&2; exit 1; } [[ "$(jq -r '.base.sha' <<< "$pull_json")" == "$BASE_SHA" ]] || { echo "::error::base_sha must match the PR base SHA" >&2; exit 1; } + nvidia_owned=false + if [[ "$(jq -r '.head.repo.owner.login // ""' <<< "$pull_json")" == "NVIDIA" && + "$(jq -r '.head.repo.owner.type // ""' <<< "$pull_json")" == "Organization" ]]; then + nvidia_owned=true + fi + if [[ "$INCLUDE_LAUNCHABLE" == "true" || ",${JOBS}," == *",staging-brev-launchable,"* ]]; then + [[ "$nvidia_owned" == "true" ]] || { echo "::error::Launchable PR E2E requires an NVIDIA-owned source repository" >&2; exit 1; } + [[ "$CHECKOUT_REPOSITORY" == "NVIDIA/NemoClaw" ]] || { echo "::error::Launchable PR E2E requires a branch in NVIDIA/NemoClaw" >&2; exit 1; } + fi + printf 'nvidia_owned=%s\n' "$nvidia_owned" >> "$GITHUB_OUTPUT" + + - id: controller_matrix + name: Build trusted controller target matrix + if: ${{ inputs.checkout_sha != '' && steps.candidate_authorization.outputs.nvidia_owned != 'true' }} + env: + JOBS: ${{ inputs.jobs }} + TARGETS: ${{ inputs.targets }} + shell: bash + run: | + set -euo pipefail + test_matrix='[]' + case "${JOBS}:${TARGETS}" in + :) + matrix='[{"id":"ubuntu-policy-custom-missing-presets-negative","runner":"ubuntu-latest"},{"id":"ubuntu-repo-cloud-langchain-deepagents-code","runner":"ubuntu-latest"},{"id":"ubuntu-repo-cloud-openclaw","runner":"ubuntu-latest"},{"id":"ubuntu-repo-docker-post-reboot-recovery","runner":"ubuntu-latest"}]' + test_matrix='[{"id":"onboard-managed-image-buildless-e2e","file":"test/onboard-managed-image-buildless-e2e.test.ts","project":"integration"},{"id":"vllm-docker-storage","file":"test/vllm-docker-storage.test.ts","project":"integration"}]' + ;; + inference-routing: | managed-image-protected-runtime: | native-runtime-qualification-producer: | :jetson-nvmap-gpu) + matrix='[]' + ;; + :ubuntu-repo-cloud-langchain-deepagents-code) + matrix='[{"id":"ubuntu-repo-cloud-langchain-deepagents-code","runner":"ubuntu-latest","label":"ubuntu-repo-cloud-langchain-deepagents-code"}]' + ;; + :ubuntu-repo-docker-post-reboot-recovery) + matrix='[{"id":"ubuntu-repo-docker-post-reboot-recovery","runner":"ubuntu-latest","label":"ubuntu-repo-docker-post-reboot-recovery"}]' + ;; + :ubuntu-repo-cloud-langchain-deepagents-code,ubuntu-repo-docker-post-reboot-recovery) + matrix='[{"id":"ubuntu-repo-cloud-langchain-deepagents-code","runner":"ubuntu-latest","label":"ubuntu-repo-cloud-langchain-deepagents-code"},{"id":"ubuntu-repo-docker-post-reboot-recovery","runner":"ubuntu-latest","label":"ubuntu-repo-docker-post-reboot-recovery"}]' + ;; + *) + echo "::error::PR E2E target is not approved by the trusted controller" >&2 + exit 1 + ;; + esac + printf 'matrix=%s\n' "${matrix}" >> "${GITHUB_OUTPUT}" + printf 'test_matrix=%s\n' "${test_matrix}" >> "${GITHUB_OUTPUT}" - name: Record trusted E2E dispatch receipt if: ${{ github.event_name == 'workflow_dispatch' }} @@ -459,7 +371,7 @@ jobs: path: ${{ runner.temp }}/nemoclaw-e2e-dispatch/dispatch.json - name: Authorize Launchable E2E maintainer dispatch - if: ${{ github.event_name == 'workflow_dispatch' && ((inputs.jobs == 'staging-brev-launchable' && inputs.targets == '') || (inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '')) }} + if: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '' && ((inputs.jobs == 'staging-brev-launchable' && inputs.targets == '') || (inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '')) }} env: ACTOR: ${{ github.actor }} GITHUB_TOKEN: ${{ github.token }} @@ -551,7 +463,81 @@ jobs: require_maintainer "$TRIGGERING_ACTOR" fi - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Check out trusted E2E planner + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ github.repository }} + ref: ${{ github.workflow_sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Set up Node for trusted E2E planning + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 22 + + - name: Install trusted E2E planner dependencies + run: npm ci --ignore-scripts --no-audit --no-fund + + - id: matrix + name: Generate E2E target matrix + env: + CHECKOUT_SHA: ${{ inputs.checkout_sha }} + CONTROLLER_MATRIX: ${{ steps.controller_matrix.outputs.matrix }} + CONTROLLER_TEST_MATRIX: ${{ steps.controller_matrix.outputs.test_matrix }} + INFERENCE_MODE: ${{ inputs.inference_mode || 'mock' }} + JOBS: ${{ inputs.jobs }} + TARGETS: ${{ inputs.targets }} + EVENT_NAME: ${{ github.event_name }} + BEFORE_SHA: ${{ github.event.before }} + CANDIDATE_SHA: ${{ github.sha }} + NEMOCLAW_E2E_CREDENTIALS_ALLOWED: ${{ (inputs.checkout_sha == '' || steps.candidate_authorization.outputs.nvidia_owned == 'true') && 'true' || 'false' }} + NVIDIA_OWNED: ${{ steps.candidate_authorization.outputs.nvidia_owned }} + run: | + set -euo pipefail + if [[ "${JOBS:-}" == "native-runtime-qualification-producer" && -z "${TARGETS:-}" ]]; then + { + echo 'matrix=[]' + echo 'test_matrix=[]' + echo 'catalogue_standard_matrix=[]' + echo 'catalogue_nvidia_api_matrix=[]' + echo 'catalogue_nvidia_inference_matrix=[]' + echo 'catalogue_github_read_matrix=[]' + echo 'catalogue_brave_nvidia_inference_matrix=[]' + echo 'selected_jobs=["native-runtime-qualification-producer"]' + echo 'selected_workflow_jobs=["native-runtime-qualification-producer"]' + echo 'hermes_selected=false' + echo 'explicit_only_jobs=native-runtime-qualification-producer' + echo 'release_required_jobs=[]' + } >>"$GITHUB_OUTPUT" + exit 0 + fi + if [ "${EVENT_NAME:-}" = "push" ]; then + [[ "${BEFORE_SHA}" =~ ^[a-f0-9]{40}$ && "${CANDIDATE_SHA}" =~ ^[a-f0-9]{40}$ ]] || { + echo "::error::Main push E2E requires exact before and candidate commit SHAs" >&2 + exit 1 + } + CHANGED_FILES="$(git diff --name-only --diff-filter=ACMRD "${BEFORE_SHA}" "${CANDIDATE_SHA}")" + export CHANGED_FILES + fi + npx --no-install tsx tools/e2e/workflow-plan.mts --ci-output + if [ -n "${CHECKOUT_SHA}" ] && [ "${NVIDIA_OWNED}" != "true" ]; then + matrix="$(sed -n 's/^matrix=//p' "${GITHUB_OUTPUT}")" + expected_controller_matrix="$(jq -c 'map({id, runner}) | sort_by(.id)' <<< "${CONTROLLER_MATRIX}")" + actual_controller_matrix="$(jq -c 'map({id, runner}) | sort_by(.id)' <<< "${matrix}")" + test_matrix="$(sed -n 's/^test_matrix=//p' "${GITHUB_OUTPUT}")" + expected_controller_test_matrix="$(jq -c 'map({id, file, project}) | sort_by(.id)' <<< "${CONTROLLER_TEST_MATRIX}")" + actual_controller_test_matrix="$(jq -c 'map({id, file, project}) | sort_by(.id)' <<< "${test_matrix}")" + if [ "${actual_controller_matrix}" != "${expected_controller_matrix}" ] || + [ "${actual_controller_test_matrix}" != "${expected_controller_test_matrix}" ]; then + : > "${GITHUB_OUTPUT}" + echo "::error::E2E planner matrix does not match controller-selected targets or shared tests" >&2 + exit 1 + fi + fi + + - name: Check out E2E candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 if: ${{ inputs.checkout_sha == '' || inputs.jobs != 'native-runtime-qualification-producer' || inputs.targets != '' }} with: repository: ${{ inputs.checkout_repository || github.repository }} @@ -566,6 +552,7 @@ jobs: CHECKOUT_REPOSITORY: ${{ inputs.checkout_repository }} CHECKOUT_SHA: ${{ inputs.checkout_sha }} GITHUB_TOKEN: ${{ github.token }} + NVIDIA_OWNED: ${{ steps.candidate_authorization.outputs.nvidia_owned }} PR_NUMBER: ${{ inputs.pr_number }} shell: bash run: | @@ -577,9 +564,18 @@ jobs: --header "X-GitHub-Api-Version: 2022-11-28" \ "https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}")" [[ "$(jq -r '.state' <<< "$pull_json")" == "open" ]] || { echo "::error::pull request must still be open" >&2; exit 1; } + [[ "$(jq -r '.base.repo.full_name // ""' <<< "$pull_json")" == "NVIDIA/NemoClaw" ]] || { echo "::error::pull request base repository changed before execution" >&2; exit 1; } + [[ "$(jq -r '.base.ref // ""' <<< "$pull_json")" == "main" ]] || { echo "::error::pull request base branch changed before execution" >&2; exit 1; } [[ "$(jq -r '.head.repo.full_name // ""' <<< "$pull_json")" == "$CHECKOUT_REPOSITORY" ]] || { echo "::error::checkout_repository changed before execution" >&2; exit 1; } [[ "$(jq -r '.head.sha' <<< "$pull_json")" == "$CHECKOUT_SHA" ]] || { echo "::error::checkout_sha changed before execution" >&2; exit 1; } [[ "$(jq -r '.base.sha' <<< "$pull_json")" == "$BASE_SHA" ]] || { echo "::error::base_sha changed before execution" >&2; exit 1; } + if [[ "$NVIDIA_OWNED" == "true" ]]; then + [[ "$(jq -r '.head.repo.owner.login // ""' <<< "$pull_json")" == "NVIDIA" && + "$(jq -r '.head.repo.owner.type // ""' <<< "$pull_json")" == "Organization" ]] || { + echo "::error::PR source repository ownership changed before execution" >&2 + exit 1 + } + fi - id: e2e_credentials name: Authorize E2E credentials @@ -589,6 +585,7 @@ jobs: CHECKOUT_SHA: ${{ inputs.checkout_sha }} EVENT_NAME: ${{ github.event_name }} EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }} + NVIDIA_OWNED: ${{ steps.candidate_authorization.outputs.nvidia_owned }} REF: ${{ github.ref }} WORKFLOW_REPOSITORY: ${{ github.repository }} WORKFLOW_SHA: ${{ github.workflow_sha }} @@ -597,7 +594,7 @@ jobs: set -euo pipefail credentials_allowed=false if [[ "$WORKFLOW_REPOSITORY" == "NVIDIA/NemoClaw" && - "$CHECKOUT_REPOSITORY" == "$WORKFLOW_REPOSITORY" && + "$NVIDIA_OWNED" == "true" && "$EVENT_NAME" == "workflow_dispatch" && "$REF" == "refs/heads/main" && "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ && @@ -788,61 +785,6 @@ jobs: echo "- Payload digest: \`${PAYLOAD_SHA256}\`" } >>"$GITHUB_STEP_SUMMARY" - - id: matrix - name: Generate E2E target matrix - env: - CHECKOUT_SHA: ${{ inputs.checkout_sha }} - CONTROLLER_MATRIX: ${{ steps.controller_matrix.outputs.matrix }} - CONTROLLER_TEST_MATRIX: ${{ steps.controller_matrix.outputs.test_matrix }} - INFERENCE_MODE: ${{ inputs.inference_mode || 'mock' }} - JOBS: ${{ inputs.jobs }} - TARGETS: ${{ inputs.targets }} - EVENT_NAME: ${{ github.event_name }} - BEFORE_SHA: ${{ github.event.before }} - CANDIDATE_SHA: ${{ github.sha }} - run: | - set -euo pipefail - if [[ "${JOBS:-}" == "native-runtime-qualification-producer" && -z "${TARGETS:-}" ]]; then - { - echo 'matrix=[]' - echo 'test_matrix=[]' - echo 'catalogue_standard_matrix=[]' - echo 'catalogue_nvidia_api_matrix=[]' - echo 'catalogue_nvidia_inference_matrix=[]' - echo 'catalogue_github_read_matrix=[]' - echo 'catalogue_brave_nvidia_inference_matrix=[]' - echo 'selected_jobs=["native-runtime-qualification-producer"]' - echo 'selected_workflow_jobs=["native-runtime-qualification-producer"]' - echo 'hermes_selected=false' - echo 'explicit_only_jobs=native-runtime-qualification-producer' - echo 'release_required_jobs=[]' - } >>"$GITHUB_OUTPUT" - exit 0 - fi - if [ "${EVENT_NAME:-}" = "push" ]; then - [[ "${BEFORE_SHA}" =~ ^[a-f0-9]{40}$ && "${CANDIDATE_SHA}" =~ ^[a-f0-9]{40}$ ]] || { - echo "::error::Main push E2E requires exact before and candidate commit SHAs" >&2 - exit 1 - } - CHANGED_FILES="$(git diff --name-only --diff-filter=ACMRD "${BEFORE_SHA}" "${CANDIDATE_SHA}")" - export CHANGED_FILES - fi - npx tsx tools/e2e/workflow-plan.mts --ci-output - if [ -n "${CHECKOUT_SHA}" ]; then - matrix="$(sed -n 's/^matrix=//p' "${GITHUB_OUTPUT}")" - expected_controller_matrix="$(jq -c 'map({id, runner}) | sort_by(.id)' <<< "${CONTROLLER_MATRIX}")" - actual_controller_matrix="$(jq -c 'map({id, runner}) | sort_by(.id)' <<< "${matrix}")" - test_matrix="$(sed -n 's/^test_matrix=//p' "${GITHUB_OUTPUT}")" - expected_controller_test_matrix="$(jq -c 'map({id, file, project}) | sort_by(.id)' <<< "${CONTROLLER_TEST_MATRIX}")" - actual_controller_test_matrix="$(jq -c 'map({id, file, project}) | sort_by(.id)' <<< "${test_matrix}")" - if [ "${actual_controller_matrix}" != "${expected_controller_matrix}" ] || - [ "${actual_controller_test_matrix}" != "${expected_controller_test_matrix}" ]; then - : > "${GITHUB_OUTPUT}" - echo "::error::E2E planner matrix does not match controller-selected targets or shared tests" >&2 - exit 1 - fi - fi - native-runtime-qualification-producer-plan: needs: generate-matrix if: ${{ github.event_name == 'workflow_dispatch' && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' && inputs.jobs == 'native-runtime-qualification-producer' && inputs.targets == '' }} @@ -2633,9 +2575,9 @@ jobs: name: Authenticate to Docker Hub uses: NVIDIA/NemoClaw/.github/actions/docker-auth-setup@05fa6b810017752ab21148cb7e9d82d12a88c92f with: - auth-required: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && inputs.checkout_sha == '' && '1' || '0' }} - username: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && inputs.checkout_sha == '' && secrets.DOCKERHUB_USERNAME || '' }} - token: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && inputs.checkout_sha == '' && secrets.DOCKERHUB_TOKEN || '' }} + auth-required: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && '1' || '0' }} + username: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_USERNAME || '' }} + token: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_TOKEN || '' }} - name: Configure live E2E trace directory env: @@ -2909,8 +2851,8 @@ jobs: artifact_layout: ${{ matrix.artifact_layout }} trusted_main: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') }} secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + DOCKERHUB_USERNAME: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_USERNAME || '' }} + DOCKERHUB_TOKEN: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_TOKEN || '' }} catalogue-nvidia-api: name: ${{ matrix.display_name }} @@ -2948,9 +2890,9 @@ jobs: artifact_layout: ${{ matrix.artifact_layout }} trusted_main: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') }} secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + DOCKERHUB_USERNAME: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_USERNAME || '' }} + DOCKERHUB_TOKEN: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_TOKEN || '' }} + NVIDIA_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.NVIDIA_API_KEY || '' }} catalogue-nvidia-inference: name: ${{ matrix.display_name }} @@ -2988,9 +2930,9 @@ jobs: artifact_layout: ${{ matrix.artifact_layout }} trusted_main: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') }} secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} + DOCKERHUB_USERNAME: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_USERNAME || '' }} + DOCKERHUB_TOKEN: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_TOKEN || '' }} + NVIDIA_INFERENCE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} catalogue-github-read: name: ${{ matrix.display_name }} @@ -3028,8 +2970,8 @@ jobs: artifact_layout: ${{ matrix.artifact_layout }} trusted_main: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') }} secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + DOCKERHUB_USERNAME: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_USERNAME || '' }} + DOCKERHUB_TOKEN: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_TOKEN || '' }} catalogue-brave-nvidia-inference: name: ${{ matrix.display_name }} @@ -3068,10 +3010,10 @@ jobs: artifact_layout: ${{ matrix.artifact_layout }} trusted_main: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') }} secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }} - NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} + DOCKERHUB_USERNAME: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_USERNAME || '' }} + DOCKERHUB_TOKEN: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_TOKEN || '' }} + BRAVE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.BRAVE_API_KEY || '' }} + NVIDIA_INFERENCE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} openshell-gateway-auth-contract: needs: generate-matrix @@ -4727,7 +4669,7 @@ jobs: - name: Run all-agent GPU, local inference, rollback, and cleanup qualification env: - NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + NVIDIA_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.NVIDIA_API_KEY || '' }} shell: bash run: | set -euo pipefail @@ -4848,7 +4790,7 @@ jobs: - name: Run Hermes live Vitest test env: - NVIDIA_INFERENCE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '' && (inputs.inference_mode || 'mock') != 'mock' && secrets.NVIDIA_INFERENCE_API_KEY || '' }} + NVIDIA_INFERENCE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && (inputs.inference_mode || 'mock') != 'mock' && secrets.NVIDIA_INFERENCE_API_KEY || '' }} run: | set -euo pipefail npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/hermes-e2e.test.ts diff --git a/test/e2e/README.md b/test/e2e/README.md index e57135d38a..080b7ffe83 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -258,11 +258,11 @@ The planner partitions that set into GitHub Actions matrices, one for each execu The execution profile owns the credentials available to its target step: - `standard` displays `no provider credential` and receives no NVIDIA API credential. -- `nvidia-api` displays `NVIDIA API key` and receives `NVIDIA_API_KEY` on trusted `main` runs. -- `nvidia-inference` displays `NVIDIA inference API key` and receives `NVIDIA_INFERENCE_API_KEY` on trusted `main` runs. +- `nvidia-api` displays `NVIDIA API key` and receives `NVIDIA_API_KEY` on trusted `main` runs and authenticated NVIDIA-owned PR runs. +- `nvidia-inference` displays `NVIDIA inference API key` and receives `NVIDIA_INFERENCE_API_KEY` on trusted `main` runs and authenticated NVIDIA-owned PR runs. - `github-read` displays `GitHub read token` and receives the job-scoped `GITHUB_TOKEN` only for the target step when `trusted_main` is `true`. - The reusable workflow enforces this boundary; PR revision callers set `trusted_main` to `false`, so their target steps receive no `GITHUB_TOKEN`. -- `brave-nvidia-inference` displays `Brave and NVIDIA inference API keys` and receives `BRAVE_API_KEY` and `NVIDIA_INFERENCE_API_KEY` on trusted `main` runs. + The reusable workflow enforces this boundary; an authenticated NVIDIA-owned PR caller sets `trusted_main` to `true`, while an external PR caller sets it to `false` and receives no `GITHUB_TOKEN`. +- `brave-nvidia-inference` displays `Brave and NVIDIA inference API keys` and receives `BRAVE_API_KEY` and `NVIDIA_INFERENCE_API_KEY` on trusted `main` runs and authenticated NVIDIA-owned PR runs. `common-egress-agent` runs 4 isolated scenario shards. The Personal stock-price shard exercises ordinary onboarding with an explicit Personal selection; it does not exercise Portable profile selection. @@ -1144,6 +1144,8 @@ teardown discards that filesystem. These credentials remain valid until they expire or an administrator revokes them in their issuing services. If cleanup fails, remove the recorded Brev workspace. Rotate or revoke each credential to remove later access. +For an NVIDIA-owned PR revision, the job builds and runs the exact candidate commit with this same credential boundary. +The PR branch must be in `NVIDIA/NemoClaw` because the image producer does not accept a sibling-repository candidate. The `NEMOCLAW_STAGING_LAUNCHABLE_ID` repository Actions variable selects the standing Launchable. Keep its value equal to the Launchable ID in the default @@ -1159,15 +1161,21 @@ The `totalRunnerMinutes` field contains the cumulative runner time for those sum A later successful attempt sets `action` to `passed-after-retry` and `flaky` to `true`. The observer ignores manual PR runs and a run superseded by a newer `main` push. -For a PR revision run, a repository maintainer or administrator leaves `jobs` and `targets` empty. The run selects: +GitHub's workflow-dispatch permission is the actor authorization for a PR revision run. +The workflow does not add a second `maintain` or `admin` role gate. +Before checkout, it verifies the open PR, exact target repository and `main` branch, current source repository and commit, base commit, and trusted workflow commit from the GitHub PR API. + +When the API reports that the PR source repository owner is the `NVIDIA` organization, empty `jobs` and `targets` select: - every default-selected free-standing workflow E2E except `Exact staging Brev Launchable`; -- every catalogue target in the `standard` profile; +- every catalogue target across all credential profiles; - every shared credential-free test; and -- these controller-selected registry targets: `ubuntu-policy-custom-missing-presets-negative`, `ubuntu-repo-cloud-langchain-deepagents-code`, `ubuntu-repo-cloud-openclaw`, and `ubuntu-repo-docker-post-reboot-recovery`. +- every default registry target. -The PR selection does not forward an NVIDIA API key, `BRAVE_API_KEY`, or `GITHUB_TOKEN` to the candidate checkout. +An NVIDIA-owned PR may also select any supported E2E job or target. +For an external PR, the bounded controller matrix remains in effect and the workflow does not forward repository credentials to candidate-controlled processes or reusable workflow callers. The run skips `jetson-nvmap-gpu` unless `allow_jetson_dispatch` is `true`. +Jetson and Launchable dispatch additionally require the PR branch to be in `NVIDIA/NemoClaw`; their operator and image-producer backends do not accept a sibling-repository candidate. It skips `llama-cpp-dgx-spark-plan` and `llama-cpp-dgx-spark-qualification` unless their runner-queue flag is `true`. The trusted workflow definition remains on `main` and binds the latest PR commit to the current PR base SHA. @@ -1185,9 +1193,10 @@ The risk plan selects the `openshell-gateway-upgrade` catalogue target and the The catalogue target covers the installer-driven OpenShell gateway upgrade handoff. The typed target covers the LangChain Deep Agents Code sandbox recreation path. -A trusted manual `main` run with empty selectors exposes these values to candidate-controlled job processes: +An NVIDIA-owned PR run with empty selectors exposes these values to candidate-controlled job processes: - Long-lived API keys from repository secrets: `NVIDIA_INFERENCE_API_KEY`, `NVIDIA_API_KEY`, and `BRAVE_API_KEY`. +- Docker Hub credentials from `DOCKERHUB_USERNAME` and `DOCKERHUB_TOKEN`, available to candidate processes through the job's temporary Docker configuration until cleanup. - Long-lived messaging credentials from repository secrets: `TELEGRAM_BOT_TOKEN_REAL`, `DISCORD_BOT_TOKEN_REAL`, `SLACK_BOT_TOKEN_REAL`, and `SLACK_APP_TOKEN_REAL`. - The job-scoped `GITHUB_TOKEN`, exposed only to the target step in the `token-rotation` and `openshell-gateway-upgrade` catalogue executions. It has `contents: read` access. @@ -1195,7 +1204,7 @@ A trusted manual `main` run with empty selectors exposes these values to candida GitHub Actions invalidates it after the reusable workflow job. - Messaging account and channel identifiers from repository secrets: `TELEGRAM_ALLOWED_IDS`, `TELEGRAM_AUTHORIZED_CHAT_IDS`, `TELEGRAM_CHAT_ID`, `TELEGRAM_CHAT_ID_E2E`, `DISCORD_CHANNEL_ID_E2E`, and `SLACK_CHANNEL_ID_E2E`. -The workflow does not rotate or revoke these API keys or messaging credentials. To remove later access, rotate or revoke every listed credential in the external service that issued it. The workflow cannot erase identifiers copied by candidate code. Review the complete candidate diff before dispatch. +The workflow does not rotate or revoke these API keys, Docker Hub credentials, or messaging credentials. To remove later access, rotate or revoke every listed credential in the external service that issued it. The workflow cannot erase identifiers copied by candidate code. Review the complete candidate diff before dispatch. Live targets can create external resources. After a failure, inspect the workflow artifacts and remove resources that target cleanup did not remove. @@ -1213,8 +1222,7 @@ Verify that the old value is invalid. After you accept this credential boundary, dispatch `native-runtime-qualification-producer` from trusted `main` for a same-repository open PR. Use the first workflow attempt. The executing workflow commit and `workflow_sha` input must equal the exact PR-recorded base commit. -The actor must have repository `maintain` or `admin` permission. -If `github.triggering_actor` differs from the actor, it must also have one of those permissions. +The dispatcher must have GitHub permission to run the workflow; the E2E workflow adds no second actor-role check. The trusted workflow binds the candidate commit, base commit, workflow commit, repository, PR, run, attempt, and 24-case plan. The unprivileged installer and live-test processes run with `env -i` under a temporary account. @@ -1248,9 +1256,8 @@ For a manual PR run, provide these inputs: - The PR source repository. - The lowercase 40-character PR base SHA. - The exact SHA of the trusted workflow commit on `main`. -- A review reason containing 10 to 500 printable characters. -For the default PR revision selection, leave `jobs` and `targets` empty and keep `include_staging_brev_launchable=false`. +For the default NVIDIA-owned PR revision selection, leave `jobs` and `targets` empty and keep `include_staging_brev_launchable=false`. Keep `allow_jetson_dispatch=false` and `allow_dgx_spark_runner_queue=false` for the default PR revision selection. If `allow_dgx_spark_runner_queue=true`, GitHub can pause the qualification job for the `approve-dgx-spark-image-qualification` environment. An authorized environment reviewer must approve it before qualification starts. @@ -1262,8 +1269,10 @@ To select native runtime qualification evidence production, set `jobs=native-run Leave `targets` empty and keep `include_staging_brev_launchable=false`. For this producer run, the executing workflow SHA, `workflow_sha` input, and PR base SHA must match. Confirm that the PR comes from `NVIDIA/NemoClaw`, the required ephemeral runner variables are configured, and the workflow has not been rerun. -A trusted `main` workflow pre-checkout step requires current `maintain` or `admin` permission. The workflow validates the exact open PR and selected mode before candidate code runs. -A second validation after checkout rejects a changed candidate commit, base commit, or PR source repository before preparation. +A trusted `main` workflow pre-checkout step validates the exact open PR and records whether its source repository has API-confirmed `NVIDIA` organization ownership. +That ownership authorizes the full ordinary plan and credential profiles; external sources retain the bounded controller plan. +A second validation after checkout rejects a changed candidate commit, base commit, PR source repository, or NVIDIA ownership before preparation. +Candidate runs cannot publish release qualification. The Actions run is advisory for the pull request and is not a required merge context. Treat it as passing evidence only when the `E2E` workflow concludes with `success` for the recorded PR number, PR source repository, candidate commit SHA, base commit SHA, and executing workflow SHA. @@ -1297,10 +1306,10 @@ configuration, or the unified E2E workflow. Compatibility schema fields may classify that guidance as required, but rendered advisor guidance remains non-authoritative. Model advice is additive and cannot downgrade the deterministic floor. PR Review Advisor recommendations remain advisory. -A maintainer decides whether to dispatch this trusted selection for the current PR -revision. The manual PR selection includes the credential-free -`inference-routing` catalogue target. It does not dispatch secret-backed targets such as -`network-policy` for PR revisions. The Advisor comment labels that boundary. +A repository-authorized user decides whether to dispatch this trusted selection for the current PR revision. +For API-confirmed NVIDIA-owned sources, the selection may include secret-backed targets such as `network-policy`. +External PR revisions retain the credential-free controller boundary. +The Advisor comment labels the requested coverage, but does not restrict an NVIDIA-owned PR to that recommendation. No PR E2E controller dispatches the risk plan. The `full-e2e` target enforces a separate hard acceptance contract for the diff --git a/test/e2e/support/dockerhub-auth-workflow-boundary.test.ts b/test/e2e/support/dockerhub-auth-workflow-boundary.test.ts index 0b2381ad85..5d36fb16f5 100644 --- a/test/e2e/support/dockerhub-auth-workflow-boundary.test.ts +++ b/test/e2e/support/dockerhub-auth-workflow-boundary.test.ts @@ -330,7 +330,7 @@ describe("shared Docker Hub authentication workflow boundary (#6961)", () => { const errors = validateMutation((workflow) => { const auth = namedStep(workflow.jobs.live, AUTH_STEP_NAME)!; const ungatedPredicate = - "github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')"; + "github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && inputs.checkout_sha == ''"; auth.with = { "auth-required": `\${{ ${ungatedPredicate} && '1' || '0' }}`, username: `\${{ ${ungatedPredicate} && secrets.DOCKERHUB_USERNAME || '' }}`, diff --git a/test/e2e/support/e2e-collaborator-permission-retry.test.ts b/test/e2e/support/e2e-collaborator-permission-retry.test.ts index 2b019bbd0c..5090a340d6 100644 --- a/test/e2e/support/e2e-collaborator-permission-retry.test.ts +++ b/test/e2e/support/e2e-collaborator-permission-retry.test.ts @@ -25,11 +25,6 @@ type PermissionScenario = | "transport-exhaustion"; const AUTHORIZATION_STEPS: AuthorizationStep[] = [ - { - deniedMessage: "Manual PR E2E requires a repository maintainer or administrator", - mismatchMessage: "Manual PR E2E permission response did not match the actor", - name: "Authenticate manual PR dispatch", - }, { deniedMessage: "Launchable E2E requires a repository maintainer or administrator", mismatchMessage: "Launchable E2E permission response did not match the actor", @@ -104,12 +99,6 @@ if [[ "$url" == *"/collaborators/"*"/permission" ]]; then exit "$curl_exit" fi -if [[ "$url" == *"/pulls/42" ]]; then - printf '%s\n' "pull" >>"$CURL_LOG" - printf '%s' '{"state":"open","head":{"repo":{"full_name":"contributor/NemoClaw"},"sha":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},"base":{"sha":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}}' - exit 0 -fi - exit 2 `, ); @@ -133,12 +122,12 @@ printf '%s\n' "$1" >>"$SLEEP_LOG" ALLOW_JETSON_DISPATCH: "false", BASE_SHA: "b".repeat(40), CHECKOUT_REPOSITORY: "contributor/NemoClaw", - CHECKOUT_SHA: stepName === "Authenticate manual PR dispatch" ? "a".repeat(40) : "", + CHECKOUT_SHA: "", CURL_LOG: curlLog, EXPECTED_WORKFLOW_SHA: workflowSha, GITHUB_REPOSITORY: "NVIDIA/NemoClaw", GITHUB_TOKEN: "private-test-token", - INCLUDE_LAUNCHABLE: stepName === "Authenticate manual PR dispatch" ? "false" : "true", + INCLUDE_LAUNCHABLE: "true", JOBS: "", PATH: `${fixture}:${process.env.PATH ?? ""}`, PERMISSION_ATTEMPT_FILE: attemptFile, diff --git a/test/e2e/support/e2e-operations-workflow-boundary.test.ts b/test/e2e/support/e2e-operations-workflow-boundary.test.ts index 1f00a78449..b85d4d034f 100644 --- a/test/e2e/support/e2e-operations-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-operations-workflow-boundary.test.ts @@ -244,7 +244,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; it("validates manual PR dispatch inputs and the checked-out commit", () => { const workflow = readE2eOperationsWorkflow(); - delete workflow.on?.workflow_dispatch?.inputs?.review_reason; + delete workflow.on?.workflow_dispatch?.inputs?.checkout_repository; const authentication = workflow.jobs["generate-matrix"].steps!.find( (step) => step.name === "Authenticate manual PR dispatch", )!; @@ -256,7 +256,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; expect(validateE2eOperationsWorkflow(workflow)).toEqual( expect.arrayContaining([ - "workflow_dispatch review_reason must be an optional string with an empty default", + "workflow_dispatch checkout_repository must be an optional string with an empty default", 'Manual PR authentication must retain "$WORKFLOW_EVENT" == "workflow_dispatch"', 'Manual PR authentication must retain "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$', 'Manual PR checkout validation must retain "$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA"', @@ -264,6 +264,35 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; ); }); + it("runs authentication when any candidate identity input is present and skips it when all are empty", () => { + const workflow = readE2eOperationsWorkflow(); + const authentication = workflow.jobs["generate-matrix"].steps!.find( + (step) => step.name === "Authenticate manual PR dispatch", + )!; + const validationError = + "Manual PR authentication must run when any candidate identity input is present"; + + expect(validateE2eOperationsWorkflow(workflow)).not.toContain(validationError); + expect(authentication.if).toBe( + "${{ inputs.pr_number != '' || inputs.checkout_sha != '' || inputs.checkout_repository != '' || inputs.base_sha != '' || inputs.workflow_sha != '' }}", + ); + + authentication.if = "${{ inputs.checkout_sha != '' }}"; + expect(validateE2eOperationsWorkflow(workflow)).toContain(validationError); + }); + + it("pins the trusted planner checkout to the workflow repository", () => { + const workflow = readE2eOperationsWorkflow(); + const checkout = workflow.jobs["generate-matrix"].steps!.find( + (step) => step.name === "Check out trusted E2E planner", + )!; + checkout.with!.repository = "untrusted/repository"; + + expect(validateE2eOperationsWorkflow(workflow)).toContain( + "generate-matrix checkout must use the selected PR source repository", + ); + }); + it("rejects changes that bypass E2E credential authorization (#9047)", () => { const workflow = readE2eOperationsWorkflow(); delete workflow.jobs["generate-matrix"].outputs!.e2e_credentials_allowed; @@ -276,6 +305,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; expect.arrayContaining([ "Manual PR credential authorization must expose only the authorization result", 'Manual PR credential authorization must retain "$WORKFLOW_REPOSITORY" == "NVIDIA/NemoClaw"', + 'Manual PR credential authorization must retain "$NVIDIA_OWNED" == "true"', 'Manual PR credential authorization must retain "$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA"', ]), ); @@ -285,14 +315,25 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; { caseName: "matching repository and requested SHAs", checkoutRepository: "NVIDIA/NemoClaw", + nvidiaOwned: true, workflowRepository: "NVIDIA/NemoClaw", checkoutShaMatches: true, workflowShaMatches: true, expectedAllowed: true, }, { - caseName: "a checkout repository outside NVIDIA/NemoClaw", + caseName: "an NVIDIA-owned sibling repository", + checkoutRepository: "NVIDIA/NemoClaw-E2E", + nvidiaOwned: true, + workflowRepository: "NVIDIA/NemoClaw", + checkoutShaMatches: true, + workflowShaMatches: true, + expectedAllowed: true, + }, + { + caseName: "a checkout repository outside NVIDIA", checkoutRepository: "contributor/NemoClaw", + nvidiaOwned: false, workflowRepository: "NVIDIA/NemoClaw", checkoutShaMatches: true, workflowShaMatches: true, @@ -301,6 +342,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; { caseName: "a workflow repository outside NVIDIA/NemoClaw", checkoutRepository: "NVIDIA/NemoClaw", + nvidiaOwned: true, workflowRepository: "contributor/NemoClaw", checkoutShaMatches: true, workflowShaMatches: true, @@ -309,6 +351,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; { caseName: "checkout_sha differs from the checked-out commit", checkoutRepository: "NVIDIA/NemoClaw", + nvidiaOwned: true, workflowRepository: "NVIDIA/NemoClaw", checkoutShaMatches: false, workflowShaMatches: true, @@ -317,6 +360,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; { caseName: "a requested workflow SHA that differs from the running workflow", checkoutRepository: "NVIDIA/NemoClaw", + nvidiaOwned: true, workflowRepository: "NVIDIA/NemoClaw", checkoutShaMatches: true, workflowShaMatches: false, @@ -326,6 +370,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; "sets E2E credential access to $expectedAllowed for $caseName (#9047)", ({ checkoutRepository, + nvidiaOwned, workflowRepository, checkoutShaMatches, workflowShaMatches, @@ -358,6 +403,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; EVENT_NAME: "workflow_dispatch", EXPECTED_WORKFLOW_SHA: expectedWorkflowSha, GITHUB_OUTPUT: output, + NVIDIA_OWNED: nvidiaOwned ? "true" : "false", REF: "refs/heads/main", WORKFLOW_REPOSITORY: workflowRepository, WORKFLOW_SHA: workflowSha, @@ -400,46 +446,28 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; ); }); - it("limits manual PR runs to controller-approved selectors or Jetson dispatch", () => { + it("requires API-confirmed NVIDIA organization ownership for full PR E2E", () => { const workflow = readE2eOperationsWorkflow(); const authentication = workflow.jobs["generate-matrix"].steps!.find( (step) => step.name === "Authenticate manual PR dispatch", )!; authentication.run = authentication.run!.replace( - "Manual PR E2E accepts only empty selectors, inference-routing, managed-image-protected-runtime, native-runtime-qualification-producer, or jetson-nvmap-gpu with its dispatch flag", - "Manual PR E2E accepts arbitrary selectors", - ); - - expect(validateE2eOperationsWorkflow(workflow)).toContain( - "Manual PR authentication must retain Manual PR E2E accepts only empty selectors, inference-routing, managed-image-protected-runtime, native-runtime-qualification-producer, or jetson-nvmap-gpu with its dispatch flag", + `"$(jq -r '.head.repo.owner.type // ""' <<< "$pull_json")" == "Organization"`, + `"$(jq -r '.head.repo.owner.type // ""' <<< "$pull_json")" == "User"`, ); - }); - - it("uses the same controller selectors as the PR Review Advisor", () => { - const workflow = readE2eOperationsWorkflow(); - const authentication = workflow.jobs["generate-matrix"].steps!.find( - (step) => step.name === "Authenticate manual PR dispatch", - )!; - authentication.run = authentication.run!.replace("inference-routing::false:false | ", ""); expect(validateE2eOperationsWorkflow(workflow)).toContain( - "Manual PR authentication must retain ::false:false | inference-routing::false:false | managed-image-protected-runtime::false:false | native-runtime-qualification-producer::false:false | :jetson-nvmap-gpu:false:true) ;;", + `Manual PR authentication must retain "$(jq -r '.head.repo.owner.type // ""' <<< "$pull_json")" == "Organization"`, ); }); it.each([ - ["maintain", "", "", "false", 0, ""], - ["maintain", "inference-routing", "", "false", 0, ""], - ["maintain", "managed-image-protected-runtime", "", "false", 0, ""], - ["maintain", "native-runtime-qualification-producer", "", "false", 0, ""], - ["maintain", "", "jetson-nvmap-gpu", "true", 0, ""], - ["maintain", "", "jetson-nvmap-gpu", "false", 1, "accepts only empty selectors"], - ["maintain", "network-policy", "", "false", 1, "accepts only empty selectors"], - ["maintain", "gpu-e2e", "", "false", 1, "accepts only empty selectors"], - ["write", "", "", "false", 1, "requires a repository maintainer or administrator"], + ["NVIDIA organization", "NVIDIA", "Organization", true], + ["external organization", "contributor", "Organization", false], + ["lookalike user", "NVIDIA", "User", false], ])( - "requires a maintainer role and bounded selector before manual PR E2E for %s with jobs %s and targets %s", - (role, jobs, targets, allowJetsonDispatch, expectedStatus, expectedStderr) => { + "records NVIDIA ownership for a %s without a duplicate actor-role gate", + (_caseName, ownerLogin, ownerType, expectedNvidiaOwned) => { const workflow = readE2eOperationsWorkflow(); const authentication = workflow.jobs["generate-matrix"].steps!.find( (step) => step.name === "Authenticate manual PR dispatch", @@ -447,20 +475,18 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; const headSha = "a".repeat(40); const baseSha = "b".repeat(40); const workflowSha = "c".repeat(40); + const checkoutRepository = `${ownerLogin}/NemoClaw`; const prefix = [ "curl() {", - ' local url="${@: -1}" output_file="" previous="" argument body', - ' for argument in "$@"; do', - ' if [[ "$previous" == "--output" ]]; then output_file="$argument"; fi', - ' previous="$argument"', - " done", - ' case "$url" in', - ` *collaborators*) body='{"user":{"login":"maintainer"},"role_name":"${role}"}'; if [[ -n "$output_file" ]]; then printf '%s' "$body" >"$output_file"; printf '200'; else printf '%s' "$body"; fi ;;`, - ` *pulls/42) printf '%s' '{"state":"open","head":{"repo":{"full_name":"contributor/NemoClaw"},"sha":"${headSha}"},"base":{"sha":"${baseSha}"}}' ;;`, + ' case "${@: -1}" in', + ` *pulls/42) printf '%s' '{"state":"open","head":{"repo":{"full_name":"${checkoutRepository}","owner":{"login":"${ownerLogin}","type":"${ownerType}"}},"sha":"${headSha}"},"base":{"repo":{"full_name":"NVIDIA/NemoClaw"},"ref":"main","sha":"${baseSha}"}}' ;;`, " *) return 1 ;;", " esac", "}", ].join("\n"); + const directory = mkdtempSync(join(tmpdir(), "nemoclaw-pr-owner-")); + const output = join(directory, "output"); + writeFileSync(output, ""); const result = spawnSync( "bash", ["--noprofile", "--norc", "-e", "-o", "pipefail", "-c", `${prefix}\n${authentication.run}`], @@ -468,21 +494,196 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; encoding: "utf8", env: { ...process.env, - ACTOR: "maintainer", - ALLOW_JETSON_DISPATCH: allowJetsonDispatch, BASE_SHA: baseSha, - CHECKOUT_REPOSITORY: "contributor/NemoClaw", + CHECKOUT_REPOSITORY: checkoutRepository, CHECKOUT_SHA: headSha, EXPECTED_WORKFLOW_SHA: workflowSha, GITHUB_REPOSITORY: "NVIDIA/NemoClaw", GITHUB_TOKEN: "token", + GITHUB_OUTPUT: output, + INCLUDE_LAUNCHABLE: "false", + JOBS: "", + PR_NUMBER: "42", + WORKFLOW_EVENT: "workflow_dispatch", + WORKFLOW_REF: "refs/heads/main", + WORKFLOW_SHA: workflowSha, + }, + }, + ); + + try { + expect(result.status, result.stderr).toBe(0); + expect(readFileSync(output, "utf8")).toBe( + `nvidia_owned=${expectedNvidiaOwned ? "true" : "false"}\n`, + ); + expect(authentication.run).not.toContain("collaborators/"); + expect(authentication.run).not.toContain("role_name"); + } finally { + rmSync(directory, { force: true, recursive: true }); + } + }, + ); + + it.each([ + [ + "an invalid source repository name", + "invalid-repository", + "a", + "b", + "c", + "::error::checkout_repository must be an owner/repository name\n", + ], + [ + "a PR commit mismatch", + "NVIDIA/NemoClaw", + "d", + "b", + "c", + "::error::checkout_sha must match the latest PR commit SHA\n", + ], + [ + "a PR base commit mismatch", + "NVIDIA/NemoClaw", + "a", + "d", + "c", + "::error::base_sha must match the PR base SHA\n", + ], + [ + "a trusted workflow commit mismatch", + "NVIDIA/NemoClaw", + "a", + "b", + "d", + "::error::workflow_sha must match the trusted main workflow SHA\n", + ], + ] as const)( + "rejects manual PR authentication for %s", + ( + _caseName, + requestedRepository, + requestedHeadCharacter, + requestedBaseCharacter, + expectedWorkflowCharacter, + expectedStderr, + ) => { + const apiHeadSha = "a".repeat(40); + const apiBaseSha = "b".repeat(40); + const workflowSha = "c".repeat(40); + const workflow = readE2eOperationsWorkflow(); + const authentication = workflow.jobs["generate-matrix"].steps!.find( + (step) => step.name === "Authenticate manual PR dispatch", + )!; + const prefix = [ + "curl() {", + ` printf '%s' '{"state":"open","head":{"repo":{"full_name":"NVIDIA/NemoClaw","owner":{"login":"NVIDIA","type":"Organization"}},"sha":"${apiHeadSha}"},"base":{"repo":{"full_name":"NVIDIA/NemoClaw"},"ref":"main","sha":"${apiBaseSha}"}}'`, + "}", + ].join("\n"); + const result = spawnSync( + "bash", + ["--noprofile", "--norc", "-e", "-o", "pipefail", "-c", `${prefix}\n${authentication.run}`], + { + encoding: "utf8", + env: { + ...process.env, + BASE_SHA: requestedBaseCharacter.repeat(40), + CHECKOUT_REPOSITORY: requestedRepository, + CHECKOUT_SHA: requestedHeadCharacter.repeat(40), + EXPECTED_WORKFLOW_SHA: expectedWorkflowCharacter.repeat(40), + GITHUB_OUTPUT: "/dev/null", + GITHUB_REPOSITORY: "NVIDIA/NemoClaw", + GITHUB_TOKEN: "token", INCLUDE_LAUNCHABLE: "false", + JOBS: "", + PR_NUMBER: "42", + WORKFLOW_EVENT: "workflow_dispatch", + WORKFLOW_REF: "refs/heads/main", + WORKFLOW_SHA: workflowSha, + }, + }, + ); + + expect(result.status).toBe(1); + expect(result.stderr).toBe(expectedStderr); + }, + ); + + it.each([ + ["NVIDIA inclusion flag", "NVIDIA/NemoClaw", "NVIDIA", "Organization", "true", "", 0, ""], + [ + "NVIDIA job selector", + "NVIDIA/NemoClaw", + "NVIDIA", + "Organization", + "false", + "staging-brev-launchable", + 0, + "", + ], + [ + "external source", + "contributor/NemoClaw", + "contributor", + "Organization", + "true", + "", + 1, + "::error::Launchable PR E2E requires an NVIDIA-owned source repository\n", + ], + [ + "NVIDIA sibling repository", + "NVIDIA/NemoClaw-fork", + "NVIDIA", + "Organization", + "false", + "staging-brev-launchable", + 1, + "::error::Launchable PR E2E requires a branch in NVIDIA/NemoClaw\n", + ], + ])( + "authorizes Launchable PR E2E for %s", + ( + _caseName, + checkoutRepository, + ownerLogin, + ownerType, + includeLaunchable, + jobs, + expectedStatus, + expectedStderr, + ) => { + const workflow = readE2eOperationsWorkflow(); + const authentication = workflow.jobs["generate-matrix"].steps!.find( + (step) => step.name === "Authenticate manual PR dispatch", + )!; + const headSha = "a".repeat(40); + const baseSha = "b".repeat(40); + const workflowSha = "c".repeat(40); + const prefix = [ + "curl() {", + ' case "${@: -1}" in', + ` *pulls/42) printf '%s' '{"state":"open","head":{"repo":{"full_name":"${checkoutRepository}","owner":{"login":"${ownerLogin}","type":"${ownerType}"}},"sha":"${headSha}"},"base":{"repo":{"full_name":"NVIDIA/NemoClaw"},"ref":"main","sha":"${baseSha}"}}' ;;`, + " *) return 1 ;;", + " esac", + "}", + ].join("\n"); + const result = spawnSync( + "bash", + ["--noprofile", "--norc", "-c", `${prefix}\n${authentication.run}`], + { + encoding: "utf8", + env: { + ...process.env, + BASE_SHA: baseSha, + CHECKOUT_REPOSITORY: checkoutRepository, + CHECKOUT_SHA: headSha, + EXPECTED_WORKFLOW_SHA: workflowSha, + GITHUB_OUTPUT: "/dev/null", + GITHUB_REPOSITORY: "NVIDIA/NemoClaw", + GITHUB_TOKEN: "token", + INCLUDE_LAUNCHABLE: includeLaunchable, JOBS: jobs, PR_NUMBER: "42", - REVIEW_REASON: "Reviewed latest PR commit", - RUN_ATTEMPT: "1", - TARGETS: targets, - TRIGGERING_ACTOR: "maintainer", WORKFLOW_EVENT: "workflow_dispatch", WORKFLOW_REF: "refs/heads/main", WORKFLOW_SHA: workflowSha, @@ -491,7 +692,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; ); expect(result.status, result.stderr).toBe(expectedStatus); - expect(result.stderr).toContain(expectedStderr); + expect(result.stderr).toBe(expectedStderr); }, ); @@ -574,6 +775,8 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; GITHUB_STEP_SUMMARY: summary, INFERENCE_MODE: "mock", JOBS: "", + NEMOCLAW_E2E_CREDENTIALS_ALLOWED: "false", + NVIDIA_OWNED: "false", TARGETS: "", }, }, @@ -657,6 +860,8 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; GITHUB_STEP_SUMMARY: summary, INFERENCE_MODE: "mock", JOBS: jobSelector, + NEMOCLAW_E2E_CREDENTIALS_ALLOWED: "false", + NVIDIA_OWNED: "false", TARGETS: targetSelector, }, }, diff --git a/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts b/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts index e72ed4faad..188cbceb8b 100644 --- a/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts @@ -97,7 +97,7 @@ function executeGenerateMatrixWithPlannerOutput( fakeNpx, [ "#!/usr/bin/env bash", - "expected=(tsx tools/e2e/workflow-plan.mts --ci-output)", + "expected=(--no-install tsx tools/e2e/workflow-plan.mts --ci-output)", 'actual=("$@")', '[[ "${#actual[@]}" -eq "${#expected[@]}" ]] || exit 97', 'for index in "${!expected[@]}"; do [[ "${actual[$index]}" == "${expected[$index]}" ]] || exit 97; done', @@ -124,6 +124,8 @@ function executeGenerateMatrixWithPlannerOutput( GITHUB_STEP_SUMMARY: path.join(directory, "summary.md"), INFERENCE_MODE: "mock", JOBS: options.jobs ?? "cloud-onboard", + NEMOCLAW_E2E_CREDENTIALS_ALLOWED: "false", + NVIDIA_OWNED: "false", PATH: `${binDirectory}:${process.env.PATH ?? ""}`, TARGETS: options.targets ?? "", }, diff --git a/test/e2e/support/hermes-workflow-boundary.test.ts b/test/e2e/support/hermes-workflow-boundary.test.ts index d01c7c82e7..b487b4683e 100644 --- a/test/e2e/support/hermes-workflow-boundary.test.ts +++ b/test/e2e/support/hermes-workflow-boundary.test.ts @@ -219,13 +219,13 @@ describe("Hermes GPU boundary", () => { expect(errors).toEqual( expect.arrayContaining([ expect.stringContaining( - "hermes-e2e run step must guard NVIDIA_INFERENCE_API_KEY behind a trusted main-branch dispatch", + "hermes-e2e run step must guard NVIDIA_INFERENCE_API_KEY behind a direct main dispatch or an authorized NVIDIA-owned PR dispatch", ), ]), ); }); - it("rejects live secret exposure to a PR checkout", () => { + it("rejects live secret exposure to a PR checkout without authorization", () => { const errors = wfErrors((workflow) => { const run = step(workflow.jobs["hermes-e2e"], "Run Hermes live Vitest test"); run.env = { @@ -237,7 +237,7 @@ describe("Hermes GPU boundary", () => { expect(errors).toEqual( expect.arrayContaining([ expect.stringContaining( - "hermes-e2e run step must guard NVIDIA_INFERENCE_API_KEY behind a trusted main-branch dispatch", + "hermes-e2e run step must guard NVIDIA_INFERENCE_API_KEY behind a direct main dispatch or an authorized NVIDIA-owned PR dispatch", ), ]), ); diff --git a/test/e2e/support/jetson-workflow-boundary.test.ts b/test/e2e/support/jetson-workflow-boundary.test.ts index 5092621e08..3f5e656a0c 100644 --- a/test/e2e/support/jetson-workflow-boundary.test.ts +++ b/test/e2e/support/jetson-workflow-boundary.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest"; import { + validateE2eWorkflow, validateE2eWorkflowBoundary, validateJetsonDispatchBoundary, } from "../../../tools/e2e/workflow-boundary.mts"; @@ -76,6 +77,20 @@ describe("Jetson nvmap GPU E2E workflow boundary", () => { ); }); + it.each([ + ["Jetson", "${{ inputs.checkout_sha != '' && !inputs.include_staging_brev_launchable }}"], + ["Launchable", "${{ inputs.checkout_sha != '' && !inputs.allow_jetson_dispatch }}"], + ])("rejects concurrency that cancels active %s dispatches", (_dispatch, cancellation) => { + const workflow = readWorkflow(); + const validationError = + "workflow concurrency must not cancel an active Jetson or Launchable dispatch"; + expect(validateE2eWorkflow(workflow)).not.toContain(validationError); + const concurrency = workflow.concurrency as Record; + concurrency["cancel-in-progress"] = cancellation; + + expect(validateE2eWorkflow(workflow)).toContain(validationError); + }); + it("rejects candidate execution or credential-bearing controller steps (#8142)", () => { const errors = validateWorkflowMutation((workflow) => { const job = (workflow.jobs as Record)["jetson-nvmap-gpu"] as { diff --git a/test/e2e/support/managed-image-protected-runtime-workflow.test.ts b/test/e2e/support/managed-image-protected-runtime-workflow.test.ts index 9c29e02cdb..148e1ad4b1 100644 --- a/test/e2e/support/managed-image-protected-runtime-workflow.test.ts +++ b/test/e2e/support/managed-image-protected-runtime-workflow.test.ts @@ -143,6 +143,23 @@ describe("protected managed-image runtime workflow", () => { ); }); + it("rejects an unguarded qualification credential", () => { + const value = workflow(); + const qualification = namedStep( + value, + "Run all-agent GPU, local inference, rollback, and cleanup qualification", + ); + qualification.env = { NVIDIA_API_KEY: "${{ secrets.NVIDIA_API_KEY }}" }; + + expect(validateManagedImageProtectedRuntimeWorkflow(value)).toEqual( + expect.arrayContaining([ + expect.stringContaining( + "managed-image-protected-runtime qualification env must bind NVIDIA_API_KEY", + ), + ]), + ); + }); + it("prevents the credentialed qualification step from executing .candidate-runtime files", () => { const value = workflow(); const qualification = namedStep( diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index 572cb2744e..16049dce24 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -43,7 +43,7 @@ function expectRequiredPodmanPackages(run: string): void { describe("native runtime qualification producer workflow", () => { it("keeps candidate execution out of the authenticated controller", () => { const generate = job("generate-matrix"); - const checkout = generate.steps?.find((entry) => entry.uses?.startsWith("actions/checkout@")); + const checkout = step(generate, "Check out E2E candidate"); expect(checkout?.if).toContain("inputs.jobs != 'native-runtime-qualification-producer'"); expect(step(generate, "Validate manual PR checkout").if).toContain( diff --git a/test/e2e/support/standard-profile-workflow-boundary.test.ts b/test/e2e/support/standard-profile-workflow-boundary.test.ts index 51240513b7..a2bb836f5c 100644 --- a/test/e2e/support/standard-profile-workflow-boundary.test.ts +++ b/test/e2e/support/standard-profile-workflow-boundary.test.ts @@ -54,6 +54,18 @@ describe("standard E2E execution profile", () => { ); }); + it("rejects an unguarded existing catalogue caller secret", () => { + const workflow = readWorkflow() as { + jobs: Record }>; + }; + workflow.jobs["catalogue-nvidia-api"]!.secrets.NVIDIA_API_KEY = + "${{ secrets.NVIDIA_API_KEY }}"; + + expect(validateStandardProfileWorkflowBoundary(workflow)).toContain( + "catalogue-nvidia-api must receive only its profile secrets", + ); + }); + it("rejects catalogue callers that bypass E2E credential authorization (#9047)", () => { const workflow = readWorkflow() as { jobs: Record }>; diff --git a/test/e2e/support/workflow-plan.test.ts b/test/e2e/support/workflow-plan.test.ts index 91f799b1a7..0ce35ea6b8 100644 --- a/test/e2e/support/workflow-plan.test.ts +++ b/test/e2e/support/workflow-plan.test.ts @@ -516,12 +516,11 @@ describe("E2E workflow plan", () => { }, ); - it("omits credentialed catalogue profiles when checkout_sha is set", () => { + it("includes every catalogue profile for an authorized NVIDIA-owned candidate", () => { const directory = mkdtempSync(path.join(tmpdir(), "nemoclaw-workflow-plan-pr-")); const output = path.join(directory, "github-output"); const summary = path.join(directory, "summary.md"); - const plan = prCandidatePlan(buildE2eWorkflowPlan()); - + const plan = buildE2eWorkflowPlan(); try { writeE2eWorkflowPlanCiOutput( {}, @@ -529,6 +528,7 @@ describe("E2E workflow plan", () => { GITHUB_OUTPUT: output, GITHUB_STEP_SUMMARY: summary, INFERENCE_MODE: "mock", + NEMOCLAW_E2E_CREDENTIALS_ALLOWED: "true", NEMOCLAW_E2E_EXPECTED_SHA: "a".repeat(40), }, ); @@ -540,7 +540,68 @@ describe("E2E workflow plan", () => { } }); - it("allows manual PR dispatch only for standard-profile targets", () => { + it("limits an unauthorized candidate without selectors to credential-free matrices", () => { + const directory = mkdtempSync(path.join(tmpdir(), "nemoclaw-workflow-plan-fork-")); + const output = path.join(directory, "github-output"); + const summary = path.join(directory, "summary.md"); + try { + writeE2eWorkflowPlanCiOutput( + {}, + { + GITHUB_OUTPUT: output, + GITHUB_STEP_SUMMARY: summary, + INFERENCE_MODE: "mock", + NEMOCLAW_E2E_CREDENTIALS_ALLOWED: "false", + NEMOCLAW_E2E_EXPECTED_SHA: "a".repeat(40), + }, + ); + + const outputLines = readFileSync(output, "utf8").split("\n"); + expect(outputLines).toEqual( + expect.arrayContaining([ + "catalogue_nvidia_api_matrix=[]", + "catalogue_nvidia_inference_matrix=[]", + "catalogue_github_read_matrix=[]", + "catalogue_brave_nvidia_inference_matrix=[]", + "selected_jobs=[]", + "hermes_selected=false", + ]), + ); + expect(outputLines).not.toContain("catalogue_standard_matrix=[]"); + } finally { + rmSync(directory, { force: true, recursive: true }); + } + }); + + it("retains controller-approved jobs for an unauthorized candidate", () => { + const directory = mkdtempSync(path.join(tmpdir(), "nemoclaw-workflow-plan-fork-jobs-")); + const output = path.join(directory, "github-output"); + const summary = path.join(directory, "summary.md"); + try { + writeE2eWorkflowPlanCiOutput( + { jobs: "managed-image-protected-runtime" }, + { + GITHUB_OUTPUT: output, + GITHUB_STEP_SUMMARY: summary, + INFERENCE_MODE: "mock", + NEMOCLAW_E2E_CREDENTIALS_ALLOWED: "false", + NEMOCLAW_E2E_EXPECTED_SHA: "a".repeat(40), + }, + ); + + const outputLines = readFileSync(output, "utf8").split("\n"); + expect(outputLines).toEqual( + expect.arrayContaining([ + 'selected_jobs=["managed-image-protected-runtime"]', + "hermes_selected=false", + ]), + ); + } finally { + rmSync(directory, { force: true, recursive: true }); + } + }); + + it("classifies only standard-profile targets as credential-free PR candidates", () => { expect( Object.fromEntries( E2E_TARGET_CATALOGUE.map((target) => [ @@ -751,6 +812,7 @@ describe("E2E workflow plan", () => { GITHUB_STEP_SUMMARY: summary, INFERENCE_MODE: "mock", JOBS: "launchable-smoke", + NEMOCLAW_E2E_CREDENTIALS_ALLOWED: "true", TARGETS: "", NEMOCLAW_E2E_EXPECTED_SHA: "a".repeat(40), }, @@ -758,10 +820,9 @@ describe("E2E workflow plan", () => { }); expect(result.status, result.stderr).toBe(0); - const expectedPlan = prCandidatePlan(plan); - expect(readFileSync(output, "utf8")).toBe(expectedCiOutput(expectedPlan)); + expect(readFileSync(output, "utf8")).toBe(expectedCiOutput(plan)); expect(readFileSync(summary, "utf8")).toBe( - renderE2eWorkflowPlanSummary(expectedPlan, { includeCoverageAudit: false }), + renderE2eWorkflowPlanSummary(plan, { includeCoverageAudit: false }), ); } finally { rmSync(directory, { force: true, recursive: true }); @@ -784,6 +845,7 @@ describe("E2E workflow plan", () => { GITHUB_STEP_SUMMARY: summary, INFERENCE_MODE: "mock", JOBS: [activeJobs, ...retiredControllerSelectorIds()].join(","), + NEMOCLAW_E2E_CREDENTIALS_ALLOWED: "true", TARGETS: "", NEMOCLAW_E2E_EXPECTED_SHA: "a".repeat(40), }, @@ -791,10 +853,9 @@ describe("E2E workflow plan", () => { }); expect(result.status, result.stderr).toBe(0); - const expectedPlan = prCandidatePlan(plan); - expect(readFileSync(output, "utf8")).toBe(expectedCiOutput(expectedPlan)); + expect(readFileSync(output, "utf8")).toBe(expectedCiOutput(plan)); expect(readFileSync(summary, "utf8")).toBe( - renderE2eWorkflowPlanSummary(expectedPlan, { includeCoverageAudit: false }), + renderE2eWorkflowPlanSummary(plan, { includeCoverageAudit: false }), ); } finally { rmSync(directory, { force: true, recursive: true }); diff --git a/tools/e2e/managed-image-protected-runtime-workflow-boundary.mts b/tools/e2e/managed-image-protected-runtime-workflow-boundary.mts index 619499ab98..a3dcc6ed2d 100644 --- a/tools/e2e/managed-image-protected-runtime-workflow-boundary.mts +++ b/tools/e2e/managed-image-protected-runtime-workflow-boundary.mts @@ -19,6 +19,8 @@ const ACTIVATION_PATH = "ci/protected-managed-image-runtime-activation-v1.json"; const LIVE_TEST_PATH = "test/e2e/live/managed-image-protected-runtime.test.ts"; const REGISTRY_IMAGE = "docker.io/library/registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373"; +const GUARDED_NVIDIA_API_KEY = + "${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.NVIDIA_API_KEY || '' }}"; // Keep lane-specific trust assertions explicit: the multiarch lane executes // candidate code directly, while this GPU lane keeps secrets in trusted code @@ -298,7 +300,7 @@ export function validateManagedImageProtectedRuntimeWorkflow(workflow: WorkflowR "Run all-agent GPU, local inference, rollback, and cleanup qualification", ); requireValues(errors, `${JOB_ID} qualification env`, record(qualification?.env), { - NVIDIA_API_KEY: "${{ secrets.NVIDIA_API_KEY }}", + NVIDIA_API_KEY: GUARDED_NVIDIA_API_KEY, }); const secretBearingSteps = workflowSteps.filter( (step) => record(step.env).NVIDIA_API_KEY !== undefined, diff --git a/tools/e2e/mcp-dev-workflow-boundary-digests.mts b/tools/e2e/mcp-dev-workflow-boundary-digests.mts index 675f8df7f9..6adab74912 100644 --- a/tools/e2e/mcp-dev-workflow-boundary-digests.mts +++ b/tools/e2e/mcp-dev-workflow-boundary-digests.mts @@ -10,7 +10,7 @@ export const MCP_DEV_JOB_EXECUTION_CONTEXT_SHA256 = export const MCP_DEV_TRUSTED_NODE_SETUP_CONTENT_SHA256 = "504821ad93c57971d0281ef1130ed6008fadd331bd56acb1a6b5e6a3358f3e49"; export const MCP_DEV_TRUSTED_PREFIX_CONTENT_SHA256 = - "c559e6cd5bf076bed8d359bbca397d4e31fbf3c11123389425917b865544940d"; + "067df18297c3b5e5175dc11de071a0f3c261aa894db6141b8b789d67f5e9c0d1"; export const MCP_DEV_POST_INSTALL_TRANSITION_CONTENT_SHA256 = "62cf2ee01ac7192f41fc7b2b071de729da8bacec1e4f693da1ec6f0b1f4723c0"; diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index a80841f1d1..d4ee5dcb91 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -8,7 +8,7 @@ import { isDeepStrictEqual } from "node:util"; import ts from "typescript"; import YAML from "yaml"; -import { PR_E2E_MANUAL_CONTROLLER_JOB_IDS, RISK_RULES } from "../advisors/risk-plan.mts"; +import { RISK_RULES } from "../advisors/risk-plan.mts"; import { validateStandardProfileWorkflowBoundary } from "./standard-profile-workflow-boundary.mts"; import { catalogueTarget, E2E_TARGET_CATALOGUE } from "./target-catalogue.mts"; @@ -230,7 +230,6 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow "pr_number", "checkout_sha", "checkout_repository", - "review_reason", "base_sha", "workflow_sha", "correlation_id", @@ -262,9 +261,11 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow } if ( workflow.concurrency?.["cancel-in-progress"] !== - "${{ inputs.checkout_sha != '' && !inputs.allow_jetson_dispatch }}" + "${{ inputs.checkout_sha != '' && !inputs.allow_jetson_dispatch && !contains(format(',{0},', inputs.jobs), ',staging-brev-launchable,') && !inputs.include_staging_brev_launchable }}" ) { - errors.push("Manual PR E2E concurrency must not cancel an active Jetson dispatch"); + errors.push( + "Manual PR E2E concurrency must not cancel an active Jetson or Launchable dispatch", + ); } const matrixJob = workflow.jobs["generate-matrix"] ?? {}; @@ -272,7 +273,7 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow const authenticationIndex = steps.findIndex( (step) => step.name === "Authenticate manual PR dispatch", ); - const checkoutIndex = steps.findIndex((step) => step.uses?.startsWith("actions/checkout@")); + const checkoutIndex = steps.findIndex((step) => step.name === "Check out E2E candidate"); const validationIndex = steps.findIndex((step) => step.name === "Validate manual PR checkout"); const credentialAuthorizationIndex = steps.findIndex( (step) => step.name === "Authorize E2E credentials", @@ -293,24 +294,22 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow } const authentication = authenticationIndex >= 0 ? steps[authenticationIndex] : {}; - if (authentication.if !== "${{ inputs.checkout_sha != '' }}") { - errors.push("Manual PR authentication must be activated only by checkout_sha"); + if ( + authentication.id !== "candidate_authorization" || + authentication.if !== + "${{ inputs.pr_number != '' || inputs.checkout_sha != '' || inputs.checkout_repository != '' || inputs.base_sha != '' || inputs.workflow_sha != '' }}" + ) { + errors.push("Manual PR authentication must run when any candidate identity input is present"); } const authEnvironment = { - ACTOR: "${{ github.actor }}", - ALLOW_JETSON_DISPATCH: "${{ inputs.allow_jetson_dispatch && 'true' || 'false' }}", BASE_SHA: "${{ inputs.base_sha }}", CHECKOUT_REPOSITORY: "${{ inputs.checkout_repository }}", CHECKOUT_SHA: "${{ inputs.checkout_sha }}", EXPECTED_WORKFLOW_SHA: "${{ inputs.workflow_sha }}", GITHUB_TOKEN: "${{ github.token }}", - INCLUDE_LAUNCHABLE: "${{ inputs.include_staging_brev_launchable }}", + INCLUDE_LAUNCHABLE: "${{ inputs.include_staging_brev_launchable && 'true' || 'false' }}", JOBS: "${{ inputs.jobs }}", PR_NUMBER: "${{ inputs.pr_number }}", - REVIEW_REASON: "${{ inputs.review_reason }}", - RUN_ATTEMPT: "${{ github.run_attempt }}", - TARGETS: "${{ inputs.targets }}", - TRIGGERING_ACTOR: "${{ github.triggering_actor }}", WORKFLOW_EVENT: "${{ github.event_name }}", WORKFLOW_REF: "${{ github.ref }}", WORKFLOW_SHA: "${{ github.workflow_sha }}", @@ -320,37 +319,31 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow errors.push(`Manual PR authentication must bind ${name}`); } const authSource = String(authentication.run ?? ""); - const acceptedJobCases = [ - "::false:false", - ...PR_E2E_MANUAL_CONTROLLER_JOB_IDS.map((jobId) => `${jobId}::false:false`), - "native-runtime-qualification-producer::false:false", - ":jetson-nvmap-gpu:false:true", - ].join(" | "); - const acceptedNames = [ - ...PR_E2E_MANUAL_CONTROLLER_JOB_IDS, - "native-runtime-qualification-producer", - "jetson-nvmap-gpu with its dispatch flag", - ]; - const acceptedJobNames = `${acceptedNames.slice(0, -1).join(", ")}, or ${acceptedNames.at(-1)}`; for (const fragment of [ '"$WORKFLOW_EVENT" == "workflow_dispatch"', '"$WORKFLOW_REF" == "refs/heads/main"', - '"$RUN_ATTEMPT" == "1"', '"$PR_NUMBER" =~ ^[1-9][0-9]*$', '"$CHECKOUT_REPOSITORY" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$', '"$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$', '"$BASE_SHA" =~ ^[a-f0-9]{40}$', - '"$REVIEW_REASON" =~ ^[[:print:]]+$', - "${#REVIEW_REASON} >= 10", - "${#REVIEW_REASON} <= 500", '"$EXPECTED_WORKFLOW_SHA" == "$WORKFLOW_SHA"', - "Manual PR E2E requires a repository maintainer or administrator", - `${acceptedJobCases}) ;;`, - `Manual PR E2E accepts only empty selectors, ${acceptedJobNames}`, "https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}", + `[[ "$(jq -r '.base.repo.full_name // ""' <<< "$pull_json")" == "NVIDIA/NemoClaw" ]]`, + `[[ "$(jq -r '.base.ref // ""' <<< "$pull_json")" == "main" ]]`, `[[ "$(jq -r '.head.repo.full_name // ""' <<< "$pull_json")" == "$CHECKOUT_REPOSITORY" ]]`, `[[ "$(jq -r '.head.sha' <<< "$pull_json")" == "$CHECKOUT_SHA" ]]`, `[[ "$(jq -r '.base.sha' <<< "$pull_json")" == "$BASE_SHA" ]]`, + '"$INCLUDE_LAUNCHABLE" == "true"', + '",${JOBS}," == *",staging-brev-launchable,"*', + '"$nvidia_owned" == "true"', + "Launchable PR E2E requires an NVIDIA-owned source repository", + '"$CHECKOUT_REPOSITORY" == "NVIDIA/NemoClaw"', + "Launchable PR E2E requires a branch in NVIDIA/NemoClaw", + `"$(jq -r '.head.repo.owner.login // ""' <<< "$pull_json")" == "NVIDIA"`, + `"$(jq -r '.head.repo.owner.type // ""' <<< "$pull_json")" == "Organization"`, + "nvidia_owned=false", + "nvidia_owned=true", + `printf 'nvidia_owned=%s\\n' "$nvidia_owned" >> "$GITHUB_OUTPUT"`, ]) { if (!authSource.includes(fragment)) errors.push(`Manual PR authentication must retain ${fragment}`); @@ -381,13 +374,22 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow errors.push("Manual PR checkout validation must skip qualification producer dispatches"); } const validationSource = String(validation.run ?? ""); + if ( + validation.env?.NVIDIA_OWNED !== "${{ steps.candidate_authorization.outputs.nvidia_owned }}" + ) { + errors.push("Manual PR checkout validation must bind authenticated NVIDIA ownership"); + } for (const fragment of [ '"$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA"', "https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}", "pull request must still be open", + "pull request base repository changed before execution", + "pull request base branch changed before execution", "checkout_repository changed before execution", "checkout_sha changed before execution", "base_sha changed before execution", + '"$NVIDIA_OWNED" == "true"', + "PR source repository ownership changed before execution", ]) { if (!validationSource.includes(fragment)) { errors.push(`Manual PR checkout validation must retain ${fragment}`); @@ -410,6 +412,7 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow CHECKOUT_SHA: "${{ inputs.checkout_sha }}", EVENT_NAME: "${{ github.event_name }}", EXPECTED_WORKFLOW_SHA: "${{ inputs.workflow_sha }}", + NVIDIA_OWNED: "${{ steps.candidate_authorization.outputs.nvidia_owned }}", REF: "${{ github.ref }}", WORKFLOW_REPOSITORY: "${{ github.repository }}", WORKFLOW_SHA: "${{ github.workflow_sha }}", @@ -422,7 +425,7 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow const authorizationSource = String(credentialAuthorization.run ?? ""); for (const fragment of [ '"$WORKFLOW_REPOSITORY" == "NVIDIA/NemoClaw"', - '"$CHECKOUT_REPOSITORY" == "$WORKFLOW_REPOSITORY"', + '"$NVIDIA_OWNED" == "true"', '"$EVENT_NAME" == "workflow_dispatch"', '"$REF" == "refs/heads/main"', '"$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$', @@ -445,6 +448,11 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow step.name === "Checkout trusted Hermes GPU runtime fixture" && step.with?.repository === "NVIDIA/NemoClaw" && step.with?.ref === "${{ github.workflow_sha }}"; + const trustedE2ePlannerCheckout = + jobName === "generate-matrix" && + step.name === "Check out trusted E2E planner" && + step.with?.repository === "${{ github.repository }}" && + step.with?.ref === "${{ github.workflow_sha }}"; const trustedReportHelperCheckout = jobName === "report-to-pr" && step.name === "Check out the trusted E2E reporting helper" && @@ -530,6 +538,7 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow step.with?.ref === "${{ github.workflow_sha }}"); const trustedCheckout = trustedHermesFixtureCheckout || + trustedE2ePlannerCheckout || trustedReportHelperCheckout || trustedReleaseQualificationCheckout || trustedRelevantE2eCheckout || @@ -553,7 +562,7 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow !trustedCheckout && step.with?.repository !== "${{ inputs.checkout_repository || github.repository }}" ) { - errors.push(`${jobName} checkout must use the selected PR head repository`); + errors.push(`${jobName} checkout must use the selected PR source repository`); } } } diff --git a/tools/e2e/standard-profile-workflow-boundary.mts b/tools/e2e/standard-profile-workflow-boundary.mts index 22a5b5d0bb..98f593fd26 100644 --- a/tools/e2e/standard-profile-workflow-boundary.mts +++ b/tools/e2e/standard-profile-workflow-boundary.mts @@ -26,6 +26,10 @@ const DEFAULT_PROFILE_PATH = join(REPO_ROOT, ".github", "workflows", "e2e-standa const PROFILE_WORKFLOW = "./.github/workflows/e2e-standard-profile.yaml"; const CHECKOUT = "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1"; const EXECUTION_PLAN_SHELL = "/bin/bash --noprofile --norc -e -o pipefail {0}"; +const TRUSTED_CALLER_CREDENTIAL_PREDICATE = + "github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true')"; +const guardedCallerSecret = (name: string): string => + `\${{ ${TRUSTED_CALLER_CREDENTIAL_PREDICATE} && secrets.${name} || '' }}`; const SKILL_AGENT_UPLOAD_PATH = `${[ "e2e-artifacts/live/skill-agent/evidence-manifest.json", "e2e-artifacts/live/skill-agent/*/artifact-summary.json", @@ -184,7 +188,7 @@ function validateProfileCallers(errors: string[], workflow: WorkflowRecord): voi const callerSecrets = record(job.secrets); if ( Object.keys(callerSecrets).sort().join(",") !== [...contract.secrets].sort().join(",") || - contract.secrets.some((name) => callerSecrets[name] !== `\${{ secrets.${name} }}`) + contract.secrets.some((name) => callerSecrets[name] !== guardedCallerSecret(name)) ) { errors.push(`${contract.job} must receive only its profile secrets`); } diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index 0a90d0945e..c329d7a440 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -207,11 +207,11 @@ const DOCKER_HUB_CLEANUP_KEYS = ["if", "name", "run", "shell"]; // The general E2E workflow runs on push/manual dispatch. Its event set is // intentionally distinct from the reusable image workflow's push/manual boundary. const TRUSTED_DOCKER_HUB_PREDICATE = - "github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && inputs.checkout_sha == ''"; + "github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true')"; const GUARDED_DOCKER_HUB_AUTH_REQUIRED = `\${{ ${TRUSTED_DOCKER_HUB_PREDICATE} && '1' || '0' }}`; const GUARDED_DOCKER_HUB_USERNAME = `\${{ ${TRUSTED_DOCKER_HUB_PREDICATE} && secrets.DOCKERHUB_USERNAME || '' }}`; const GUARDED_DOCKER_HUB_TOKEN = `\${{ ${TRUSTED_DOCKER_HUB_PREDICATE} && secrets.DOCKERHUB_TOKEN || '' }}`; -const GUARDED_HERMES_E2E_INFERENCE_KEY = `\${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '' && (inputs.inference_mode || 'mock') != 'mock' && secrets.NVIDIA_INFERENCE_API_KEY || '' }}`; +const GUARDED_HERMES_E2E_INFERENCE_KEY = `\${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && (inputs.inference_mode || 'mock') != 'mock' && secrets.NVIDIA_INFERENCE_API_KEY || '' }}`; const RUNNER_ROUTING_OUTPUT = "${{ steps.runner_routing.outputs.runner_routing }}"; const RUNNER_ROUTING_STEP_NAME = "Build trusted larger-runner routing"; const RUNNER_ROUTING_SCRIPT = [ @@ -1666,7 +1666,7 @@ function validateHermesE2EJob(errors: string[], jobs: WorkflowRecord): void { const runVitestEnv = asRecord(runVitest?.env); if (runVitestEnv.NVIDIA_INFERENCE_API_KEY !== GUARDED_HERMES_E2E_INFERENCE_KEY) { errors.push( - "hermes-e2e run step must guard NVIDIA_INFERENCE_API_KEY behind a trusted main-branch dispatch without a PR checkout and the inference mode condition", + "hermes-e2e run step must guard NVIDIA_INFERENCE_API_KEY behind a direct main dispatch or an authorized NVIDIA-owned PR dispatch, plus the inference mode condition", ); } requireRunContains(errors, runVitest, "tools/e2e/live-vitest-invocation.mts run --test-path"); @@ -1857,9 +1857,9 @@ function validateFullE2eConcurrency(errors: string[], workflow: WorkflowRecord): } if ( concurrency["cancel-in-progress"] !== - "${{ inputs.checkout_sha != '' && !inputs.allow_jetson_dispatch }}" + "${{ inputs.checkout_sha != '' && !inputs.allow_jetson_dispatch && !contains(format(',{0},', inputs.jobs), ',staging-brev-launchable,') && !inputs.include_staging_brev_launchable }}" ) { - errors.push("workflow concurrency must not cancel an active Jetson dispatch"); + errors.push("workflow concurrency must not cancel an active Jetson or Launchable dispatch"); } } @@ -1895,7 +1895,7 @@ function validateStagingBrevLaunchableJob(errors: string[], jobs: WorkflowRecord "Authorize Launchable E2E maintainer dispatch", ); const expectedAuthorizationSelector = - "${{ github.event_name == 'workflow_dispatch' && ((inputs.jobs == 'staging-brev-launchable' && inputs.targets == '') || (inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '')) }}"; + "${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '' && ((inputs.jobs == 'staging-brev-launchable' && inputs.targets == '') || (inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '')) }}"; if (authorization?.if !== expectedAuthorizationSelector) { errors.push("Launchable E2E maintainer authorization must cover exact and full dispatches"); } @@ -1923,9 +1923,7 @@ function validateStagingBrevLaunchableJob(errors: string[], jobs: WorkflowRecord ]) { requireRunContains(errors, authorization, required); } - const generateCheckout = generateSteps.find((step) => - stringValue(step.uses).startsWith("actions/checkout@"), - ); + const generateCheckout = namedStep(generateSteps, "Check out E2E candidate"); if ( authorization && generateCheckout && @@ -2172,17 +2170,15 @@ function validateTrustedE2eDispatchReceipt( } const authentication = namedStep(generateSteps, "Authenticate manual PR dispatch"); - const candidateCheckout = generateSteps.find((step) => - stringValue(step.uses).startsWith("actions/checkout@"), - ); + const candidateCheckout = namedStep(generateSteps, "Check out E2E candidate"); const authenticationIndex = authentication ? generateSteps.indexOf(authentication) : -1; const receiptIndex = dispatchReceipt ? generateSteps.indexOf(dispatchReceipt) : -1; const uploadIndex = dispatchUpload ? generateSteps.indexOf(dispatchUpload) : -1; const checkoutIndex = candidateCheckout ? generateSteps.indexOf(candidateCheckout) : -1; const trustedPrefix = [ - "Build trusted controller target matrix", "Build trusted larger-runner routing", "Authenticate manual PR dispatch", + "Build trusted controller target matrix", "Record trusted E2E dispatch receipt", "Upload trusted E2E dispatch receipt", ]; @@ -2192,7 +2188,7 @@ function validateTrustedE2eDispatchReceipt( trustedPrefix, ) || authenticationIndex < 0 || - receiptIndex !== authenticationIndex + 1 || + receiptIndex !== authenticationIndex + 2 || uploadIndex !== receiptIndex + 1 || checkoutIndex <= uploadIndex ) { @@ -2202,6 +2198,80 @@ function validateTrustedE2eDispatchReceipt( } } +function validateTrustedE2ePlannerBoundary( + errors: string[], + generateSteps: WorkflowRecord[], + generate: WorkflowRecord | undefined, + candidateCheckout: WorkflowRecord | undefined, +): void { + const trustedPlannerCheckout = requireStep( + errors, + generateSteps, + "Check out trusted E2E planner", + ); + const trustedPlannerSetup = requireStep( + errors, + generateSteps, + "Set up Node for trusted E2E planning", + ); + const trustedPlannerInstall = requireStep( + errors, + generateSteps, + "Install trusted E2E planner dependencies", + ); + requireFullShaAction(errors, trustedPlannerCheckout, "trusted E2E planner checkout"); + if ( + !isDeepStrictEqual(asRecord(trustedPlannerCheckout?.with), { + repository: "${{ github.repository }}", + ref: "${{ github.workflow_sha }}", + "fetch-depth": 0, + "persist-credentials": false, + }) + ) { + errors.push("trusted E2E planner checkout must use the workflow commit without credentials"); + } + requireFullShaAction(errors, trustedPlannerSetup, "trusted E2E planner Node setup"); + if ( + !isDeepStrictEqual(asRecord(trustedPlannerSetup?.with), { + "node-version": 22, + }) + ) { + errors.push("trusted E2E planner must use Node 22"); + } + if (trustedPlannerInstall?.run !== "npm ci --ignore-scripts --no-audit --no-fund") { + errors.push("trusted E2E planner dependencies must install without lifecycle scripts"); + } + const trustedPlannerIndex = trustedPlannerCheckout + ? generateSteps.indexOf(trustedPlannerCheckout) + : -1; + const trustedSetupIndex = trustedPlannerSetup ? generateSteps.indexOf(trustedPlannerSetup) : -1; + const trustedInstallIndex = trustedPlannerInstall + ? generateSteps.indexOf(trustedPlannerInstall) + : -1; + const generateIndex = generate ? generateSteps.indexOf(generate) : -1; + const candidateCheckoutIndex = candidateCheckout ? generateSteps.indexOf(candidateCheckout) : -1; + if ( + trustedPlannerIndex < 0 || + trustedSetupIndex <= trustedPlannerIndex || + trustedInstallIndex <= trustedSetupIndex || + generateIndex <= trustedInstallIndex || + candidateCheckoutIndex <= generateIndex + ) { + errors.push("trusted E2E planning must finish before candidate checkout and execution"); + } + + const generateEnv = asRecord(generate?.env); + if ( + generateEnv.NEMOCLAW_E2E_CREDENTIALS_ALLOWED !== + "${{ (inputs.checkout_sha == '' || steps.candidate_authorization.outputs.nvidia_owned == 'true') && 'true' || 'false' }}" + ) { + errors.push("matrix generation step must bind NVIDIA-owned candidate authorization"); + } + if (generateEnv.NVIDIA_OWNED !== "${{ steps.candidate_authorization.outputs.nvidia_owned }}") { + errors.push("matrix generation step must bind the authenticated PR repository owner"); + } +} + export function validateE2eWorkflow(workflowValue: unknown): string[] { const workflow = asRecord(workflowValue); const errors: string[] = []; @@ -2307,8 +2377,11 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { if (controllerMatrix?.id !== "controller_matrix") { errors.push("trusted controller matrix step must use id controller_matrix"); } - if (controllerMatrix?.if !== "${{ inputs.checkout_sha != '' }}") { - errors.push("trusted controller matrix step must run only for controller dispatches"); + if ( + controllerMatrix?.if !== + "${{ inputs.checkout_sha != '' && steps.candidate_authorization.outputs.nvidia_owned != 'true' }}" + ) { + errors.push("trusted controller matrix step must run only for external PR dispatches"); } if (controllerMatrix?.shell !== "bash") { errors.push("trusted controller matrix step must use bash"); @@ -2392,16 +2465,24 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { controllerMatrix, `printf 'matrix=%s\\n' "\${matrix}" >> "\${GITHUB_OUTPUT}"`, ); - const generateCheckout = generateSteps.find((step) => - stringValue(step.uses).startsWith("actions/checkout@"), - ); + const generateCheckout = requireStep(errors, generateSteps, "Check out E2E candidate"); if (!generateCheckout) errors.push("generate-matrix job missing checkout step"); + const candidateAuthorization = generateSteps.find( + (step) => stringValue(step.id) === "candidate_authorization", + ); + if ( + controllerMatrix && + candidateAuthorization && + generateSteps.indexOf(controllerMatrix) <= generateSteps.indexOf(candidateAuthorization) + ) { + errors.push("external controller matrix must run after PR ownership authentication"); + } if ( controllerMatrix && generateCheckout && generateSteps.indexOf(controllerMatrix) >= generateSteps.indexOf(generateCheckout) ) { - errors.push("trusted controller matrix step must run before PR checkout"); + errors.push("external controller matrix must run before PR checkout"); } requireFullShaAction(errors, generateCheckout, "generate-matrix checkout"); if (asRecord(generateCheckout?.with)["persist-credentials"] !== false) { @@ -2409,6 +2490,7 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { } validateLargerRunnerRouting(errors, jobs, generateMatrix, generateSteps, generateCheckout); const generate = requireStep(errors, generateSteps, "Generate E2E target matrix"); + validateTrustedE2ePlannerBoundary(errors, generateSteps, generate, generateCheckout); const generateEnv = asRecord(generate?.env); if (generateEnv.CHECKOUT_SHA !== "${{ inputs.checkout_sha }}") { errors.push("matrix generation step must bind controller checkout through CHECKOUT_SHA env"); @@ -2426,10 +2508,14 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { errors.push("matrix generation step must pass targets through TARGETS env"); } validateInferenceModeGeneration(errors, generate, generateEnv); - requireRunContains(errors, generate, "npx tsx tools/e2e/workflow-plan.mts"); + requireRunContains(errors, generate, "npx --no-install tsx tools/e2e/workflow-plan.mts"); requireRunContains(errors, generate, "--ci-output"); requireRunContains(errors, generate, "git diff --name-only --diff-filter=ACMRD"); - requireRunContains(errors, generate, 'if [ -n "${CHECKOUT_SHA}" ]'); + requireRunContains( + errors, + generate, + 'if [ -n "${CHECKOUT_SHA}" ] && [ "${NVIDIA_OWNED}" != "true" ]', + ); requireRunContains(errors, generate, "GITHUB_OUTPUT"); requireRunContains(errors, generate, "expected_controller_matrix="); requireRunContains(errors, generate, "actual_controller_matrix="); diff --git a/tools/e2e/workflow-plan.mts b/tools/e2e/workflow-plan.mts index c094ecdb5d..43670a7aa4 100644 --- a/tools/e2e/workflow-plan.mts +++ b/tools/e2e/workflow-plan.mts @@ -804,6 +804,22 @@ export function withoutCredentialedCatalogueProfiles(plan: E2eWorkflowPlan): E2e }; } +function restrictUnauthorizedCandidatePlan( + plan: E2eWorkflowPlan, + hasPlannerSelectors: boolean, +): E2eWorkflowPlan { + const candidatePlan = withoutCredentialedCatalogueProfiles(plan); + const { coverageMatrix: _coverageMatrix, ...planWithoutCoverage } = candidatePlan; + return withCoverageMatrix( + { + ...planWithoutCoverage, + selectedJobs: hasPlannerSelectors ? plan.selectedJobs : [], + hermesSelected: hasPlannerSelectors && plan.hermesSelected, + }, + readFreeStandingJobsInventory(), + ); +} + export function renderE2eWorkflowPlanSummary( plan: E2eWorkflowPlan, options: { includeCoverageAudit?: boolean } = {}, @@ -901,15 +917,20 @@ export function writeE2eWorkflowPlanCiOutput( controllerMap.retiredSelectorSelected && !hasPlannerSelectors ? emptyE2eWorkflowPlan() : buildE2eWorkflowPlan(plannerSelectors, { changedFiles }); + const candidateRevision = COMMIT_SHA_PATTERN.test(environment.NEMOCLAW_E2E_EXPECTED_SHA ?? ""); + const credentialsAllowed = environment.NEMOCLAW_E2E_CREDENTIALS_ALLOWED === "true"; const plan = validateE2eWorkflowPlan( - COMMIT_SHA_PATTERN.test(environment.NEMOCLAW_E2E_EXPECTED_SHA ?? "") - ? withoutCredentialedCatalogueProfiles(planned) + candidateRevision && !credentialsAllowed + ? restrictUnauthorizedCandidatePlan(planned, hasPlannerSelectors) : planned, ); + const expectedHermes = + candidateRevision && !credentialsAllowed && !hasPlannerSelectors + ? false + : expectedHermesSelection(plannerSelectors, controllerMap.retiredSelectorSelected); if ( !changedFiles && - plan.hermesSelected !== - expectedHermesSelection(plannerSelectors, controllerMap.retiredSelectorSelected) + plan.hermesSelected !== expectedHermes ) { throw new Error("E2E planner changed the trusted Hermes selection"); }