diff --git a/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md b/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md index 6f80642f226..2b3a1346048 100644 --- a/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md +++ b/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md @@ -16,7 +16,8 @@ request does not authorize `Staging Brev Launchable`. ## Route the Request -- For E2E against a pull request revision, read and follow [Manual PR Runs](references/manual-pr.md). +- For E2E against a pull request revision, including failure-triggered comparison with its exact + base, read and follow [Manual PR Runs](references/manual-pr.md). - To dispatch ordinary, focused, staging Launchable, or full E2E on `main`, read and follow [Main Runs](references/main-runs.md) and the Launchable boundary below. - For a release decision inspection, use the section below. Do not load a dispatch reference unless the maintainer requests a new run. diff --git a/.agents/skills/nemoclaw-maintainer-e2e/references/manual-pr.md b/.agents/skills/nemoclaw-maintainer-e2e/references/manual-pr.md index bf56b1c8ad6..ac09cccc696 100644 --- a/.agents/skills/nemoclaw-maintainer-e2e/references/manual-pr.md +++ b/.agents/skills/nemoclaw-maintainer-e2e/references/manual-pr.md @@ -4,8 +4,9 @@ # Manual PR E2E Use this mode when a maintainer requests E2E for a pull request. The trusted workflow stays on -`main` and checks out the latest PR commit. The result is advisory and does not create a required PR -check. +`main` and first checks out the latest PR commit. Replay the same selector against the exact PR base +only after a candidate failure remains unresolved. The result is advisory and does not create a +required PR check. ## Credential Boundary @@ -48,6 +49,7 @@ test "$(jq -r .base.ref <<<"$PR_JSON")" = main HEAD_SHA="$(jq -r .head.sha <<<"$PR_JSON")" BASE_SHA="$(jq -r .base.sha <<<"$PR_JSON")" HEAD_REPOSITORY="$(jq -r .head.repo.full_name <<<"$PR_JSON")" +BASE_REPOSITORY="$(jq -r .base.repo.full_name <<<"$PR_JSON")" HEAD_OWNER="$(jq -r .head.repo.owner.login <<<"$PR_JSON")" HEAD_OWNER_TYPE="$(jq -r .head.repo.owner.type <<<"$PR_JSON")" [[ "$HEAD_SHA" =~ ^[0-9a-f]{40}$ ]] @@ -68,14 +70,15 @@ Jetson and Launchable runs require a branch in `NVIDIA/NemoClaw`. Jetson also re `allow_jetson_dispatch=true` and the reviewed service configuration in [Jetson Dispatch Controller](../../../../test/e2e/docs/jetson-dispatch.md). -Set the requested selectors and flags, or leave them empty: +Set the requested selectors and flags, or leave them empty. Record them once so an exact-base replay +cannot silently change the selected behavior: ```bash E2E_JOBS="${E2E_JOBS:-}" E2E_TARGETS="${E2E_TARGETS:-}" ALLOW_JETSON_DISPATCH="${ALLOW_JETSON_DISPATCH:-false}" INCLUDE_STAGING_BREV_LAUNCHABLE="${INCLUDE_STAGING_BREV_LAUNCHABLE:-false}" -CORRELATION_ID="$(python3 -c 'import uuid; print(uuid.uuid4())')" +HEAD_CORRELATION_ID="$(python3 -c 'import uuid; print(uuid.uuid4())')" gh workflow run .github/workflows/e2e.yaml \ --repo NVIDIA/NemoClaw \ --ref main \ @@ -90,7 +93,7 @@ gh workflow run .github/workflows/e2e.yaml \ -f "checkout_repository=${HEAD_REPOSITORY}" \ -f "base_sha=${BASE_SHA}" \ -f "workflow_sha=${WORKFLOW_SHA}" \ - -f "correlation_id=${CORRELATION_ID}" + -f "correlation_id=${HEAD_CORRELATION_ID}" ``` GitHub's permission to dispatch the workflow authorizes the actor. The workflow does not repeat that @@ -99,7 +102,7 @@ repository-role check. The trusted pre-checkout step validates: - the open PR; - the target repository and branch; - the source repository and owner; -- the latest PR commit SHA; +- the selected latest PR commit or exact PR base SHA; - the base SHA; - the workflow SHA; and - whether the source can receive the selected jobs and credentials. @@ -108,11 +111,11 @@ It then records and uploads the immutable `nemoclaw-e2e-dispatch-v2` receipt bef execution. The matrix planner and its dependencies come from the trusted workflow commit. A second validation after checkout rejects changed PR identity or ownership. -## Find and Verify the Run +## Find and Verify the Head Run ```bash set -euo pipefail -RUN_TITLE="E2E PR #${PR_NUMBER} (${CORRELATION_ID})" +RUN_TITLE="E2E PR #${PR_NUMBER} (${HEAD_CORRELATION_ID})" MATCHES='[]' for POLL_INDEX in $(seq 1 30); do RUNS="$(gh run list --repo NVIDIA/NemoClaw --workflow e2e.yaml \ @@ -126,12 +129,15 @@ for POLL_INDEX in $(seq 1 30); do done test "$(jq 'length' <<<"$MATCHES")" -eq 1 RUN_ID="$(jq -r '.[0].databaseId' <<<"$MATCHES")" -gh run watch "$RUN_ID" --repo NVIDIA/NemoClaw --exit-status +gh run watch "$RUN_ID" --repo NVIDIA/NemoClaw --exit-status || true RUN_JSON="$(gh api "repos/NVIDIA/NemoClaw/actions/runs/${RUN_ID}")" jq -e --arg sha "$WORKFLOW_SHA" ' .run_attempt >= 1 and .head_sha == $sha and - .status == "completed" and .conclusion == "success" + .status == "completed" and (.conclusion | type == "string") ' <<<"$RUN_JSON" >/dev/null +HEAD_RUN_ID="$RUN_ID" +HEAD_RUN_URL="$(jq -r .html_url <<<"$RUN_JSON")" +HEAD_CONCLUSION="$(jq -r .conclusion <<<"$RUN_JSON")" CURRENT_PR="$(gh api "repos/NVIDIA/NemoClaw/pulls/${PR_NUMBER}")" test "$(jq -r .state <<<"$CURRENT_PR")" = open test "$(jq -r .head.sha <<<"$CURRENT_PR")" = "$HEAD_SHA" @@ -144,17 +150,85 @@ test "$(jq -r .head.repo.owner.type <<<"$CURRENT_PR")" = "$HEAD_OWNER_TYPE" If the run is not visible after bounded polling, do not dispatch again. Inspect GitHub Actions for the correlation ID. Clean up resources from any matching run. +If the head run succeeds, report it and stop. If it fails, inspect the failed job with the read-only +`nemoclaw-maintainer-classify-ci-failure` skill when that skill is available. Pass the workflow and +job evidence the skill requests. Do not recreate its signature list or classification logic here. +If it classifies a known infrastructure failure, report `infrastructure` and stop without a base +replay. If the skill is unavailable or returns `unclassified`, the candidate failure remains +unresolved and may proceed to the exact-base replay below. + +## Replay an Unresolved Failure Against the Exact Base + +Before dispatch, prove that the PR still has the head and base captured for the failed head run. +Use the same trusted workflow SHA, eligible jobs and targets, and inference mode. The selected source +changes from the PR head repository and SHA to the PR base repository and SHA. + +Do not use an exact-base replay for Jetson or DGX Spark. Those selectors need dedicated hardware and +cannot provide a trustworthy base comparison with their opt-in flags disabled. + +```bash +set -euo pipefail +CURRENT_PR="$(gh api "repos/NVIDIA/NemoClaw/pulls/${PR_NUMBER}")" +test "$(jq -r .state <<<"$CURRENT_PR")" = open +test "$(jq -r .head.sha <<<"$CURRENT_PR")" = "$HEAD_SHA" +test "$(jq -r .base.sha <<<"$CURRENT_PR")" = "$BASE_SHA" +test "$(jq -r .head.repo.full_name <<<"$CURRENT_PR")" = "$HEAD_REPOSITORY" +test "$(jq -r .base.repo.full_name <<<"$CURRENT_PR")" = "$BASE_REPOSITORY" +test "$BASE_REPOSITORY" = NVIDIA/NemoClaw + +BASE_CORRELATION_ID="$(python3 -c 'import uuid; print(uuid.uuid4())')" +gh workflow run .github/workflows/e2e.yaml \ + --repo NVIDIA/NemoClaw \ + --ref main \ + -f "targets=${E2E_TARGETS}" \ + -f "jobs=${E2E_JOBS}" \ + -f inference_mode=mock \ + -f "include_staging_brev_launchable=${INCLUDE_STAGING_BREV_LAUNCHABLE}" \ + -f allow_jetson_dispatch=false \ + -f allow_dgx_spark_runner_queue=false \ + -f "pr_number=${PR_NUMBER}" \ + -f "checkout_sha=${BASE_SHA}" \ + -f "checkout_repository=${BASE_REPOSITORY}" \ + -f "base_sha=${BASE_SHA}" \ + -f "workflow_sha=${WORKFLOW_SHA}" \ + -f "correlation_id=${BASE_CORRELATION_ID}" +``` + +Find the base run with the same bounded lookup used above and this exact title: + +```bash +RUN_TITLE="E2E PR #${PR_NUMBER} (${BASE_CORRELATION_ID})" +``` + +Verify that its Actions run `head_sha` is the same `WORKFLOW_SHA`. Record its run ID, URL, and +conclusion as `BASE_RUN_ID`, `BASE_RUN_URL`, and `BASE_CONCLUSION`. Re-read the PR and require that +its state, head SHA, base SHA, head repository, and base repository still match the captured values. +Classify a failed base job with the same read-only CI classification skill when available. + +Report exactly one outcome: + +- `candidate regression`: the unresolved head run failed and the exact-base run passed. +- `base already broken`: both runs show the same reproducible product failure. +- `infrastructure`: the read-only classifier identifies a known infrastructure failure in either + run. +- `unresolved`: the base run fails differently, a run or identity is incomplete, or the available + evidence does not support one of the outcomes above. + +The report must include both workflow URLs, `HEAD_SHA`, `BASE_SHA`, `WORKFLOW_SHA`, the identical +selectors, and the classifier result. Do not automatically dispatch the base run, retry either run, +or publish a custom commit status. + Return: - the PR number; - the source repository; - the source repository owner; -- the latest PR commit SHA; +- the tested head and, when replayed, base commit SHAs; - the base SHA; - the workflow SHA; -- the correlation ID; +- the head and, when replayed, base correlation IDs; - the selectors; -- the workflow URL; and -- the result. +- the head and, when replayed, base workflow URLs; and +- the result or comparison outcome. A changed source repository, owner, latest PR commit SHA, or base SHA invalidates the run claim. diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index c95b1d7599c..6f65f624ad2 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# Manual PR runs bind the candidate SHA independently from reusable image publication. +# Manual PR runs bind the selected head or base SHA independently from reusable image publication. name: E2E / Main and Manual Suite run-name: "${{ inputs.checkout_sha != '' && format('E2E PR #{0} ({1})', inputs.pr_number, inputs.correlation_id) || inputs.correlation_id != '' && inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '' && !inputs.allow_jetson_dispatch && !inputs.allow_dgx_spark_runner_queue && format('E2E full {0} ({1})', github.ref_name, inputs.correlation_id) || inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '' && !inputs.allow_jetson_dispatch && !inputs.allow_dgx_spark_runner_queue && format('E2E full {0}', github.ref_name) || inputs.correlation_id != '' && format('E2E {0} ({1})', github.ref_name, inputs.correlation_id) || format('E2E {0}', github.ref_name) }}" @@ -68,12 +68,12 @@ on: default: false type: boolean checkout_sha: - description: Optional lowercase 40-character latest PR commit SHA for manual E2E. + description: Optional lowercase 40-character PR head or exact PR base SHA for manual E2E. required: false default: "" type: string checkout_repository: - description: Optional PR source repository for manual E2E. + description: Optional repository containing the selected PR head or base SHA for manual E2E. required: false default: "" type: string @@ -395,6 +395,8 @@ jobs: name: Authenticate manual PR dispatch if: ${{ inputs.pr_number != '' || inputs.checkout_sha != '' || inputs.checkout_repository != '' || inputs.base_sha != '' || inputs.workflow_sha != '' }} env: + ALLOW_DGX_SPARK_RUNNER_QUEUE: ${{ inputs.allow_dgx_spark_runner_queue && 'true' || 'false' }} + ALLOW_JETSON_DISPATCH: ${{ inputs.allow_jetson_dispatch && 'true' || 'false' }} BASE_SHA: ${{ inputs.base_sha }} CHECKOUT_REPOSITORY: ${{ inputs.checkout_repository }} CHECKOUT_SHA: ${{ inputs.checkout_sha }} @@ -403,6 +405,7 @@ jobs: INCLUDE_LAUNCHABLE: ${{ inputs.include_staging_brev_launchable && 'true' || 'false' }} JOBS: ${{ inputs.jobs }} PR_NUMBER: ${{ inputs.pr_number }} + TARGETS: ${{ inputs.targets }} WORKFLOW_EVENT: ${{ github.event_name }} WORKFLOW_REF: ${{ github.ref }} WORKFLOW_SHA: ${{ github.workflow_sha }} @@ -427,9 +430,19 @@ jobs: "https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}")" [[ "$(jq -r '.state' <<< "$pull_json")" == "open" ]] || { echo "::error::pull request must be open" >&2; exit 1; } [[ "$(jq -r '.base.repo.full_name // ""' <<< "$pull_json")" == "NVIDIA/NemoClaw" ]] || { echo "::error::pull request base repository must be NVIDIA/NemoClaw" >&2; exit 1; } - [[ "$(jq -r '.head.repo.full_name // ""' <<< "$pull_json")" == "$CHECKOUT_REPOSITORY" ]] || { echo "::error::checkout_repository must match the PR source repository" >&2; exit 1; } - [[ "$(jq -r '.head.sha' <<< "$pull_json")" == "$CHECKOUT_SHA" ]] || { echo "::error::checkout_sha must match the latest PR commit SHA" >&2; exit 1; } + [[ "$(jq -r '.base.ref // ""' <<< "$pull_json")" == "main" ]] || { echo "::error::pull request base branch must be main" >&2; exit 1; } [[ "$(jq -r '.base.sha' <<< "$pull_json")" == "$BASE_SHA" ]] || { echo "::error::base_sha must match the PR base SHA" >&2; exit 1; } + if [[ "$CHECKOUT_SHA" == "$BASE_SHA" ]]; then + [[ "$ALLOW_JETSON_DISPATCH" != "true" && "$ALLOW_DGX_SPARK_RUNNER_QUEUE" != "true" ]] || { echo "::error::exact-base E2E cannot launch dedicated hardware dispatches" >&2; exit 1; } + [[ ",${TARGETS}," != *",jetson-nvmap-gpu,"* && + ",${JOBS}," != *",jetson-nvmap-gpu,"* && + ",${TARGETS}," != *",llama-cpp-dgx-spark-qualification,"* && + ",${JOBS}," != *",llama-cpp-dgx-spark-qualification,"* ]] || { echo "::error::exact-base E2E cannot select dedicated hardware jobs" >&2; exit 1; } + [[ "$(jq -r '.base.repo.full_name // ""' <<< "$pull_json")" == "$CHECKOUT_REPOSITORY" ]] || { echo "::error::checkout_repository must match the PR base repository" >&2; exit 1; } + else + [[ "$(jq -r '.head.repo.full_name // ""' <<< "$pull_json")" == "$CHECKOUT_REPOSITORY" ]] || { echo "::error::checkout_repository must match the PR source repository" >&2; exit 1; } + [[ "$(jq -r '.head.sha' <<< "$pull_json")" == "$CHECKOUT_SHA" ]] || { echo "::error::checkout_sha must match the latest PR commit SHA" >&2; exit 1; } + fi nvidia_owned=false if [[ "$(jq -r '.head.repo.owner.login // ""' <<< "$pull_json")" == "NVIDIA" && "$(jq -r '.head.repo.owner.type // ""' <<< "$pull_json")" == "Organization" ]]; then @@ -764,9 +777,14 @@ jobs: "https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}")" [[ "$(jq -r '.state' <<< "$pull_json")" == "open" ]] || { echo "::error::pull request must still be open" >&2; exit 1; } [[ "$(jq -r '.base.repo.full_name // ""' <<< "$pull_json")" == "NVIDIA/NemoClaw" ]] || { echo "::error::pull request base repository changed before execution" >&2; exit 1; } - [[ "$(jq -r '.head.repo.full_name // ""' <<< "$pull_json")" == "$CHECKOUT_REPOSITORY" ]] || { echo "::error::checkout_repository changed before execution" >&2; exit 1; } - [[ "$(jq -r '.head.sha' <<< "$pull_json")" == "$CHECKOUT_SHA" ]] || { echo "::error::checkout_sha changed before execution" >&2; exit 1; } + [[ "$(jq -r '.base.ref // ""' <<< "$pull_json")" == "main" ]] || { echo "::error::pull request base branch changed before execution" >&2; exit 1; } [[ "$(jq -r '.base.sha' <<< "$pull_json")" == "$BASE_SHA" ]] || { echo "::error::base_sha changed before execution" >&2; exit 1; } + if [[ "$CHECKOUT_SHA" == "$BASE_SHA" ]]; then + [[ "$(jq -r '.base.repo.full_name // ""' <<< "$pull_json")" == "$CHECKOUT_REPOSITORY" ]] || { echo "::error::checkout_repository changed before base execution" >&2; exit 1; } + else + [[ "$(jq -r '.head.repo.full_name // ""' <<< "$pull_json")" == "$CHECKOUT_REPOSITORY" ]] || { echo "::error::checkout_repository changed before head execution" >&2; exit 1; } + [[ "$(jq -r '.head.sha' <<< "$pull_json")" == "$CHECKOUT_SHA" ]] || { echo "::error::checkout_sha changed before head execution" >&2; exit 1; } + fi if [[ "$NVIDIA_OWNED" == "true" ]]; then [[ "$(jq -r '.head.repo.owner.login // ""' <<< "$pull_json")" == "NVIDIA" && "$(jq -r '.head.repo.owner.type // ""' <<< "$pull_json")" == "Organization" ]] || { diff --git a/test/e2e/README.md b/test/e2e/README.md index 07f5db05d89..130330a3e50 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -51,7 +51,9 @@ It builds the CLI from the trusted workflow checkout and never executes or resto #### Artifact Identity -For a pull request (PR) run, `checkout_sha` identifies the candidate source commit. +For a pull request (PR) run, `checkout_sha` identifies the selected head or exact base source +commit. A base replay sets `checkout_sha` equal to `base_sha` and keeps the same trusted workflow +SHA and selectors as the failed head run. The trusted workflow runs from `github.workflow_sha`. A push or manual run uses `github.sha` when `checkout_sha` is empty. @@ -1452,7 +1454,7 @@ Dispatch a new run after recovery. If a case fails, use the GitHub Actions job log. Inspect a case artifact only when its upload step completed. -For a manual PR run, provide these inputs: +For the initial manual PR run, provide these inputs: - The current PR number. - The lowercase 40-character SHA of the latest PR commit. @@ -1460,6 +1462,13 @@ For a manual PR run, provide these inputs: - The lowercase 40-character PR base SHA. - The SHA of the trusted workflow commit on `main`. +If the head run fails, use the read-only CI failure classifier when it is available. Stop without a +base replay when it identifies known infrastructure failure. Only an unresolved candidate failure +may be replayed with the same selectors against the exact PR base: set `checkout_sha` to the +recorded base SHA and `checkout_repository` to `NVIDIA/NemoClaw`, while leaving `base_sha` and +`workflow_sha` unchanged. The workflow run is the evidence; do not publish a separate commit status +or automatically replay the base. + For the default NVIDIA-owned PR revision selection, leave `jobs` and `targets` empty and keep `include_staging_brev_launchable=false`. Keep `allow_jetson_dispatch=false` and `allow_dgx_spark_runner_queue=false` for the default PR revision selection. If `allow_dgx_spark_runner_queue=true`, GitHub can pause the qualification job for the `approve-dgx-spark-image-qualification` environment. @@ -1474,12 +1483,15 @@ For this producer run, the executing workflow SHA, `workflow_sha` input, and PR Confirm that the PR comes from `NVIDIA/NemoClaw`, the required ephemeral runner variables are configured, and the workflow has not been rerun. A trusted `main` workflow pre-checkout step validates the open PR and records whether its source repository has API-confirmed `NVIDIA` organization ownership. That ownership authorizes the full ordinary plan and credential profiles; external sources retain the bounded controller plan. -A second validation after checkout rejects a changed candidate commit, base commit, PR source repository, or NVIDIA ownership before preparation. +A second validation after checkout rejects a changed selected commit, base commit, repository, or +NVIDIA ownership before preparation. Candidate runs cannot publish release qualification. The Actions run is advisory for the pull request and is not a required merge context. -Treat it as passing evidence only when the `E2E` workflow concludes with `success` for the recorded PR number, PR source repository, candidate commit SHA, base commit SHA, and executing workflow SHA. -A changed PR source repository, candidate commit SHA, or base commit SHA invalidates the evidence and requires a new manual run. +Treat a head or base run as passing evidence only when the `E2E` workflow concludes with `success` +for the recorded PR number, selected repository, selected commit SHA, base commit SHA, and executing +workflow SHA. A changed PR source repository, head commit SHA, or base commit SHA invalidates a +head-to-base comparison. The platform-evidence workflow runs on configured pushes to `main` and supports manual dispatch for branch diagnosis. The experimental portable-profile workflow can run for pull requests, matching `main` pushes, and manual dispatch. diff --git a/test/e2e/support/e2e-operations-workflow-boundary.test.ts b/test/e2e/support/e2e-operations-workflow-boundary.test.ts index 7d6a388e4e6..f9dc2fe1d30 100644 --- a/test/e2e/support/e2e-operations-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-operations-workflow-boundary.test.ts @@ -468,6 +468,18 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; }); it.each([ + [ + "an exact PR base revision", + "NVIDIA/NemoClaw", + "b", + "b", + "c", + "refs/heads/main", + "", + "", + 0, + "", + ], [ "an invalid source repository name", "invalid-repository", @@ -475,6 +487,8 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; "b", "c", "refs/heads/main", + "", + "", 1, "::error::checkout_repository must be an owner/repository name\n", ], @@ -485,6 +499,8 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; "d", "c", "refs/heads/main", + "", + "", 1, "::error::base_sha must match the PR base SHA\n", ], @@ -495,6 +511,8 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; "b", "d", "refs/heads/main", + "", + "", 1, "::error::workflow_sha must match the trusted main workflow SHA\n", ], @@ -505,9 +523,33 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; "b", "c", "refs/heads/pr-controlled-workflow", + "", + "", 0, "", ], + ...( + [ + ["Jetson", "jetson-nvmap-gpu"], + ["DGX Spark", "llama-cpp-dgx-spark-qualification"], + ] as const + ).flatMap(([name, selector]) => + (["job", "target"] as const).map( + (channel) => + [ + `an exact-base ${name} ${channel}`, + "NVIDIA/NemoClaw", + "b", + "b", + "c", + "refs/heads/main", + channel === "job" ? selector : "", + channel === "target" ? selector : "", + 1, + "::error::exact-base E2E cannot select dedicated hardware jobs\n", + ] as const, + ), + ), ] as const)( "handles manual PR authentication for %s", ( @@ -517,6 +559,8 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; requestedBaseCharacter, expectedWorkflowCharacter, workflowRef, + jobs, + targets, expectedStatus, expectedStderr, ) => { @@ -539,6 +583,8 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; encoding: "utf8", env: { ...process.env, + ALLOW_DGX_SPARK_RUNNER_QUEUE: "false", + ALLOW_JETSON_DISPATCH: "false", BASE_SHA: requestedBaseCharacter.repeat(40), CHECKOUT_REPOSITORY: requestedRepository, CHECKOUT_SHA: requestedHeadCharacter.repeat(40), @@ -547,8 +593,9 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; GITHUB_REPOSITORY: "NVIDIA/NemoClaw", GITHUB_TOKEN: "token", INCLUDE_LAUNCHABLE: "false", - JOBS: "", + JOBS: jobs, PR_NUMBER: "42", + TARGETS: targets, WORKFLOW_EVENT: "workflow_dispatch", WORKFLOW_REF: workflowRef, WORKFLOW_SHA: workflowSha, @@ -561,6 +608,40 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; }, ); + it("revalidates an exact PR base after checkout", () => { + const workflow = readE2eOperationsWorkflow(); + const validation = workflow.jobs["generate-matrix"].steps!.find( + (step) => step.name === "Validate manual PR checkout", + )!; + const headSha = "a".repeat(40); + const baseSha = "b".repeat(40); + const prefix = [ + "git() { printf '%s\\n' \"$CHECKOUT_SHA\"; }", + "curl() {", + ` printf '%s' '{"state":"open","head":{"repo":{"full_name":"NVIDIA/NemoClaw","owner":{"login":"NVIDIA","type":"Organization"}},"sha":"${headSha}"},"base":{"repo":{"full_name":"NVIDIA/NemoClaw"},"ref":"main","sha":"${baseSha}"}}'`, + "}", + ].join("\n"); + const result = spawnSync( + "bash", + ["--noprofile", "--norc", "-e", "-o", "pipefail", "-c", `${prefix}\n${validation.run}`], + { + encoding: "utf8", + env: { + ...process.env, + BASE_SHA: baseSha, + CHECKOUT_REPOSITORY: "NVIDIA/NemoClaw", + CHECKOUT_SHA: baseSha, + GITHUB_REPOSITORY: "NVIDIA/NemoClaw", + GITHUB_TOKEN: "token", + NVIDIA_OWNED: "true", + PR_NUMBER: "42", + }, + }, + ); + + expect(result.status, result.stderr).toBe(0); + }); + it.each([ ["NVIDIA inclusion flag", "NVIDIA/NemoClaw", "NVIDIA", "Organization", "true", "", 0, ""], [ diff --git a/test/e2e/support/pr-managed-image-publication.test.ts b/test/e2e/support/pr-managed-image-publication.test.ts index 4244020a702..7121397cf4a 100644 --- a/test/e2e/support/pr-managed-image-publication.test.ts +++ b/test/e2e/support/pr-managed-image-publication.test.ts @@ -254,6 +254,17 @@ describe("exact PR managed-image publication", () => { expect(fs.existsSync(input.outputPath)).toBe(false); }); + it("uses the exact PR base as the base-history cohort", async () => { + const input = { ...resolverInput(), candidateSha: BASE_SHA }; + const download = vi.fn(downloadContract); + + await expect( + resolvePrManagedImageCatalog(input, candidateRequest({ imageChanged: true }), download), + ).resolves.toBe("base-cohort"); + expect(download).not.toHaveBeenCalled(); + expect(fs.existsSync(input.outputPath)).toBe(false); + }); + it("writes one exact candidate catalog after an immutable image-input change", async () => { const input = resolverInput(); diff --git a/test/e2e/support/workflow-plan.test.ts b/test/e2e/support/workflow-plan.test.ts index 874d7b78dac..edddbb9ac6a 100644 --- a/test/e2e/support/workflow-plan.test.ts +++ b/test/e2e/support/workflow-plan.test.ts @@ -265,7 +265,7 @@ describe("E2E workflow plan", () => { expect(selectedWorkflowJobs(plan)).toEqual(["catalogue-nvidia-api"]); }); - it("emits the required workflow fields for migrated targets", () => { + it("emits required fields and catalogue workflow jobs for migrated targets", () => { const plan = buildE2eWorkflowPlan({ jobs: "gateway-guard-recovery,hermes-slack,network-policy,openclaw-inference-switch,openclaw-tui-chat-correlation,sandbox-operations", }); @@ -305,11 +305,16 @@ describe("E2E workflow plan", () => { display_name: "Inference: OpenClaw switches providers and remains responsive", }), ); - const retainedJobs = readFreeStandingJobsInventory().allowedJobs; - - expect(retainedJobs).not.toEqual( - expect.arrayContaining(["hermes-slack", "openclaw-inference-switch", "sandbox-operations"]), + expect(selectedWorkflowJobs(plan)).toEqual([ + "catalogue-nvidia-inference", + "catalogue-standard", + ]); + const migratedTargetIds = ["hermes-slack", "openclaw-inference-switch", "sandbox-operations"]; + const retainedMigratedJobs = readFreeStandingJobsInventory().allowedJobs.filter((id) => + migratedTargetIds.includes(id), ); + + expect(retainedMigratedJobs).toEqual([]); }); it.each([ diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index d6f3ceb6ff4..fc78d641d82 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -380,6 +380,8 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow errors.push("Manual PR authentication must run when any candidate identity input is present"); } const authEnvironment = { + ALLOW_DGX_SPARK_RUNNER_QUEUE: "${{ inputs.allow_dgx_spark_runner_queue && 'true' || 'false' }}", + ALLOW_JETSON_DISPATCH: "${{ inputs.allow_jetson_dispatch && 'true' || 'false' }}", BASE_SHA: "${{ inputs.base_sha }}", CHECKOUT_REPOSITORY: "${{ inputs.checkout_repository }}", CHECKOUT_SHA: "${{ inputs.checkout_sha }}", @@ -387,6 +389,7 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow INCLUDE_LAUNCHABLE: "${{ inputs.include_staging_brev_launchable && 'true' || 'false' }}", JOBS: "${{ inputs.jobs }}", PR_NUMBER: "${{ inputs.pr_number }}", + TARGETS: "${{ inputs.targets }}", WORKFLOW_EVENT: "${{ github.event_name }}", WORKFLOW_REF: "${{ github.ref }}", WORKFLOW_SHA: "${{ github.workflow_sha }}", @@ -409,6 +412,16 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow '"$EXPECTED_WORKFLOW_SHA" == "$WORKFLOW_SHA"', "https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}", `[[ "$(jq -r '.base.repo.full_name // ""' <<< "$pull_json")" == "NVIDIA/NemoClaw" ]]`, + `[[ "$(jq -r '.base.ref // ""' <<< "$pull_json")" == "main" ]]`, + 'if [[ "$CHECKOUT_SHA" == "$BASE_SHA" ]]', + '"$ALLOW_JETSON_DISPATCH" != "true"', + '"$ALLOW_DGX_SPARK_RUNNER_QUEUE" != "true"', + '",${TARGETS}," != *",jetson-nvmap-gpu,"*', + '",${JOBS}," != *",jetson-nvmap-gpu,"*', + '",${TARGETS}," != *",llama-cpp-dgx-spark-qualification,"*', + '",${JOBS}," != *",llama-cpp-dgx-spark-qualification,"*', + "exact-base E2E cannot select dedicated hardware jobs", + `[[ "$(jq -r '.base.repo.full_name // ""' <<< "$pull_json")" == "$CHECKOUT_REPOSITORY" ]]`, `[[ "$(jq -r '.head.repo.full_name // ""' <<< "$pull_json")" == "$CHECKOUT_REPOSITORY" ]]`, `[[ "$(jq -r '.head.sha' <<< "$pull_json")" == "$CHECKOUT_SHA" ]]`, `[[ "$(jq -r '.base.sha' <<< "$pull_json")" == "$BASE_SHA" ]]`, @@ -479,9 +492,12 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow "https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}", "pull request must still be open", "pull request base repository changed before execution", - "checkout_repository changed before execution", - "checkout_sha changed before execution", + "pull request base branch changed before execution", "base_sha changed before execution", + 'if [[ "$CHECKOUT_SHA" == "$BASE_SHA" ]]', + `[[ "$(jq -r '.base.repo.full_name // ""' <<< "$pull_json")" == "$CHECKOUT_REPOSITORY" ]]`, + `[[ "$(jq -r '.head.repo.full_name // ""' <<< "$pull_json")" == "$CHECKOUT_REPOSITORY" ]]`, + `[[ "$(jq -r '.head.sha' <<< "$pull_json")" == "$CHECKOUT_SHA" ]]`, '"$NVIDIA_OWNED" == "true"', '[[ -n "$GITHUB_TOKEN" ]]', 'auth_args=(--header "Authorization: Bearer ${GITHUB_TOKEN}")', diff --git a/tools/e2e/pr-managed-image-publication.mts b/tools/e2e/pr-managed-image-publication.mts index df4f8cf00f0..40fc111a275 100644 --- a/tools/e2e/pr-managed-image-publication.mts +++ b/tools/e2e/pr-managed-image-publication.mts @@ -235,16 +235,19 @@ function validatePr( expected.baseSha, "pull request base commit", ); - exactString( - record(pull.head, "pull request source").sha, - expected.candidateSha, - "pull request source commit", - ); exactString( record(record(pull.base, "pull request base").repo, "pull request base repository").full_name, REPOSITORY, "pull request base repository", ); + if (expected.candidateSha === expected.baseSha && expected.candidateRepository === REPOSITORY) { + return; + } + exactString( + record(pull.head, "pull request source").sha, + expected.candidateSha, + "pull request source commit", + ); exactString( record(record(pull.head, "pull request source").repo, "pull request source repository") .full_name,