diff --git a/.agents/skills/nemoclaw-maintainer-cut-release-tag/SKILL.md b/.agents/skills/nemoclaw-maintainer-cut-release-tag/SKILL.md index 56962b1681c..ea1ada0c3e0 100644 --- a/.agents/skills/nemoclaw-maintainer-cut-release-tag/SKILL.md +++ b/.agents/skills/nemoclaw-maintainer-cut-release-tag/SKILL.md @@ -29,6 +29,9 @@ gh run view --repo NVIDIA/NemoClaw --log Extract the exact `https://github.com/brevdev/nemoclaw-image/actions/runs/` URL printed by the source run, give that link to the maintainer immediately, and tell them to follow it to terminal success. Treat dispatch acceptance as an intermediate state, not proof of production image promotion: the downstream run must succeed and its summary must show successful runtime E2E validation and promotion of the `nemoclaw-brev-cpu` image family. +A selected staging image remains an evidence handoff until the downstream workflow accepts immutable +image identity. The current downstream workflow accepts only the semver tag and rebuilds the production +image. Do not report the selected staging image as promoted. A rejected dispatch fails the trigger run but does not move or roll back `lkg`. Deleting `lkg` does not dispatch an image build. The downstream scheduled reconciliation remains available if the event-driven dispatch fails or is delayed. @@ -40,7 +43,10 @@ The downstream scheduled reconciliation remains available if the event-driven di - Treat the dated MDX entry as the canonical release history. A conventional Release Notes page or post-tag Announcement draft cannot replace it. - If `origin/main` changes after plan generation, regenerate the plan before cutting the tag. - Before asking for release confirmation, satisfy the canonical [pre-tag E2E evidence policy](../nemoclaw-maintainer-policies/references/release-train.md#pre-tag-e2e-evidence) for that commit. -- Run full mode unless one existing full run for the candidate SHA contains complete workflow E2E and `Exact staging Brev Launchable` evidence. +- Run ordinary mode for the exact-candidate release ledger when complete default E2E evidence does + not already exist. Qualify the release image separately. +- Show the newest qualified GCP image candidate before asking whether to select it for a future + exact-image handoff or run the selective Launchable test again. - Ask the maintainer to paste the confirmation phrase from the plan before cutting the tag. - Push only the semver tag (`vX.Y.Z`) from the agent-controlled step. - Never push `latest` or `lkg` from this skill. @@ -108,6 +114,19 @@ The script writes a plan outside the checkout root, for example: ../nemoclaw-release-v0.0.58/plan.json ``` +Keep the plan and evidence paths explicit so evidence returned by `nemoclaw-maintainer-e2e` +survives that skill's invocation: + +```bash +PLAN_JSON="" +RELEASE_DIR="$(dirname "$PLAN_JSON")" +export EVIDENCE_DIR="$RELEASE_DIR/e2e-evidence" +install -d -m 0700 "$EVIDENCE_DIR" +``` + +Pass this exported absolute path into every `nemoclaw-maintainer-e2e` evidence collection and +validation invocation. Do not let that skill replace it with a temporary directory. + ### Step 2: Show Plan, E2E Evidence, and Ask for Confirmation Read the generated `plan.json` and show the maintainer: @@ -135,32 +154,36 @@ npm run release:e2e-evidence -- \ ``` The preflight derives every required execution from one empty-selector dispatch. -The full run includes every default-selected workflow E2E plus `Exact staging Brev Launchable`. +The ordinary run includes every default-selected workflow E2E and does not build a Launchable image. Accepted release evidence requires `allowJetsonDispatch: false`, `allowJetsonRunnerQueue: false`, and `allowDgxSparkRunnerQueue: false`. The required denominator excludes `jetson-nvmap-gpu`, `llama-cpp-dgx-spark-plan`, and `llama-cpp-dgx-spark-qualification`. Each job that declares `RELEASE_E2E_ACTIVATION_PATH` requires that path at the candidate SHA. A missing activation path is a preflight failure. -Check whether one existing full run for the candidate SHA contains complete evidence. If it does not, load `nemoclaw-maintainer-e2e` and dispatch one full run. Do not combine evidence from different workflow run IDs. Do not substitute a selective run for full-run evidence. +Check whether one existing ordinary run for the candidate SHA contains complete default E2E +evidence. If it does not, load `nemoclaw-maintainer-e2e` and dispatch one ordinary run. Do not combine +evidence from different workflow run IDs. Image qualification remains a separate decision below. Monitor the dispatched correlation ID with one bounded status query. -Before accepting full-mode exact Brev evidence, require: +Before accepting exact-candidate release-ledger evidence, require: - the workflow `head_sha` to equal the plan candidate SHA; -- the trusted dispatch receipt to prove empty selectors, `include_staging_brev_launchable=true`, `allowJetsonRunnerQueue: false`, and `allowDgxSparkRunnerQueue: false`; a v1 receipt may omit `allowJetsonDispatch` but must set it to `false` when present, while a v2 receipt must include `allowJetsonDispatch: false`; +- a v2 trusted dispatch receipt with `prNumber: null`, repository and candidate repository both + `NVIDIA/NemoClaw`, and base, workflow, and candidate SHAs all equal to the plan candidate SHA; +- the receipt to prove empty selectors, `include_staging_brev_launchable=false`, + `allowJetsonDispatch: false`, `allowJetsonRunnerQueue: false`, and + `allowDgxSparkRunnerQueue: false`; - the workflow conclusion to be `success`; -- the `Exact staging Brev Launchable` job conclusion to be `success`; -- the job URL and selected successful Launchable job attempt; -- Launchable E2E identity for the same SHA; and -- cleanup evidence that reports the qualified workspace as `ABSENT`. +- every default-selected workflow execution to have successful evidence or an itemized exception. Treat a skipped job as missing evidence even when the workflow concludes `success`. -If the plan candidate SHA changes, discard the run and Launchable E2E evidence. -Run full mode again for the new candidate SHA. +If the plan candidate SHA changes, discard the run and release ledger. +Run ordinary mode again for the new candidate SHA. No release-note-only delta exception is currently defined. -For the accepted full run, reuse `run-$RUN_ID.json` and `jobs-$RUN_ID.json` returned by `nemoclaw-maintainer-e2e`, and collect the workflow-produced dispatch receipt. +For the accepted ordinary run, reuse `run-$RUN_ID.json` and `jobs-$RUN_ID.json` returned by +`nemoclaw-maintainer-e2e`, and collect the workflow-produced dispatch receipt. If those files were not returned, collect them once: ```bash @@ -183,7 +206,8 @@ gh run download "$RUN_ID" \ Use the latest existing receipt artifact, not the run's latest attempt number. A partial rerun can leave `generate-matrix` successful and therefore reuse its earlier receipt; the ledger permits that earlier receipt only when it binds the same run and its attempt does not exceed the run's latest attempt. -Successful workflow E2E and `Exact staging Brev Launchable` evidence may accumulate across rerun attempts of that workflow run. Evidence from another workflow run does not satisfy the ledger. +Successful default E2E evidence may accumulate across rerun attempts of that workflow run. Evidence +from another workflow run does not satisfy the ledger. Create `manifest.json` in the private evidence directory: @@ -203,21 +227,95 @@ Create `manifest.json` in the private evidence directory: Do not type empty-selector claims or selector lists into the manifest. The helper derives them from the workflow-produced receipt and rejects a receipt whose selector fields disagree with its empty-selector flag. Build the ledger with `npm run release:e2e-evidence -- --manifest "$EVIDENCE_DIR/manifest.json"`. The helper derives the denominator from the workflow, preserves matrix rows as separate semantic identifiers, binds every run and its actual dispatch inputs to the candidate SHA, and keeps an earlier successful attempt when a later attempt fails. -The manifest and helper cover the workflow-derived test execution ledger only. They do not replace exact Brev Launchable E2E acceptance: keep the raw `dispatch.json`, `launchable-e2e.json`, and `cleanup.json` validation in `nemoclaw-maintainer-e2e`, and carry its validated return beside this ledger or record the required Launchable E2E exception. +The manifest and helper cover the exact-candidate release ledger only. Image qualification separately +validates `dispatch.json`, `launchable-e2e.json`, and `cleanup.json` through +`nemoclaw-maintainer-e2e`. Reject a failed workflow run before presenting the ledger. Rerun its failed jobs until the same workflow run concludes with `success`. Exceptions apply only to missing or skipped executions in that otherwise successful run. +Keep the exact-candidate release ledger separate from the image choice. An itemized exception can +allow release-ledger review, but it never qualifies an image. + +Load `nemoclaw-maintainer-e2e` to discover the newest historically validated image whose candidate SHA is +an ancestor of the release SHA. Eligible evidence can come from a trusted `main` push, selective +Launchable run, or full run. It must include a successful `Exact staging Brev Launchable` job, +`fullE2e: passed`, valid immutable image identity in `brevdevprod`, and verified cleanup. Reject +expired, missing, invalid, PR-shaped, and non-ancestor evidence. Select the newest qualified result by +`jobCompletedAt`, not by workflow creation time. Keep its `validated.json` under `EVIDENCE_DIR`. + +When a qualified candidate exists, set `VALIDATED_JSON` to the validator output for the newest qualified +ancestor. Before the image-choice prompt, calculate its commit distance: + +```bash +VALIDATED_JSON="" +test -f "$VALIDATED_JSON" +VALIDATED_SHA="$(jq -er .candidateSha "$VALIDATED_JSON")" +RELEASE_SHA="$(jq -er .originMainCommit "$PLAN_JSON")" +git merge-base --is-ancestor "$VALIDATED_SHA" "$RELEASE_SHA" +COMMIT_DISTANCE="$(git rev-list --count "$VALIDATED_SHA..$RELEASE_SHA")" +``` + +Reject a selected image when the ancestry check fails. A nonzero distance is allowed only for a +historically validated ancestor and requires explicit maintainer confirmation in the image-choice prompt. + +If no qualified ancestor image exists, tell the maintainer that no prior image can be selected and ask +whether to dispatch a fresh selective Launchable run or stop. Do not read `VALIDATED_JSON` or calculate a +distance until that run succeeds and its evidence validates. After success, set `VALIDATED_JSON`, run +the ancestry and distance commands, present the fresh image evidence, and continue with that image +selected. Skip the prior-or-fresh choice below. + +When a qualified ancestor image exists, show the maintainer: + +- the planned release tag and commit SHA; +- the exact GCP image URI, numeric ID, and self-link; +- the source image family, image-repository SHA, image creation timestamp, build or reuse result, + and origin workflow run and attempt; +- the E2E workflow URL, `runCreatedAt`, selected job `jobCompletedAt`, job URL, and selected attempt; + and +- `COMMIT_DISTANCE` as the number of commits between the validated image and release commit. + +Then ask the maintainer to choose one action: + +1. Select this exact validated image as the proposed production-image handoff. +2. Dispatch another selective `Exact staging Brev Launchable` run for the release commit and wait + for its result. + +This is an evidence-only selection. It does not persist an authenticated handoff in the semver tag, +change the current `lkg` dispatch, or cause image reuse; the current downstream workflow still +rebuilds. Do not describe either choice as a promotion. A future exact-image promotion workflow must +re-describe the selected image immediately before mutation and require `READY` status plus exact +project, name, numeric ID, and self-link matches. Historical artifacts alone do not prove that the +image still exists. + +If the maintainer chooses a fresh run, load `nemoclaw-maintainer-e2e`, dispatch Launchable mode, and +wait. Replace the selected image with that run's evidence; do not change or combine the separate +exact-candidate release ledger. Set `VALIDATED_JSON` to the fresh validator output, rerun the ancestry +and commit-distance commands above, and present all replacement image fields and timestamps before +continuing. +If the fresh run fails, report the failure. Ask whether to rerun it or return to the prior validated +image when one was qualified; otherwise offer only rerun or stop. Do not select a prior image without +confirmation. + Before showing the confirmation prompt, present: - the candidate SHA; - the number of tests with successful evidence out of the number required by the workflow; - each required test mapped to a successful run or job URL and attempt; and -- when accepted full-mode exact Brev evidence exists, its workflow URL, `Exact staging Brev Launchable` job URL, selected evidence attempt, Launchable E2E identity, and cleanup result; and -- a separate itemized maintainer exception for each missing or skipped execution in the accepted successful workflow run, including its test identifier, run links, current result, and rationale; and -- a separate itemized maintainer exception for missing or invalid exact Brev Launchable E2E evidence in the accepted successful workflow run, including run and job URLs, the missing or invalid receipt, and rationale. - -Do not ask for the phrase until the workflow run concludes with `success` and each test and the exact Brev Launchable E2E job has successful evidence or its own permitted itemized exception. -Immediately before asking, refresh `origin/main` once and compare its full SHA with the plan. If it moved, discard all prior candidate-bound evidence, regenerate the plan, rerun preflight and the full E2E workflow for the new SHA, capture a new manifest, and rebuild the ledger before requesting confirmation. +- the separate qualified image evidence's workflow URL, `Exact staging Brev Launchable` job URL, + selected evidence attempt, Launchable E2E identity, and cleanup result; and +- the maintainer's validated-image choice, exact GCP image identity, image and workflow timestamps, + selected job completion timestamp, and commit distance; and +- a separate itemized maintainer exception for each missing or skipped execution in the accepted + successful workflow run, including its test identifier, run links, current result, and rationale. + +Do not ask for the phrase until the ordinary workflow concludes with `success`, each default E2E has +successful evidence or its own permitted itemized exception, and the maintainer has selected a fully +validated image. Immediately before asking, refresh `origin/main` once and compare its full SHA with +the plan. If it moved, discard all prior candidate-bound release-ledger evidence, regenerate the plan, +rerun preflight and ordinary E2E for the new SHA, capture a new manifest, and rebuild the ledger. +Discard the prior image decision too. Repeat the image discovery and selection procedure above for +the new release SHA: find the newest qualified ancestor, present its evidence and recalculated +distance, and obtain the maintainer's prior-or-fresh choice again before requesting confirmation. Exercise the configured Git signing backend before asking for confirmation: @@ -359,10 +457,13 @@ If the Announcement is valid, return its URL with the release artifacts and mark - Plan generation fails: fix the named precondition, then regenerate the plan. - Planned changelog entry is missing or malformed: stop before plan generation and run the pre-tag `nemoclaw-contributor-update-docs` workflow. Use post-release recovery only when the tag already exists. -- Full-mode E2E waits in the Launchable concurrency queue: keep the run pending until the earlier Launchable E2E job finishes. -- Full-mode E2E ran for another SHA: reject the run and dispatch full mode for the plan candidate SHA. -- `Exact staging Brev Launchable` was skipped in an otherwise successful candidate run: dispatch full mode again or record the required itemized maintainer exception. -- Launchable E2E or cleanup evidence is missing or invalid in an otherwise successful candidate run: dispatch full mode again or record the separate itemized maintainer exception. Do not infer Launchable E2E success from the workflow conclusion. +- Ordinary release-ledger E2E ran for another SHA: reject the run and dispatch ordinary mode for the + plan candidate SHA. +- A fresh Launchable image run waits in the non-cancelling concurrency queue: keep it pending until + the earlier Launchable job finishes. +- Launchable E2E or cleanup evidence is missing or invalid: choose another historically validated ancestor + image or dispatch Launchable mode again. Do not infer image qualification from a workflow + conclusion. - `origin/main` moved after plan generation: regenerate the plan and ask for the new confirmation phrase. - Remote semver tag already exists: stop; do not retag unless the maintainer explicitly starts protected-tag remediation. - Signing preflight fails: fix the reported Git signer or signing-key failure. Run the preflight again before requesting confirmation. diff --git a/.agents/skills/nemoclaw-maintainer-cut-release-tag/scripts/release-e2e-evidence.mts b/.agents/skills/nemoclaw-maintainer-cut-release-tag/scripts/release-e2e-evidence.mts index 0eba947839a..dde999ec890 100644 --- a/.agents/skills/nemoclaw-maintainer-cut-release-tag/scripts/release-e2e-evidence.mts +++ b/.agents/skills/nemoclaw-maintainer-cut-release-tag/scripts/release-e2e-evidence.mts @@ -27,9 +27,9 @@ export type ReleaseE2ePreflight = { candidateSha: string; dispatches: { completeRun: { - includeStagingBrevLaunchable: true; + includeStagingBrevLaunchable: false; jobs: ""; - mode: "full"; + mode: "ordinary"; targets: ""; }; }; @@ -156,24 +156,18 @@ function validateDispatchIdentity( ): string { requireEqual(dispatch.candidateSha, candidateSha, `${label}.candidateSha`); const kind = stringField(dispatch, "kind", label); - if (kind === "nemoclaw-e2e-dispatch-v1") return candidateSha; if (kind !== "nemoclaw-e2e-dispatch-v2") { - throw new Error( - `${label}.kind must equal "nemoclaw-e2e-dispatch-v1" or "nemoclaw-e2e-dispatch-v2"`, - ); + throw new Error(`${label}.kind must equal "nemoclaw-e2e-dispatch-v2"`); } requireEqual(dispatch.repository, "NVIDIA/NemoClaw", `${label}.repository`); const candidateRepository = requireRepository(dispatch, "candidateRepository", label); const baseSha = requireSha(dispatch, "baseSha", label); const workflowSha = requireSha(dispatch, "workflowSha", label); - if (dispatch.prNumber === null) { - requireEqual(candidateRepository, "NVIDIA/NemoClaw", `${label}.candidateRepository`); - requireEqual(baseSha, candidateSha, `${label}.baseSha`); - requireEqual(workflowSha, candidateSha, `${label}.workflowSha`); - } else { - numberField(dispatch, "prNumber", label); - } + requireEqual(dispatch.prNumber, null, `${label}.prNumber`); + requireEqual(candidateRepository, "NVIDIA/NemoClaw", `${label}.candidateRepository`); + requireEqual(baseSha, candidateSha, `${label}.baseSha`); + requireEqual(workflowSha, candidateSha, `${label}.workflowSha`); return workflowSha; } @@ -352,9 +346,17 @@ export function buildReleaseE2ePreflight(input: { const inventory = readFreeStandingJobsInventory(workflowPath); const plan = input.plan ?? buildE2eWorkflowPlan(); const pathExists = input.candidatePathExists ?? candidatePathExists; - const defaultJobIds = inventory.workflowJobs.filter( + const workflowJobIds = inventory.workflowJobs.filter( (jobId) => jobId !== "shared-e2e" && !OPT_IN_HARDWARE_JOB_IDS.has(jobId), ); + const launchableE2eJobs = workflowJobIds.filter((jobId) => + isLaunchableE2eJob(jobId, record(jobs[jobId], `workflow.jobs.${jobId}`)), + ); + if (launchableE2eJobs.length !== 1) { + throw new Error(`expected exactly one Launchable E2E job, found ${launchableE2eJobs.length}`); + } + const launchableE2eJobId = launchableE2eJobs[0]!; + const defaultJobIds = workflowJobIds.filter((jobId) => jobId !== launchableE2eJobId); for (const jobId of defaultJobIds) { const activationPath = releaseActivationPath( record(jobs[jobId], `workflow.jobs.${jobId}`), @@ -366,13 +368,6 @@ export function buildReleaseE2ePreflight(input: { ); } } - const launchableE2eJobs = defaultJobIds.filter((jobId) => - isLaunchableE2eJob(jobId, record(jobs[jobId], `workflow.jobs.${jobId}`)), - ); - if (launchableE2eJobs.length !== 1) { - throw new Error(`expected exactly one Launchable E2E job, found ${launchableE2eJobs.length}`); - } - const launchableE2eJobId = launchableE2eJobs[0]!; const executions = [ ...defaultJobIds.flatMap((jobId) => jobExecutions(jobId, record(jobs[jobId], `workflow.jobs.${jobId}`), "default", plan), @@ -398,9 +393,9 @@ export function buildReleaseE2ePreflight(input: { candidateSha: input.candidateSha, dispatches: { completeRun: { - includeStagingBrevLaunchable: true, + includeStagingBrevLaunchable: false, jobs: "", - mode: "full", + mode: "ordinary", targets: "", }, }, @@ -476,7 +471,7 @@ export function buildReleaseE2eLedger( ); requireEqual( booleanField(dispatch, "includeStagingBrevLaunchable", `${label}.dispatch`), - true, + false, `${label}.dispatch.includeStagingBrevLaunchable`, ); requireEqual( diff --git a/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md b/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md index 8f5adc3dbe7..47c1b110a8c 100644 --- a/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md +++ b/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md @@ -11,7 +11,8 @@ description: Dispatches and verifies trusted GitHub Actions E2E for NemoClaw mai Use `.github/workflows/e2e.yaml` from trusted `main`. Every push to `main` selects the default workflow E2E jobs. Push runs skip `jetson-nvmap-gpu`, `llama-cpp-dgx-spark-plan`, and `llama-cpp-dgx-spark-qualification` because push events cannot set the required workflow dispatch flags. -Pre-tag evidence still requires the full `workflow_dispatch` mode described below. +Pre-tag release-ledger evidence uses ordinary `workflow_dispatch` mode. Release-image evidence uses +the separate fully validated Launchable result described below. Do not substitute local `npm run test:live-e2e` unless the maintainer explicitly requests local execution. ## Manual PR E2E @@ -154,7 +155,7 @@ A changed head repository, head SHA, or base SHA invalidates the evidence and re | “Run the full E2E suite” | Full | empty | `true` | | “deploy pre-release full E2E” | Full | empty | `true` | | “run pre-tag full E2E” | Full | empty | `true` | -| “run release-candidate E2E” | Full | empty | `true` | +| “run release-candidate E2E” | Ordinary | empty | `false` | A generic E2E request must not authorize the Brev Launchable path. Do not infer full mode from words such as “all” or “complete.” @@ -294,18 +295,23 @@ Queued, waiting, or accepted dispatch state is not success. ## Verify the Result -Create a private temporary evidence directory: +Create a private evidence directory. A release caller sets `EVIDENCE_DIR` to its durable private +release directory; a standalone invocation removes its temporary directory on exit: ```bash -EVIDENCE_DIR="$(mktemp -d)" +if [ -z "${EVIDENCE_DIR:-}" ]; then + EVIDENCE_DIR="$(mktemp -d)" + trap 'rm -rf "$EVIDENCE_DIR"' EXIT +else + install -d -m 0700 "$EVIDENCE_DIR" +fi chmod 700 "$EVIDENCE_DIR" -trap 'rm -rf "$EVIDENCE_DIR"' EXIT gh api "repos/NVIDIA/NemoClaw/actions/runs/$RUN_ID" >"$EVIDENCE_DIR/run-$RUN_ID.json" gh api "repos/NVIDIA/NemoClaw/actions/runs/$RUN_ID/jobs?filter=latest&per_page=100" \ >"$EVIDENCE_DIR/jobs-latest-$RUN_ID.json" ``` -For full-mode or release evidence, collect every attempt for the matrix-preserving ledger: +For release-ledger evidence and every Launchable image-evidence validation, collect every attempt: ```bash gh api --paginate --slurp \ @@ -313,7 +319,10 @@ gh api --paginate --slurp \ >"$EVIDENCE_DIR/jobs-$RUN_ID.json" ``` -Reuse `run-$RUN_ID.json` and `jobs-$RUN_ID.json` as the `nemoclaw-maintainer-cut-release-tag` manifest inputs and as the full-mode validator inputs. Do not fetch the same run again. `jobs-latest-$RUN_ID.json` is only for ordinary and Launchable modes. +Reuse `run-$RUN_ID.json` and `jobs-$RUN_ID.json` as the +`nemoclaw-maintainer-cut-release-tag` manifest inputs when building a release ledger and as validator +inputs when validating image evidence. Do not fetch the same run again. Use +`jobs-latest-$RUN_ID.json` only for an ordinary status report that does not validate image evidence. For ordinary and Launchable modes, require `run-$RUN_ID.json` to report: @@ -324,7 +333,8 @@ For ordinary and Launchable modes, require `run-$RUN_ID.json` to report: For Launchable mode, also require `jobs-latest-$RUN_ID.json` to contain one completed, successful `Exact staging Brev Launchable` job. Return the workflow and job URLs. -For full mode, select and download the Launchable E2E artifact for the latest successful Launchable job attempt: +For full mode, selective Launchable mode, or a trusted `main` push selected for an image handoff, +download the artifact for the latest successful Launchable job attempt: ```bash EVIDENCE_ATTEMPT="$(jq -er ' @@ -335,57 +345,99 @@ EVIDENCE_ATTEMPT="$(jq -er ' (.run_attempt | type) == "number") | .run_attempt] | unique | sort | last // error("no successful Launchable attempt") ' "$EVIDENCE_DIR/jobs-$RUN_ID.json")" -FULL_E2E_DIR="$EVIDENCE_DIR/full-$EVIDENCE_ATTEMPT" +FULL_E2E_DIR="$EVIDENCE_DIR/image-$RUN_ID-$EVIDENCE_ATTEMPT" install -d -m 0700 "$FULL_E2E_DIR" gh run download "$RUN_ID" --repo NVIDIA/NemoClaw \ --name "staging-brev-launchable-${CANDIDATE_SHA}-${RUN_ID}-${EVIDENCE_ATTEMPT}" \ --dir "$FULL_E2E_DIR" +VALIDATOR_MODE_ARGS=() +if [ -n "${EVIDENCE_MODE:-}" ]; then + VALIDATOR_MODE_ARGS=(--mode "$EVIDENCE_MODE") +fi node --experimental-strip-types --no-warnings \ .agents/skills/nemoclaw-maintainer-e2e/scripts/validate-full-e2e-evidence.mts \ + "${VALIDATOR_MODE_ARGS[@]}" \ --candidate-sha "$CANDIDATE_SHA" \ --run-json "$EVIDENCE_DIR/run-$RUN_ID.json" \ --jobs-json "$EVIDENCE_DIR/jobs-$RUN_ID.json" \ --dispatch-json "$FULL_E2E_DIR/dispatch.json" \ --launchable-e2e-json "$FULL_E2E_DIR/launchable-e2e.json" \ - --cleanup-json "$FULL_E2E_DIR/cleanup.json" + --cleanup-json "$FULL_E2E_DIR/cleanup.json" \ + >"$FULL_E2E_DIR/validated.json" ``` -The validator requires: +Leave `EVIDENCE_MODE` unset for the legacy full-run validator path. For a release-image evidence +candidate, set it to `full`, `launchable`, or `push`. That stricter path rejects PR-shaped evidence. + +Every validator mode requires: - the workflow run to succeed for the selected SHA; -- `dispatch.json` to bind the same run, empty selectors, `include_staging_brev_launchable=true`, `allowJetsonRunnerQueue: false`, `allowDgxSparkRunnerQueue: false`, and the selected successful Launchable job attempt; +- `dispatch.json` to bind the same run and selected successful Launchable job attempt; - `allowJetsonDispatch: false` in every v2 `dispatch.json` receipt; - `allowJetsonDispatch` to be absent or `false` in every v1 `dispatch.json` receipt; +- `allowJetsonRunnerQueue: false` and `allowDgxSparkRunnerQueue: false`; - `Exact staging Brev Launchable` to conclude `success` in the selected current or earlier attempt of the same workflow run; - `launchable-e2e.json` to identify the selected SHA in the repository and provision records; +- `launchable-e2e.json` to bind the booted GCP image URI to its project, name, numeric ID, + self-link, source family, image origin, creation time, and image-repository SHA; +- the source project to equal trusted project `brevdevprod`; - the booted repository to be unmodified; - the in-guest full E2E to pass; and - `cleanup.json` to report the same workspace as `ABSENT`. +The selector contract is mode-specific: `full` requires empty selectors and +`include_staging_brev_launchable=true`; `launchable` requires +`jobs=staging-brev-launchable`, empty targets, and the opt-in false; `push` requires empty selectors +and the opt-in false. + A skipped, cancelled, queued, or failed Launchable E2E job is not evidence. -A Launchable-mode run is not full-mode or pre-tag release evidence. +A Launchable-mode run is not release-ledger evidence. It is qualified historical image evidence after +the stricter direct `main` validator succeeds. A missing, mismatched, or failed cleanup receipt is not evidence. +## Select Release Image Evidence + +When `nemoclaw-maintainer-cut-release-tag` requests an image choice, list successful E2E workflow +runs on `main` newest first. Inspect each run's `Exact staging Brev Launchable` job and artifact by +using the collection and validator steps above. Derive `EVIDENCE_MODE` from the trusted receipt: + +- use `push` for a `push` event with empty selectors and no Launchable opt-in; +- use `launchable` for `workflow_dispatch` with `jobs=staging-brev-launchable`; and +- use `full` for `workflow_dispatch` with empty selectors and + `include_staging_brev_launchable=true`. + +Reject an expired or missing artifact, a failed or skipped job, invalid evidence, and PR-shaped +evidence. An itemized release-ledger exception is not image evidence. Return every valid candidate +needed for the caller's ancestry check, sorted by `jobCompletedAt` newest first. The caller chooses +the newest historically validated ancestor of the release SHA and keeps its `validated.json` in the private +release evidence directory. + ## Bind Release Evidence -If no release plan exists, label a successful full run against `origin/main` as provisional release evidence. +If no release plan exists, label a successful ordinary run against `origin/main` as provisional +release-ledger evidence. Return validated Launchable evidence separately when available. Return: +- the exact `validated.json` path; - candidate SHA; - workflow run URL and conclusion; +- workflow run ID, creation timestamp, and selected job completion timestamp; - `Exact staging Brev Launchable` job URL; - selected successful Launchable job attempt; -- Launchable E2E identity; and +- exact GCP image and Launchable E2E identity; and - cleanup result. -If the release candidate SHA changes, discard the earlier full run and dispatch full mode for the new SHA. +If the release candidate SHA changes, discard the earlier release ledger and dispatch ordinary mode +for the new SHA. Discard the prior image decision, rediscover the newest qualified ancestor for the +new SHA, present its evidence, and obtain the image choice again. No release-note-only delta exception is currently defined. When `nemoclaw-maintainer-cut-release-tag` invokes this skill, return the validated fields for its pre-tag E2E evidence ledger. -The trusted `dispatch.json` receipt proves that full mode used empty selectors and included `Exact staging Brev Launchable`. -For Jetson dispatch, a v2 receipt requires `allowJetsonDispatch: false`. -A v1 receipt may omit `allowJetsonDispatch`, but it must be `false` when present. -Both receipt versions require the optional DGX Spark runner path to be disabled. +The release-ledger `dispatch.json` is a v2 direct `main` receipt. It proves ordinary mode used empty +selectors and excluded `Exact staging Brev Launchable`; `prNumber` is null, both repositories are +`NVIDIA/NemoClaw`, and its base, workflow, and candidate SHAs are identical. It also requires +`allowJetsonDispatch: false` and both optional runner queues disabled. Image qualification validates +its own mode-specific v1 or v2 receipt. The release evidence ledger proves the result of each workflow E2E. Do not ask for the release confirmation phrase in this skill. diff --git a/.agents/skills/nemoclaw-maintainer-e2e/scripts/validate-full-e2e-evidence.mts b/.agents/skills/nemoclaw-maintainer-e2e/scripts/validate-full-e2e-evidence.mts index f38b36ae847..2572ec42869 100644 --- a/.agents/skills/nemoclaw-maintainer-e2e/scripts/validate-full-e2e-evidence.mts +++ b/.agents/skills/nemoclaw-maintainer-e2e/scripts/validate-full-e2e-evidence.mts @@ -9,6 +9,12 @@ type JsonRecord = Record; const SHA_PATTERN = /^[0-9a-f]{40}$/u; const REPOSITORY_PATTERN = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u; +const DECIMAL_ID_PATTERN = /^[1-9][0-9]*$/u; +const GCP_PROJECT_PATTERN = /^[a-z][a-z0-9-]{4,28}[a-z0-9]$/u; +const GCP_IMAGE_NAME_PATTERN = /^[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?$/u; +const EXPECTED_GCP_PROJECT = "brevdevprod"; +const RFC3339_PATTERN = + /^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]+)?(?:Z|[+-][0-9]{2}:[0-9]{2})$/u; export interface FullE2eEvidenceInput { candidateSha: string; @@ -19,6 +25,8 @@ export interface FullE2eEvidenceInput { run: unknown; } +export type BrevImageEvidenceMode = "full" | "launchable" | "push"; + export interface FullE2eEvidenceSummary { attempt: number; candidateSha: string; @@ -32,17 +40,33 @@ export interface FullE2eEvidenceSummary { allowDgxSparkRunnerQueue: false; allowJetsonDispatch: false; allowJetsonRunnerQueue: false; - emptySelectors: true; - includeStagingBrevLaunchable: true; + emptySelectors: boolean; + includeStagingBrevLaunchable: boolean; }; + evidenceMode: BrevImageEvidenceMode; + jobCompletedAt: string; jobUrl: string; launchableE2e: { fullE2e: "passed"; + image: { + imageCreationTimestamp: string; + imageId: string; + imageName: string; + imageOriginWorkflowRunAttempt: number; + imageOriginWorkflowRunId: string; + imageRepositorySha: string; + imageSelfLink: string; + observedFamily: "nemoclaw-brev-staging-cpu"; + project: string; + result: "built" | "reused"; + }; producerRunId: string; provisionSha: string; repoClean: true; repoSha: string; }; + runCreatedAt: string; + runId: number; runUrl: string; } @@ -82,6 +106,14 @@ function positiveIntegerField(value: JsonRecord, key: string, owner: string): nu return Number(field); } +function timestampField(value: JsonRecord, key: string, owner: string): string { + const timestamp = stringField(value, key, owner); + if (!RFC3339_PATTERN.test(timestamp) || Number.isNaN(Date.parse(timestamp))) { + throw new Error(`${owner}.${key} must be an RFC 3339 timestamp`); + } + return timestamp; +} + function requireEqual(actual: unknown, expected: unknown, owner: string): void { if (actual !== expected) { throw new Error(`${owner} must equal ${JSON.stringify(expected)}`); @@ -110,7 +142,11 @@ function requireRepository(value: JsonRecord, key: string, owner: string): strin return repository; } -function validateDispatchIdentity(dispatch: JsonRecord, candidateSha: string): string { +function validateDispatchIdentity( + dispatch: JsonRecord, + candidateSha: string, + requireDirectMain: boolean, +): string { requireEqual(dispatch.candidateSha, candidateSha, "dispatch.candidateSha"); const kind = stringField(dispatch, "kind", "dispatch"); if (kind === "nemoclaw-e2e-dispatch-v1") return candidateSha; @@ -130,18 +166,26 @@ function validateDispatchIdentity(dispatch: JsonRecord, candidateSha: string): s requireEqual(workflowSha, candidateSha, "dispatch.workflowSha"); } else { positiveIntegerField(dispatch, "prNumber", "dispatch"); + if (requireDirectMain) { + throw new Error("dispatch.prNumber must be null for release image evidence"); + } } return workflowSha; } -export function validateFullE2eEvidence(input: FullE2eEvidenceInput): FullE2eEvidenceSummary { +function validateEvidence( + input: FullE2eEvidenceInput, + evidenceMode: BrevImageEvidenceMode, + requireDirectMain: boolean, +): FullE2eEvidenceSummary { if (!SHA_PATTERN.test(input.candidateSha)) { throw new Error("candidate SHA must be a lowercase 40-character SHA"); } const run = record(input.run, "run"); + const expectedEvent = evidenceMode === "push" ? "push" : "workflow_dispatch"; requireEqual(run.head_branch, "main", "run.head_branch"); - requireEqual(run.event, "workflow_dispatch", "run.event"); + requireEqual(run.event, expectedEvent, "run.event"); requireEqual(run.path, ".github/workflows/e2e.yaml", "run.path"); requireEqual(run.status, "completed", "run.status"); requireEqual(run.conclusion, "success", "run.conclusion"); @@ -149,29 +193,55 @@ export function validateFullE2eEvidence(input: FullE2eEvidenceInput): FullE2eEvi const runUrl = stringField(run, "html_url", "run"); requireGitHubUrl(runUrl, "run.html_url"); const runId = positiveIntegerField(run, "id", "run"); + const runCreatedAt = timestampField(run, "created_at", "run"); const dispatch = record(input.dispatch, "dispatch"); - const expectedWorkflowSha = validateDispatchIdentity(dispatch, input.candidateSha); + const expectedWorkflowSha = validateDispatchIdentity( + dispatch, + input.candidateSha, + requireDirectMain, + ); requireEqual(run.head_sha, expectedWorkflowSha, "run.head_sha"); - requireEqual(dispatch.eventName, "workflow_dispatch", "dispatch.eventName"); + requireEqual(dispatch.eventName, expectedEvent, "dispatch.eventName"); requireEqual(dispatch.workflowRunId, String(runId), "dispatch.workflowRunId"); const receiptAttempt = positiveIntegerField(dispatch, "workflowRunAttempt", "dispatch"); if (receiptAttempt > attempt) { throw new Error("dispatch.workflowRunAttempt exceeds run.run_attempt"); } - requireEqual(dispatch.jobs, "", "dispatch.jobs"); requireEqual(dispatch.targets, "", "dispatch.targets"); - requireEqual( - dispatch.includeStagingBrevLaunchable, - true, - "dispatch.includeStagingBrevLaunchable", - ); + if (evidenceMode === "full") { + requireEqual(dispatch.jobs, "", "dispatch.jobs"); + requireEqual( + dispatch.includeStagingBrevLaunchable, + true, + "dispatch.includeStagingBrevLaunchable", + ); + requireEqual(dispatch.emptySelectors, true, "dispatch.emptySelectors"); + } else if (evidenceMode === "launchable") { + requireEqual(dispatch.jobs, "staging-brev-launchable", "dispatch.jobs"); + requireEqual( + dispatch.includeStagingBrevLaunchable, + false, + "dispatch.includeStagingBrevLaunchable", + ); + requireEqual(dispatch.emptySelectors, false, "dispatch.emptySelectors"); + } else { + requireEqual(dispatch.jobs, "", "dispatch.jobs"); + requireEqual( + dispatch.includeStagingBrevLaunchable, + false, + "dispatch.includeStagingBrevLaunchable", + ); + requireEqual(dispatch.emptySelectors, true, "dispatch.emptySelectors"); + } requireEqual(dispatch.allowJetsonRunnerQueue, false, "dispatch.allowJetsonRunnerQueue"); - if (dispatch.kind === "nemoclaw-e2e-dispatch-v2" || Object.hasOwn(dispatch, "allowJetsonDispatch")) { + if ( + dispatch.kind === "nemoclaw-e2e-dispatch-v2" || + Object.hasOwn(dispatch, "allowJetsonDispatch") + ) { requireEqual(dispatch.allowJetsonDispatch, false, "dispatch.allowJetsonDispatch"); } requireEqual(dispatch.allowDgxSparkRunnerQueue, false, "dispatch.allowDgxSparkRunnerQueue"); - requireEqual(dispatch.emptySelectors, true, "dispatch.emptySelectors"); const matchingJobs = jobRecords(input.jobs).filter( (job) => job.name === "Exact staging Brev Launchable", @@ -198,6 +268,7 @@ export function validateFullE2eEvidence(input: FullE2eEvidenceInput): FullE2eEvi } const job = successfulJobs[0]!.job; const jobUrl = stringField(job, "html_url", "Exact staging Brev Launchable"); + const jobCompletedAt = timestampField(job, "completed_at", "Exact staging Brev Launchable"); requireGitHubUrl(jobUrl, "Exact staging Brev Launchable html_url"); if (!jobUrl.startsWith(`${runUrl}/job/`)) { throw new Error("Exact staging Brev Launchable html_url must belong to the workflow run"); @@ -209,9 +280,80 @@ export function validateFullE2eEvidence(input: FullE2eEvidenceInput): FullE2eEvi const producer = record(launchableE2e.producer, "launchableE2e.producer"); requireEqual(producer.status, "success", "launchableE2e.producer.status"); const producerRunId = stringField(producer, "runId", "launchableE2e.producer"); + if (!DECIMAL_ID_PATTERN.test(producerRunId)) { + throw new Error("launchableE2e.producer.runId must be a positive decimal ID"); + } + + const image = record(launchableE2e.image, "launchableE2e.image"); + const project = stringField(image, "project", "launchableE2e.image"); + if (!GCP_PROJECT_PATTERN.test(project)) { + throw new Error("launchableE2e.image.project must be a canonical GCP project ID"); + } + requireEqual(project, EXPECTED_GCP_PROJECT, "launchableE2e.image.project"); + const imageName = stringField(image, "imageName", "launchableE2e.image"); + if (!GCP_IMAGE_NAME_PATTERN.test(imageName)) { + throw new Error("launchableE2e.image.imageName must be a canonical GCP image name"); + } + const imageId = stringField(image, "imageId", "launchableE2e.image"); + if (!DECIMAL_ID_PATTERN.test(imageId)) { + throw new Error("launchableE2e.image.imageId must be a positive decimal ID"); + } + const imageSelfLink = stringField(image, "imageSelfLink", "launchableE2e.image"); + const expectedImageSelfLink = `https://www.googleapis.com/compute/v1/projects/${project}/global/images/${imageName}`; + requireEqual(imageSelfLink, expectedImageSelfLink, "launchableE2e.image.imageSelfLink"); + const imageCreationTimestamp = timestampField( + image, + "imageCreationTimestamp", + "launchableE2e.image", + ); + const imageRepositorySha = requireSha(image, "imageRepositorySha", "launchableE2e.image"); + const imageOriginWorkflowRunId = stringField( + image, + "imageOriginWorkflowRunId", + "launchableE2e.image", + ); + if (!DECIMAL_ID_PATTERN.test(imageOriginWorkflowRunId)) { + throw new Error("launchableE2e.image.imageOriginWorkflowRunId must be a positive decimal ID"); + } + const imageOriginWorkflowRunAttempt = positiveIntegerField( + image, + "imageOriginWorkflowRunAttempt", + "launchableE2e.image", + ); + requireEqual( + image.observedFamily, + "nemoclaw-brev-staging-cpu", + "launchableE2e.image.observedFamily", + ); + const result = stringField(image, "result", "launchableE2e.image"); + if (result !== "built" && result !== "reused") { + throw new Error('launchableE2e.image.result must equal "built" or "reused"'); + } + if (result === "built") { + requireEqual( + imageOriginWorkflowRunId, + producerRunId, + "launchableE2e.image.imageOriginWorkflowRunId", + ); + requireEqual( + imageOriginWorkflowRunAttempt, + 1, + "launchableE2e.image.imageOriginWorkflowRunAttempt", + ); + } const boot = record(launchableE2e.boot, "launchableE2e.boot"); + requireEqual( + boot.bootImage, + `projects/${project}/global/images/${imageName}`, + "launchableE2e.boot.bootImage", + ); requireEqual(boot.repoSha, input.candidateSha, "launchableE2e.boot.repoSha"); requireEqual(boot.provisionSha, input.candidateSha, "launchableE2e.boot.provisionSha"); + requireEqual( + boot.imageRepositorySha, + imageRepositorySha, + "launchableE2e.boot.imageRepositorySha", + ); requireEqual(boot.repoClean, true, "launchableE2e.boot.repoClean"); const workspace = record(launchableE2e.workspace, "launchableE2e.workspace"); @@ -239,21 +381,48 @@ export function validateFullE2eEvidence(input: FullE2eEvidenceInput): FullE2eEvi allowDgxSparkRunnerQueue: false, allowJetsonDispatch: false, allowJetsonRunnerQueue: false, - emptySelectors: true, - includeStagingBrevLaunchable: true, + emptySelectors: evidenceMode !== "launchable", + includeStagingBrevLaunchable: evidenceMode === "full", }, + evidenceMode, + jobCompletedAt, jobUrl, launchableE2e: { fullE2e: "passed", + image: { + imageCreationTimestamp, + imageId, + imageName, + imageOriginWorkflowRunAttempt, + imageOriginWorkflowRunId, + imageRepositorySha, + imageSelfLink, + observedFamily: "nemoclaw-brev-staging-cpu", + project, + result, + }, producerRunId, provisionSha: input.candidateSha, repoClean: true, repoSha: input.candidateSha, }, + runCreatedAt, + runId, runUrl, }; } +export function validateFullE2eEvidence(input: FullE2eEvidenceInput): FullE2eEvidenceSummary { + return validateEvidence(input, "full", false); +} + +export function validateBrevImageEvidence( + input: FullE2eEvidenceInput, + evidenceMode: BrevImageEvidenceMode, +): FullE2eEvidenceSummary { + return validateEvidence(input, evidenceMode, true); +} + function readJson(file: string): unknown { return JSON.parse(fs.readFileSync(file, "utf8")); } @@ -266,6 +435,7 @@ function main(): void { "dispatch-json": { type: "string" }, "jobs-json": { type: "string" }, "launchable-e2e-json": { type: "string" }, + mode: { type: "string" }, "run-json": { type: "string" }, }, strict: true, @@ -281,14 +451,19 @@ function main(): void { if (!values[name]) throw new Error(`--${name} is required`); } - const summary = validateFullE2eEvidence({ + const input = { candidateSha: values["candidate-sha"]!, cleanup: readJson(values["cleanup-json"]!), dispatch: readJson(values["dispatch-json"]!), jobs: readJson(values["jobs-json"]!), launchableE2e: readJson(values["launchable-e2e-json"]!), run: readJson(values["run-json"]!), - }); + }; + const mode = values.mode; + if (mode !== undefined && mode !== "full" && mode !== "launchable" && mode !== "push") { + throw new Error('--mode must equal "full", "launchable", or "push"'); + } + const summary = mode ? validateBrevImageEvidence(input, mode) : validateFullE2eEvidence(input); process.stdout.write(`${JSON.stringify(summary, null, 2)}\n`); } diff --git a/.agents/skills/nemoclaw-maintainer-evening/SKILL.md b/.agents/skills/nemoclaw-maintainer-evening/SKILL.md index 330f2a7327b..745aa68e7e0 100644 --- a/.agents/skills/nemoclaw-maintainer-evening/SKILL.md +++ b/.agents/skills/nemoclaw-maintainer-evening/SKILL.md @@ -55,12 +55,15 @@ The version is already known, so use a patch bump unless the maintainer selects Show the commit, changelog, carry-forward plan, label-retirement plan, and release notes draft. After the release plan captures the candidate SHA, load `nemoclaw-maintainer-e2e`. -Run full mode unless one existing full run for the candidate SHA contains complete workflow E2E and `Exact staging Brev Launchable` evidence. +Run ordinary mode unless one existing ordinary run for the candidate SHA contains complete default +workflow E2E evidence. Review the pre-tag E2E evidence ledger from `.github/workflows/e2e.yaml` at that commit. -Require the accepted workflow run to conclude with `success`. Require successful `Exact staging Brev Launchable` evidence with matching Launchable E2E identity and verified workspace absence, or record the permitted itemized exception described below. +Require the accepted workflow run to conclude with `success`. Separately select the newest +historically validated ancestor image or let the maintainer request a fresh selective Launchable run. Each missing or skipped execution in that successful run requires its own itemized maintainer exception. -Missing or invalid Launchable E2E evidence in that successful run requires a separate itemized exception with run and job URLs, the missing or invalid receipt, and rationale. -Do not ask for the release confirmation phrase until the run succeeds and each required execution has successful evidence or a permitted exception. +An exception never qualifies an image. Do not ask for the release confirmation phrase until the +ordinary run succeeds, each required default execution has successful evidence or a permitted +exception, and the maintainer selects fully validated image evidence. Tag the confirmed release commit with `vX.Y.Z`. Let the workflow move `latest`, carry open work forward, and delete the released label. @@ -71,7 +74,7 @@ Prepare the Announcement draft for the maintainer to post. After the tag is cut and release notes are drafted or posted by the maintainer, present the final summary: - **Tag**: `v0.0.8` at commit `abc1234` -- **Pre-tag E2E evidence**: 12/13 tests and exact Brev Launchable E2E passing for the candidate SHA; 1 itemized maintainer exception +- **Pre-tag E2E evidence**: 12/13 default tests passing for the candidate SHA; 1 itemized maintainer exception; validated Brev image selected from an ancestor 3 commits behind - **Release notes draft**: `../nemoclaw-release-v0.0.8/release-note-draft.md` - **Shipped**: 4 items (#1234, #1235, #1236, #1237) - **Moved to v0.0.9**: 1 item (#1238 — still needs CI fix) diff --git a/.agents/skills/nemoclaw-maintainer-policies/references/release-train.md b/.agents/skills/nemoclaw-maintainer-policies/references/release-train.md index 0e98df5b46e..ae61b0f49e8 100644 --- a/.agents/skills/nemoclaw-maintainer-policies/references/release-train.md +++ b/.agents/skills/nemoclaw-maintainer-policies/references/release-train.md @@ -54,25 +54,57 @@ Before asking for the release confirmation phrase, build and show an evidence le - Preflight the candidate workflow and existing candidate evidence before dispatching new work. - Derive the denominator from the candidate workflow. Do not copy it into a second release test list. - Require every declared `RELEASE_E2E_ACTIVATION_PATH` to exist at the candidate SHA. A missing path is a preflight failure. -- Require the workflow-produced trusted dispatch receipt to bind the accepted run candidate SHA, run ID, attempt, and selector inputs. -- Run `nemoclaw-maintainer-e2e` in full mode when the ledger lacks complete evidence for the candidate SHA. -- Require one completed, successful full workflow run for all default-selected workflow E2E jobs and the full-mode additions, including `Exact staging Brev Launchable`. -- Require the trusted dispatch receipt to record `allowJetsonRunnerQueue: false` and `allowDgxSparkRunnerQueue: false`. A v1 receipt may omit `allowJetsonDispatch` but must record `false` when the field is present. A v2 receipt must record `allowJetsonDispatch: false`. +- Require a v2 workflow-produced trusted dispatch receipt with `prNumber: null`, repository and + candidate repository both `NVIDIA/NemoClaw`, and base, workflow, and candidate SHAs all equal to + the release candidate. Bind the accepted run ID, attempt, and selector inputs. +- Run `nemoclaw-maintainer-e2e` in ordinary mode when the ledger lacks complete evidence for the + candidate SHA. +- Require one completed, successful ordinary workflow run for all default-selected workflow E2E + jobs. Qualify the release image separately so release-ledger completion does not trigger an image + build. +- Require the trusted dispatch receipt to record `allowJetsonDispatch: false`, + `allowJetsonRunnerQueue: false`, and `allowDgxSparkRunnerQueue: false`. - Exclude `jetson-nvmap-gpu`, `llama-cpp-dgx-spark-plan`, and `llama-cpp-dgx-spark-qualification` from the required denominator. -- Require the trusted dispatch receipt to bind the workflow run and an attempt no later than the run's latest attempt. The receipt must record empty selectors and `include_staging_brev_launchable=true`. -- Require the Launchable E2E receipt to identify the candidate SHA in the repository and provision records. -- Require the cleanup receipt to identify the qualified workspace and report `ABSENT`. +- Require the trusted dispatch receipt to bind the workflow run and an attempt no later than the + run's latest attempt. The receipt must record empty selectors and + `include_staging_brev_launchable=false`. - Every E2E execution selected by the accepted dispatch must have at least one completed, successful execution for the candidate SHA. - Treat each expanded matrix execution as a separate ledger entry. Use its matrix `id`, or all distinguishing matrix dimensions when no single ID exists, in the test identifier so results for distinct expansions are never collapsed under the parent job. - Successful evidence may accumulate across rerun attempts of that workflow run. Evidence from another workflow run does not satisfy the ledger. A later failure does not erase an earlier successful execution for the same test and SHA. - Skipped, unexecuted, queued, in-progress, cancelled, and failing results do not count as successful evidence. - Map each test with successful evidence to its successful run or job URL and attempt number. - Each missing or skipped execution in the accepted successful workflow run requires its own itemized maintainer exception. Record the test identifier, relevant run links or available evidence, the current result, and the rationale. -- Missing or invalid exact Brev Launchable E2E evidence in the accepted successful workflow run requires a separate itemized maintainer exception. Record the run and job URLs, the missing or invalid receipt, and the rationale. The accepted workflow run must be completed and have a `success` conclusion. A failed workflow run cannot supply the release ledger. Rerun its failed jobs until the workflow concludes with `success`. An itemized test exception applies only to a missing or skipped execution in that otherwise successful run. -Each test and the exact Brev Launchable E2E job in the accepted successful workflow run must have successful evidence or its own permitted itemized exception before release confirmation. Immediately before confirmation, compare `origin/main` with the planned SHA. If the candidate SHA changes, discard the ledger and its exceptions, including Launchable E2E evidence. Regenerate the release plan and repeat the review for the new SHA. This does not freeze `main` or prevent merges. No release-note-only delta exception is currently defined. +Keep the exact-candidate release ledger separate from the image choice. Before release confirmation, +show the newest historically validated image whose candidate SHA is an ancestor of the release commit. +Evidence may come from a trusted `main` push, selective Launchable run, or full run, but it must have +a successful `Exact staging Brev Launchable` job, valid immutable image identity in `brevdevprod`, +`fullE2e: passed`, and verified cleanup. An itemized exception never qualifies an image. + +Include the image creation time, workflow creation time, selected job completion time, and commit +distance from the validated SHA to the release commit. Ask the maintainer to select that image as the +proposed exact-image handoff or run the selective Launchable test again and wait. A nonzero distance +requires explicit confirmation. If a fresh run fails, do not return to the earlier image without +maintainer confirmation. Image selection records evidence for the proposed handoff; it does not by +itself promote a GCP image family. + +The selection is evidence only: the current `lkg` path does not consume it and still rebuilds. Do not +claim that the selected image was reused or promoted. Before a future exact-image promotion mutates a +family, its trusted downstream workflow must re-describe the image and require `READY` status plus +exact project, name, numeric ID, and self-link matches. Retained evidence does not prove that the +image still exists. + +Each default-selected test in the accepted successful workflow run must have successful evidence or +its own permitted itemized exception before release confirmation. The image choice must independently +have complete Launchable E2E and cleanup evidence; no exception can qualify an image. Immediately +before confirmation, compare `origin/main` with the planned SHA. If the candidate SHA changes, +discard the ledger and its exceptions, regenerate the release plan, and repeat the review for +the new SHA. Discard the prior image decision, discover the newest qualified ancestor for the new +release SHA, present its evidence and recalculated distance, and obtain the prior-or-fresh choice +again. This does not freeze `main` or prevent merges. No release-note-only delta exception is +currently defined. ## Carry Forward diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index deedc25bd5b..20b1e5210f6 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -723,6 +723,9 @@ jobs: - name: Record E2E dispatch identity env: + ALLOW_DGX_SPARK_RUNNER_QUEUE: ${{ inputs.allow_dgx_spark_runner_queue && 'true' || 'false' }} + ALLOW_JETSON_DISPATCH: ${{ inputs.allow_jetson_dispatch && 'true' || 'false' }} + ALLOW_JETSON_RUNNER_QUEUE: "false" CANDIDATE_SHA: ${{ env.CANDIDATE_SHA }} DISPATCH_JOBS: ${{ inputs.jobs }} DISPATCH_TARGETS: ${{ inputs.targets }} @@ -738,6 +741,9 @@ jobs: --arg jobs "$DISPATCH_JOBS" \ --arg targets "$DISPATCH_TARGETS" \ --arg workflowRunId "$RUN_ID" \ + --argjson allowDgxSparkRunnerQueue "$ALLOW_DGX_SPARK_RUNNER_QUEUE" \ + --argjson allowJetsonDispatch "$ALLOW_JETSON_DISPATCH" \ + --argjson allowJetsonRunnerQueue "$ALLOW_JETSON_RUNNER_QUEUE" \ --argjson includeStagingBrevLaunchable "$INCLUDE_STAGING_BREV_LAUNCHABLE" \ --argjson workflowRunAttempt "$RUN_ATTEMPT" \ '{ @@ -746,6 +752,9 @@ jobs: eventName: $eventName, workflowRunId: $workflowRunId, workflowRunAttempt: $workflowRunAttempt, + allowDgxSparkRunnerQueue: $allowDgxSparkRunnerQueue, + allowJetsonDispatch: $allowJetsonDispatch, + allowJetsonRunnerQueue: $allowJetsonRunnerQueue, jobs: $jobs, targets: $targets, includeStagingBrevLaunchable: $includeStagingBrevLaunchable, diff --git a/test/brev-launchable-e2e.test.ts b/test/brev-launchable-e2e.test.ts index fdfbd96fca9..87a4388e216 100644 --- a/test/brev-launchable-e2e.test.ts +++ b/test/brev-launchable-e2e.test.ts @@ -25,9 +25,20 @@ function fixture( bootImage?: string; deleteFails?: boolean; e2eFails?: boolean; + imageId?: string; + imageProject?: string; imageRepositorySha?: string; + imageSelfLink?: string; missingProvisionReceipt?: boolean; - omitReceiptField?: "imageName" | "imageRepositorySha" | "project"; + omitReceiptField?: + | "imageCreationTimestamp" + | "imageId" + | "imageName" + | "imageOriginWorkflowRunId" + | "imageRepositorySha" + | "imageSelfLink" + | "project" + | "result"; provisionImageRepositorySha?: string; provisionSha?: string; ready?: boolean; @@ -94,14 +105,21 @@ elif [ "$1 $2" = 'run download' ]; then done mkdir -p "$directory" jq -n --arg sha "$FAKE_RECEIPT_SHA" --arg correlation "$CORRELATION_ID" \ + --arg imageCreationTimestamp "$FAKE_IMAGE_CREATION_TIMESTAMP" \ + --arg imageId "$FAKE_IMAGE_ID" \ + --arg imageProject "$FAKE_IMAGE_PROJECT" \ --arg imageRepositorySha "$FAKE_IMAGE_REPOSITORY_SHA" \ + --arg imageSelfLink "$FAKE_IMAGE_SELF_LINK" \ --arg omit "$FAKE_OMIT_RECEIPT_FIELD" '{ kind:"nemoclaw-exact-image-manifest",nemoclawSha:$sha,correlationId:$correlation, - requesterWorkflowRunId:"789",requesterWorkflowRunAttempt:1, + requesterRepository:"NVIDIA/NemoClaw",requesterWorkflowRunId:"789",requesterWorkflowRunAttempt:1, imageRepository:"brevdev/nemoclaw-image",producerWorkflow:".github/workflows/build-launchable-e2e-image.yml", - workflowRunId:"123",workflowRunAttempt:1, - status:"READY",channel:"staging",variant:"cpu",observedFamily:"nemoclaw-brev-staging-cpu", - project:"brevdevprod",imageName:"nemoclaw-test-image",imageRepositorySha:$imageRepositorySha + workflowRunId:"123",workflowRunAttempt:1,imageOriginWorkflowRunId:"123",imageOriginWorkflowRunAttempt:1, + imageKind:"compute#image",status:"READY",channel:"staging",variant:"cpu", + observedFamily:"nemoclaw-brev-staging-cpu",result:"built", + project:$imageProject,imageName:"nemoclaw-test-image",imageId:$imageId, + imageSelfLink:$imageSelfLink,imageCreationTimestamp:$imageCreationTimestamp, + imageRepositorySha:$imageRepositorySha } | if $omit == "" then . else del(.[$omit]) end' > "$directory/nemoclaw-image-manifest.v1.json" else exit 2 @@ -187,7 +205,13 @@ printf 'NEMOCLAW_FULL_E2E_PASSED\\n' FAKE_CALLS: calls, FAKE_DELETE_FAILS: options.deleteFails ? "1" : "0", FAKE_E2E_FAILS: options.e2eFails ? "1" : "0", + FAKE_IMAGE_CREATION_TIMESTAMP: "2026-08-11T20:00:00Z", + FAKE_IMAGE_ID: options.imageId ?? "5831758261704356816", + FAKE_IMAGE_PROJECT: options.imageProject ?? "brevdevprod", FAKE_IMAGE_REPOSITORY_SHA: options.imageRepositorySha ?? "b".repeat(40), + FAKE_IMAGE_SELF_LINK: + options.imageSelfLink ?? + "https://www.googleapis.com/compute/v1/projects/brevdevprod/global/images/nemoclaw-test-image", FAKE_MISSING_PROVISION_RECEIPT: options.missingProvisionReceipt ? "1" : "0", FAKE_OMIT_RECEIPT_FIELD: options.omitReceiptField ?? "", FAKE_PROVISION_IMAGE_REPOSITORY_SHA: @@ -274,6 +298,19 @@ describe("focused staging Brev Launchable lane", () => { candidateSha, fullE2e: "passed", producer: { runId: "123", status: "success" }, + image: { + imageCreationTimestamp: "2026-08-11T20:00:00Z", + imageId: "5831758261704356816", + imageName: "nemoclaw-test-image", + imageOriginWorkflowRunAttempt: 1, + imageOriginWorkflowRunId: "123", + imageRepositorySha: "b".repeat(40), + imageSelfLink: + "https://www.googleapis.com/compute/v1/projects/brevdevprod/global/images/nemoclaw-test-image", + observedFamily: "nemoclaw-brev-staging-cpu", + project: "brevdevprod", + result: "built", + }, boot: { bootImage: "projects/brevdevprod/global/images/nemoclaw-test-image", sourcePath: "/opt/nemoclaw-image/NemoClaw", @@ -286,7 +323,7 @@ describe("focused staging Brev Launchable lane", () => { }); }); - it("blocks E2E for a wrong receipt, incomplete readiness, or booted checkout mismatch", () => { + it("blocks deployment when producer evidence does not match the candidate", () => { const receipt = fixture({ receiptSha: "b".repeat(40) }); const receiptResult = run(receipt.env); expect(receiptResult.status).not.toBe(0); @@ -296,6 +333,9 @@ describe("focused staging Brev Launchable lane", () => { for (const malformed of [ fixture({ omitReceiptField: "project" }), fixture({ omitReceiptField: "imageName" }), + fixture({ omitReceiptField: "imageCreationTimestamp" }), + fixture({ omitReceiptField: "imageOriginWorkflowRunId" }), + fixture({ omitReceiptField: "result" }), fixture({ imageRepositorySha: "not-a-sha" }), ]) { const malformedResult = run(malformed.env); @@ -305,7 +345,9 @@ describe("focused staging Brev Launchable lane", () => { /brev create|full-e2e\.test\.ts/u, ); } + }); + it("blocks E2E when the workspace is not ready or boots another image", () => { const unready = fixture({ ready: false }); const unreadyResult = run({ ...unready.env, BREV_READY_TIMEOUT_SECONDS: "1" }); expect(unreadyResult.status).not.toBe(0); @@ -320,7 +362,9 @@ describe("focused staging Brev Launchable lane", () => { expect(wrongImageResult.stderr).toContain("booted image does not match the producer handoff"); expect(fs.readFileSync(wrongImage.calls, "utf8")).not.toContain("full-e2e.test.ts"); expect(fs.existsSync(wrongImage.state)).toBe(false); + }); + it("blocks E2E when the booted runtime differs from the producer evidence", () => { for (const boot of [ fixture({ repoSha: "b".repeat(40) }), fixture({ provisionSha: "b".repeat(40) }), @@ -341,6 +385,29 @@ describe("focused staging Brev Launchable lane", () => { } }, 90_000); + it("rejects producer evidence that cannot identify one GCP image", () => { + for (const malformed of [ + fixture({ omitReceiptField: "imageId" }), + fixture({ imageId: "not-an-id" }), + fixture({ + imageProject: "other-project", + imageSelfLink: + "https://www.googleapis.com/compute/v1/projects/other-project/global/images/nemoclaw-test-image", + }), + fixture({ + imageSelfLink: + "https://www.googleapis.com/compute/v1/projects/brevdevprod/global/images/other", + }), + ]) { + const result = run(malformed.env); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("producer receipt does not match the candidate"); + expect(fs.readFileSync(malformed.calls, "utf8")).not.toMatch( + /brev create|full-e2e\.test\.ts/u, + ); + } + }); + it("reports E2E failure only after verified workspace cleanup", () => { const { env, state, workDir } = fixture({ e2eFails: true }); const result = run(env); @@ -407,6 +474,11 @@ describe("focused staging Brev Launchable lane", () => { ).toMatchObject({ candidateSha, boot: { bootImage: "projects/brevdevprod/global/images/nemoclaw-test-image" }, + image: { + imageId: "5831758261704356816", + imageSelfLink: + "https://www.googleapis.com/compute/v1/projects/brevdevprod/global/images/nemoclaw-test-image", + }, fullE2e: "pending", }); }); diff --git a/test/e2e/README.md b/test/e2e/README.md index 1397870f5e1..4ab8f451781 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -444,7 +444,10 @@ Launchable`. Each push run also selects this job as part of the complete main ru A manual run with `include_staging_brev_launchable=true` and empty `jobs` and `targets` selectors runs the default workflow E2E selection plus the Launchable E2E job. -This is the full run required for pre-tag evidence. Each full dispatch uses +This is full mode for an explicitly combined request. The pre-tag release ledger uses ordinary mode +with empty selectors and `include_staging_brev_launchable=false`; release image qualification +separately uses a historically validated Launchable result from a qualified ancestor or a fresh selective run. +Each full dispatch uses `github.run_id` in its workflow concurrency identity, so another full dispatch cannot supersede it while it waits. The trusted `main` workflow dispatch verifies that the dispatching and rerunning actors have repository `maintain` or diff --git a/test/maintainer-e2e-skill.test.ts b/test/maintainer-e2e-skill.test.ts index a4ae9584015..1dc5ef85bec 100644 --- a/test/maintainer-e2e-skill.test.ts +++ b/test/maintainer-e2e-skill.test.ts @@ -4,9 +4,13 @@ import fs from "node:fs"; import path from "node:path"; import { describe, expect, it } from "vitest"; -import { validateFullE2eEvidence } from "../.agents/skills/nemoclaw-maintainer-e2e/scripts/validate-full-e2e-evidence.mts"; +import { + validateBrevImageEvidence, + validateFullE2eEvidence, +} from "../.agents/skills/nemoclaw-maintainer-e2e/scripts/validate-full-e2e-evidence.mts"; const candidateSha = "a".repeat(40); +const imageRepositorySha = "d".repeat(40); const workflowSha = "b".repeat(40); function validEvidence() { @@ -35,6 +39,7 @@ function validEvidence() { jobs: { jobs: [ { + completed_at: "2026-07-24T11:30:00Z", conclusion: "success", html_url: "https://github.com/NVIDIA/NemoClaw/actions/runs/100/job/200", name: "Exact staging Brev Launchable", @@ -46,17 +51,33 @@ function validEvidence() { }, launchableE2e: { boot: { + bootImage: "projects/brevdevprod/global/images/nemoclaw-test-image", + imageRepositorySha, provisionSha: candidateSha, repoClean: true, repoSha: candidateSha, }, candidateSha, fullE2e: "passed", + image: { + imageCreationTimestamp: "2026-07-24T09:30:00Z", + imageId: "5831758261704356816", + imageName: "nemoclaw-test-image", + imageOriginWorkflowRunAttempt: 1, + imageOriginWorkflowRunId: "99", + imageRepositorySha, + imageSelfLink: + "https://www.googleapis.com/compute/v1/projects/brevdevprod/global/images/nemoclaw-test-image", + observedFamily: "nemoclaw-brev-staging-cpu", + project: "brevdevprod", + result: "built", + }, producer: { runId: "99", status: "success" }, workspace: { id: "workspace-123", name: "nclaw-e2e-100-2" }, }, run: { conclusion: "success", + created_at: "2026-07-24T10:00:00Z", event: "workflow_dispatch", head_branch: "main", head_sha: candidateSha, @@ -125,14 +146,31 @@ describe("nemoclaw-maintainer-e2e evidence validation", () => { emptySelectors: true, includeStagingBrevLaunchable: true, }, + evidenceMode: "full", + jobCompletedAt: "2026-07-24T11:30:00Z", jobUrl: "https://github.com/NVIDIA/NemoClaw/actions/runs/100/job/200", launchableE2e: { fullE2e: "passed", + image: { + imageCreationTimestamp: "2026-07-24T09:30:00Z", + imageId: "5831758261704356816", + imageName: "nemoclaw-test-image", + imageOriginWorkflowRunAttempt: 1, + imageOriginWorkflowRunId: "99", + imageRepositorySha, + imageSelfLink: + "https://www.googleapis.com/compute/v1/projects/brevdevprod/global/images/nemoclaw-test-image", + observedFamily: "nemoclaw-brev-staging-cpu", + project: "brevdevprod", + result: "built", + }, producerRunId: "99", provisionSha: candidateSha, repoClean: true, repoSha: candidateSha, }, + runCreatedAt: "2026-07-24T10:00:00Z", + runId: 100, runUrl: "https://github.com/NVIDIA/NemoClaw/actions/runs/100", }); }); @@ -152,6 +190,52 @@ describe("nemoclaw-maintainer-e2e evidence validation", () => { }); }); + it("accepts image evidence from direct main full, selective, and push runs (#7487)", () => { + const full = validDirectMainV2Evidence(); + expect(validateBrevImageEvidence(full, "full")).toMatchObject({ + evidenceMode: "full", + jobCompletedAt: "2026-07-24T11:30:00Z", + }); + + const launchable = validEvidence(); + launchable.dispatch.emptySelectors = false; + launchable.dispatch.includeStagingBrevLaunchable = false; + launchable.dispatch.jobs = "staging-brev-launchable"; + expect(validateBrevImageEvidence(launchable, "launchable")).toMatchObject({ + dispatch: { emptySelectors: false, includeStagingBrevLaunchable: false }, + evidenceMode: "launchable", + }); + + const push = validEvidence(); + push.dispatch.eventName = "push"; + push.dispatch.includeStagingBrevLaunchable = false; + push.run.event = "push"; + expect(validateBrevImageEvidence(push, "push")).toMatchObject({ + dispatch: { emptySelectors: true, includeStagingBrevLaunchable: false }, + evidenceMode: "push", + }); + + const reused = validEvidence(); + reused.launchableE2e.image.result = "reused"; + reused.launchableE2e.image.imageOriginWorkflowRunId = "77"; + reused.launchableE2e.image.imageOriginWorkflowRunAttempt = 3; + expect(validateBrevImageEvidence(reused, "full")).toMatchObject({ + launchableE2e: { + image: { + imageOriginWorkflowRunAttempt: 3, + imageOriginWorkflowRunId: "77", + result: "reused", + }, + }, + }); + }); + + it("rejects PR-shaped release image evidence (#7487)", () => { + expect(() => validateBrevImageEvidence(validV2Evidence(), "full")).toThrow( + "dispatch.prNumber must be null", + ); + }); + it.each([ ["candidateRepository", "contributor/NemoClaw", "dispatch.candidateRepository"], ["baseSha", "c".repeat(40), "dispatch.baseSha"], @@ -256,6 +340,70 @@ describe("nemoclaw-maintainer-e2e evidence validation", () => { }, "launchableE2e.boot.repoSha", ], + [ + "a boot URI that does not identify the producer image", + (evidence: ReturnType) => { + evidence.launchableE2e.boot.bootImage = "projects/brevdevprod/global/images/another-image"; + }, + "launchableE2e.boot.bootImage", + ], + [ + "an image ID that is not decimal", + (evidence: ReturnType) => { + evidence.launchableE2e.image.imageId = "not-an-id"; + }, + "launchableE2e.image.imageId", + ], + [ + "an image outside the trusted GCP project", + (evidence: ReturnType) => { + evidence.launchableE2e.image.project = "other-project"; + }, + "launchableE2e.image.project", + ], + [ + "a malformed image creation timestamp", + (evidence: ReturnType) => { + evidence.launchableE2e.image.imageCreationTimestamp = "not-a-timestamp"; + }, + "launchableE2e.image.imageCreationTimestamp", + ], + [ + "a built image whose origin does not match the producer", + (evidence: ReturnType) => { + evidence.launchableE2e.image.imageOriginWorkflowRunId = "98"; + }, + "launchableE2e.image.imageOriginWorkflowRunId", + ], + [ + "an image self-link that does not identify the producer image", + (evidence: ReturnType) => { + evidence.launchableE2e.image.imageSelfLink = + "https://www.googleapis.com/compute/v1/projects/brevdevprod/global/images/another-image"; + }, + "launchableE2e.image.imageSelfLink", + ], + [ + "an image-repository SHA that does not match the boot receipt", + (evidence: ReturnType) => { + evidence.launchableE2e.boot.imageRepositorySha = "e".repeat(40); + }, + "launchableE2e.boot.imageRepositorySha", + ], + [ + "a malformed workflow timestamp", + (evidence: ReturnType) => { + evidence.run.created_at = "not-a-timestamp"; + }, + "run.created_at", + ], + [ + "a malformed selected-job timestamp", + (evidence: ReturnType) => { + evidence.jobs.jobs[0]!.completed_at = "not-a-timestamp"; + }, + "Exact staging Brev Launchable.completed_at", + ], [ "a cleanup receipt without verified absence", (evidence: ReturnType) => { @@ -313,7 +461,9 @@ describe("nemoclaw-maintainer-e2e workflow routing", () => { expect(skill).toContain("run pre-tag full E2E"); expect(skill).toContain("run release-candidate E2E"); expect(skill).toContain("must not authorize the Brev Launchable path"); - expect(skill).toContain("Pre-tag evidence still requires the full `workflow_dispatch` mode"); + expect(skill).toContain( + "Pre-tag release-ledger evidence uses ordinary `workflow_dispatch` mode", + ); expect(skill).toContain( "an authorized environment reviewer must approve it before qualification starts", ); @@ -332,8 +482,25 @@ describe("nemoclaw-maintainer-e2e workflow routing", () => { expect(skill).toContain("cleanup.json"); expect(skill).toContain("dispatch.json"); expect(skill).toContain("validate-full-e2e-evidence.mts"); - expect(skill).toContain("provisional release evidence"); + expect(skill).toContain('>"$FULL_E2E_DIR/validated.json"'); + expect(skill).toContain( + "workflow run ID, creation timestamp, and selected job completion timestamp", + ); + expect(skill).toContain("exact GCP image and Launchable E2E identity"); + expect(skill).toMatch(/provisional\s+release-ledger evidence/u); expect(skill).toContain("If the release candidate SHA changes"); expect(skill).toContain("nemoclaw-maintainer-cut-release-tag"); }); + + it("selects only direct main, historically validated release image evidence (#7487)", () => { + expect(skill).toContain("## Select Release Image Evidence"); + expect(skill).toContain("trusted `main` push"); + expect(skill).toContain("jobs=staging-brev-launchable"); + expect(skill).toContain("include_staging_brev_launchable=true"); + expect(skill).toContain("rejects PR-shaped evidence"); + expect(skill).toContain("An itemized release-ledger exception is not image evidence"); + expect(skill).toContain("sorted by `jobCompletedAt` newest first"); + expect(skill).toContain("The selector contract is mode-specific"); + expect(skill).toContain("A Launchable-mode run is not release-ledger evidence"); + }); }); diff --git a/test/maintainer-skills-policy.test.ts b/test/maintainer-skills-policy.test.ts index c5025a937f3..45c677d3786 100644 --- a/test/maintainer-skills-policy.test.ts +++ b/test/maintainer-skills-policy.test.ts @@ -210,8 +210,8 @@ describe("maintainer skills follow canonical workflow policy", () => { ); expect(policy).toContain("itemized maintainer exception"); expect(policy).toContain("If the candidate SHA changes"); - expect(policy).toContain("This does not freeze `main` or prevent merges"); - expect(policy).toContain("Require one completed, successful full workflow run"); + expect(policy).toMatch(/This does not freeze `main` or\s+prevent merges/u); + expect(policy).toContain("Require one completed, successful ordinary workflow run"); expect(policy).toContain("discard the ledger and its exceptions"); expect(policy).toContain("selector inputs"); expect(release).toContain('"dispatchJson"'); @@ -222,8 +222,10 @@ describe("maintainer skills follow canonical workflow policy", () => { expect(release).toContain("actions/runs/$RUN_ID/artifacts"); expect(release).toContain("sort_by(.created_at)"); expect(release).not.toContain("RECEIPT_ATTEMPT"); - expect(release).toContain("rerun preflight and the full E2E workflow"); + expect(release).toContain("rerun preflight and ordinary E2E"); expect(release).toContain("Immediately before asking, refresh `origin/main` once"); + expect(release).toContain("Discard the prior image decision too"); + expect(release).toContain("obtain the maintainer's prior-or-fresh choice again"); const evidenceSummary = release.indexOf("Before showing the confirmation prompt"); const confirmationPrompt = release.indexOf( "Ask the maintainer to paste this phrase", @@ -234,9 +236,7 @@ describe("maintainer skills follow canonical workflow policy", () => { expect(evening).toContain( "Each missing or skipped execution in that successful run requires its own itemized maintainer exception", ); - expect(evening).toContain( - "Missing or invalid Launchable E2E evidence in that successful run requires a separate", - ); + expect(evening).toContain("An exception never qualifies an image"); expect(evening).toContain("Tag the confirmed release commit with `vX.Y.Z`"); expect(evening).not.toContain("tag `main`"); expect(dailyFlow).toContain("capture the candidate SHA and review every E2E test"); @@ -247,7 +247,7 @@ describe("maintainer skills follow canonical workflow policy", () => { expect(priorities).toContain("Record the release SHA and required E2E evidence"); }); - it("requires full-mode exact Brev Launchable evidence before release confirmation (#7487)", () => { + it("keeps the exact release ledger separate from qualified Brev image evidence (#7487)", () => { const e2e = read(".agents/skills/nemoclaw-maintainer-e2e/SKILL.md"); const evening = read(".agents/skills/nemoclaw-maintainer-evening/SKILL.md"); const release = read(".agents/skills/nemoclaw-maintainer-cut-release-tag/SKILL.md"); @@ -263,44 +263,70 @@ describe("maintainer skills follow canonical workflow policy", () => { expect(e2e).toContain("jobs?filter=all&per_page=100"); expect(e2e).toContain("Reuse `run-$RUN_ID.json` and `jobs-$RUN_ID.json`"); expect(release).toContain("reuse `run-$RUN_ID.json` and `jobs-$RUN_ID.json`"); - expect(release).toContain("load `nemoclaw-maintainer-e2e` and dispatch one full run"); + expect(release).toContain("load `nemoclaw-maintainer-e2e` and dispatch one ordinary run"); expect(release).toContain("Treat a skipped job as missing evidence"); - expect(release).toContain("include_staging_brev_launchable=true"); - expect(release).toContain("cleanup evidence that reports the qualified workspace as `ABSENT`"); + expect(release).toContain("include_staging_brev_launchable=false"); expect(release).toContain( "a separate itemized maintainer exception for each missing or skipped execution", ); - expect(release).toContain( - "a separate itemized maintainer exception for missing or invalid exact Brev Launchable E2E evidence", - ); - expect(release).toContain("when accepted full-mode exact Brev evidence exists"); + expect(release).toContain("the separate qualified image evidence's workflow URL"); + expect(release).toContain("Select this exact validated image as the proposed"); + expect(release).toContain("Dispatch another selective `Exact staging Brev Launchable` run"); + expect(release).toContain("runCreatedAt"); + expect(release).toContain("jobCompletedAt"); + expect(release).toContain("git rev-list --count"); + expect(release).toContain("git merge-base --is-ancestor"); + expect(release).toContain("number of commits between the validated image and release commit"); + expect(release).not.toContain('test "$VALIDATED_SHA" = "$RELEASE_SHA"'); + expect(release).toContain("A nonzero distance is allowed"); + expect(release).toContain("If no qualified ancestor image exists"); + expect(release).toContain("Do not read `VALIDATED_JSON` or calculate a"); + expect(release).toContain("rerun the ancestry"); + expect(release).toContain("present all replacement image fields and timestamps"); + expect(release).toContain("An itemized exception can"); + expect(release).toContain("it never qualifies an image"); + expect(release).toContain('export EVIDENCE_DIR="$RELEASE_DIR/e2e-evidence"'); + expect(release).toContain("Pass this exported absolute path into every"); + expect(release).toContain("Do not report the selected staging image as promoted"); + expect(release).toContain("This is an evidence-only selection"); + expect(release).toContain("Historical artifacts alone do not prove that the"); expect( - release.indexOf("load `nemoclaw-maintainer-e2e` and dispatch one full run"), + release.indexOf("load `nemoclaw-maintainer-e2e` and dispatch one ordinary run"), ).toBeLessThan(release.indexOf("Ask the maintainer to paste this phrase")); expect(evening).toContain("load `nemoclaw-maintainer-e2e`"); expect(evening).toContain( - "Run full mode unless one existing full run for the candidate SHA contains complete workflow E2E", + "Run ordinary mode unless one existing ordinary run for the candidate SHA contains complete default", ); expect(release).toContain( - "Run full mode unless one existing full run for the candidate SHA contains complete workflow E2E", + "Run ordinary mode for the exact-candidate release ledger when complete default E2E evidence", ); expect(policy).toContain("A failed workflow run cannot supply the release ledger"); expect(release).toContain("Reject a failed workflow run before presenting the ledger"); expect(evening).not.toContain("readiness variable"); - expect(policy).toContain("Require one completed, successful full workflow run"); + expect(policy).toContain("Require one completed, successful ordinary workflow run"); expect(policy).toContain( - "Run `nemoclaw-maintainer-e2e` in full mode when the ledger lacks complete evidence", + "Run `nemoclaw-maintainer-e2e` in ordinary mode when the ledger lacks complete evidence", ); - expect(policy).toContain("including `Exact staging Brev Launchable`"); - expect(policy).toContain("cleanup receipt"); + expect(policy).toContain("Qualify the release image separately"); + expect(policy).toContain("complete Launchable E2E and cleanup evidence"); expect(policy).toContain("trusted dispatch receipt"); expect(policy).toContain( "Each missing or skipped execution in the accepted successful workflow run", ); + expect(policy).toContain("workflow creation time"); + expect(policy).toMatch(/commit\s+distance/u); expect(policy).toContain( - "Missing or invalid exact Brev Launchable E2E evidence in the accepted successful workflow run", + "newest historically validated image whose candidate SHA is an ancestor", + ); + expect(policy).toContain("selected job completion time"); + expect(policy).toContain("An itemized exception never qualifies an image"); + expect(policy).toMatch( + /select that image as the\s+proposed exact-image handoff or run the selective Launchable test again/u, ); - expect(policy).toContain("No release-note-only delta exception is currently defined"); + expect(policy).toMatch(/it does not by\s+itself promote a GCP image family/u); + expect(policy).toContain("current `lkg` path does not consume it and still rebuilds"); + expect(policy).toContain("Discard the prior image decision"); + expect(policy).toMatch(/No release-note-only delta exception is\s+currently defined/u); expect(skillsGuide).toContain("`nemoclaw-maintainer-e2e`"); }); diff --git a/test/release-e2e-evidence.test.ts b/test/release-e2e-evidence.test.ts index 203fe95bd93..c4886a8a663 100644 --- a/test/release-e2e-evidence.test.ts +++ b/test/release-e2e-evidence.test.ts @@ -37,7 +37,7 @@ function runEvidence( ): ReleaseE2eRunEvidence { const attempt = options.attempt ?? 1; const runId = 1001; - const receiptVersion = options.receiptVersion ?? 1; + const receiptVersion = options.receiptVersion ?? 2; const executions = plan.executions.filter( (execution) => execution.group === group && (options.only?.(execution) ?? true), ); @@ -49,11 +49,11 @@ function runEvidence( allowJetsonRunnerQueue: false, ...(receiptVersion === 2 ? { - baseSha: "c".repeat(40), + baseSha: candidateSha, candidateRepository: "NVIDIA/NemoClaw", - prNumber: 8583, + prNumber: null, repository: "NVIDIA/NemoClaw", - workflowSha, + workflowSha: candidateSha, } : {}), candidateSha, @@ -84,7 +84,7 @@ function runEvidence( status: "completed", conclusion: "success", run_attempt: attempt, - head_sha: options.sha ?? (receiptVersion === 2 ? workflowSha : candidateSha), + head_sha: options.sha ?? candidateSha, html_url: `https://github.com/NVIDIA/NemoClaw/actions/runs/${runId}`, }, }; @@ -105,14 +105,28 @@ function directMainV2Evidence( return evidence; } +function prV2Evidence( + plan: ReleaseE2ePreflight, + group: ReleaseE2eExecution["group"], +): ReleaseE2eRunEvidence { + const evidence = directMainV2Evidence(plan, group); + Object.assign(evidence.dispatch as Record, { + baseSha: "c".repeat(40), + prNumber: 8583, + workflowSha, + }); + (evidence.run as Record).head_sha = workflowSha; + return evidence; +} + describe("release E2E evidence", () => { it("derives one complete release E2E run from the workflow", () => { const plan = preflight(); expect(plan.dispatches.completeRun).toEqual({ - includeStagingBrevLaunchable: true, + includeStagingBrevLaunchable: false, jobs: "", - mode: "full", + mode: "ordinary", targets: "", }); expect(plan.launchableE2eJobId).toBe("staging-brev-launchable"); @@ -126,17 +140,15 @@ describe("release E2E evidence", () => { ); }); - it("accepts v2 evidence bound to the candidate and trusted workflow SHAs", () => { + it("rejects PR-shaped v2 evidence from the release ledger", () => { const plan = preflight(); - const ledger = buildReleaseE2eLedger(plan, [ - runEvidence(plan, "default", { receiptVersion: 2 }), - ]); - expect(ledger.successfulCount).toBe(ledger.requiredCount); - expect(ledger.missingCount).toBe(0); + expect(() => buildReleaseE2eLedger(plan, [prV2Evidence(plan, "default")])).toThrow( + "runs[0].dispatch.prNumber must equal null", + ); }); - it("accepts direct-main v2 evidence with identical repository and SHA identities", () => { + it("accepts direct main v2 evidence with identical repository and SHA identities", () => { const plan = preflight(); const ledger = buildReleaseE2eLedger(plan, [directMainV2Evidence(plan, "default")]); @@ -148,7 +160,7 @@ describe("release E2E evidence", () => { ["candidateRepository", "contributor/NemoClaw", "runs[0].dispatch.candidateRepository"], ["baseSha", "c".repeat(40), "runs[0].dispatch.baseSha"], ["workflowSha", workflowSha, "runs[0].dispatch.workflowSha"], - ])("rejects direct-main v2 evidence with a mismatched %s identity", (field, value, message) => { + ])("rejects direct main v2 evidence with a mismatched %s identity", (field, value, message) => { const plan = preflight(); const evidence = directMainV2Evidence(plan, "default"); (evidence.dispatch as Record)[field] = value; @@ -165,20 +177,20 @@ describe("release E2E evidence", () => { ["workflowSha", "short", "runs[0].dispatch.workflowSha"], ])("rejects v2 evidence with a mismatched %s", (field, value, message) => { const plan = preflight(); - const evidence = runEvidence(plan, "default", { receiptVersion: 2 }); + const evidence = directMainV2Evidence(plan, "default"); (evidence.dispatch as Record)[field] = value; expect(() => buildReleaseE2eLedger(plan, [evidence])).toThrow(message); }); - it("rejects a v2 run whose head is the candidate instead of the trusted workflow", () => { + it("rejects a direct main v2 run whose head differs from the candidate", () => { const plan = preflight(); + const evidence = directMainV2Evidence(plan, "default"); + (evidence.run as Record).head_sha = workflowSha; - expect(() => - buildReleaseE2eLedger(plan, [ - runEvidence(plan, "default", { receiptVersion: 2, sha: candidateSha }), - ]), - ).toThrow("runs[0].run.head_sha must equal"); + expect(() => buildReleaseE2eLedger(plan, [evidence])).toThrow( + "runs[0].run.head_sha must equal", + ); }); it("rejects a missing activation path for a default E2E", () => { @@ -265,42 +277,34 @@ describe("release E2E evidence", () => { ).toThrow("release E2E evidence requires exactly one workflow run, received 2"); }); - it("requires the full run to include staging Brev Launchable", () => { + it("keeps image-building Launchable work out of the release ledger run", () => { const plan = preflight(); const evidence = runEvidence(plan, "default"); - (evidence.dispatch as Record).includeStagingBrevLaunchable = false; + (evidence.dispatch as Record).includeStagingBrevLaunchable = true; expect(() => buildReleaseE2eLedger(plan, [evidence])).toThrow( - "runs[0].dispatch.includeStagingBrevLaunchable must equal true", + "runs[0].dispatch.includeStagingBrevLaunchable must equal false", ); }); it.each([ - [2 as const, "allowJetsonDispatch", "runs[0].dispatch.allowJetsonDispatch must equal false"], - [1 as const, "allowJetsonDispatch", "runs[0].dispatch.allowJetsonDispatch must equal false"], - [ - 1 as const, - "allowJetsonRunnerQueue", - "runs[0].dispatch.allowJetsonRunnerQueue must equal false", - ], - [ - 1 as const, - "allowDgxSparkRunnerQueue", - "runs[0].dispatch.allowDgxSparkRunnerQueue must equal false", - ], - ])("rejects v%s release evidence that opts into %s", (receiptVersion, field, message) => { + ["allowJetsonDispatch", "runs[0].dispatch.allowJetsonDispatch must equal false"], + ["allowJetsonRunnerQueue", "runs[0].dispatch.allowJetsonRunnerQueue must equal false"], + ["allowDgxSparkRunnerQueue", "runs[0].dispatch.allowDgxSparkRunnerQueue must equal false"], + ])("rejects v2 release evidence that opts into %s", (field, message) => { const plan = preflight(); - const evidence = runEvidence(plan, "default", { receiptVersion }); + const evidence = directMainV2Evidence(plan, "default"); (evidence.dispatch as Record)[field] = true; expect(() => buildReleaseE2eLedger(plan, [evidence])).toThrow(message); }); - it("keeps allowJetsonDispatch optional for v1 release evidence", () => { + it("rejects v1 release-ledger evidence", () => { const plan = preflight(); const evidence = runEvidence(plan, "default", { receiptVersion: 1 }); - delete (evidence.dispatch as Record).allowJetsonDispatch; - expect(buildReleaseE2eLedger(plan, [evidence]).missingCount).toBe(0); + expect(() => buildReleaseE2eLedger(plan, [evidence])).toThrow( + 'runs[0].dispatch.kind must equal "nemoclaw-e2e-dispatch-v2"', + ); }); it("reports a failed matrix row without collapsing its successful siblings", () => { diff --git a/tools/e2e/brev-launchable-e2e.sh b/tools/e2e/brev-launchable-e2e.sh index 96b3f505d36..02fcf80add1 100755 --- a/tools/e2e/brev-launchable-e2e.sh +++ b/tools/e2e/brev-launchable-e2e.sh @@ -166,17 +166,31 @@ jq -e --arg sha "$CANDIDATE_SHA" --arg correlation "$CORRELATION_ID" \ --arg requester "$GITHUB_RUN_ID" --argjson attempt "$GITHUB_RUN_ATTEMPT" --arg run "$producer_run" ' .kind == "nemoclaw-exact-image-manifest" and .nemoclawSha == $sha and .correlationId == $correlation and .requesterWorkflowRunId == $requester and - .requesterWorkflowRunAttempt == $attempt and .imageRepository == "brevdev/nemoclaw-image" and + .requesterWorkflowRunAttempt == $attempt and .requesterRepository == "NVIDIA/NemoClaw" and + .imageRepository == "brevdev/nemoclaw-image" and .producerWorkflow == ".github/workflows/build-launchable-e2e-image.yml" and .workflowRunId == $run and .workflowRunAttempt == 1 and .status == "READY" and - .channel == "staging" and .variant == "cpu" and + .channel == "staging" and .variant == "cpu" and .imageKind == "compute#image" and .observedFamily == "nemoclaw-brev-staging-cpu" and - (.project | type) == "string" and (.project | length) > 0 and - (.imageName | type) == "string" and (.imageName | length) > 0 and + .project == "brevdevprod" and + (.imageName | test("^[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?$")) and + (.imageId | test("^[1-9][0-9]*$")) and + .imageSelfLink == ("https://www.googleapis.com/compute/v1/projects/" + .project + "/global/images/" + .imageName) and + (.imageCreationTimestamp | type == "string" and + test("^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\\.[0-9]+)?(Z|[+-][0-9]{2}:[0-9]{2})$")) and + (.imageOriginWorkflowRunId | test("^[1-9][0-9]*$")) and + (.imageOriginWorkflowRunAttempt | type == "number" and . >= 1 and floor == .) and + (.result == "built" or .result == "reused") and + (if .result == "built" then + .imageOriginWorkflowRunId == $run and .imageOriginWorkflowRunAttempt == 1 + else true end) and (.imageRepositorySha | test("^[0-9a-f]{40}$"))' \ "$manifest" >/dev/null || die "producer receipt does not match the candidate" expected_boot_image="projects/$(jq -er .project "$manifest")/global/images/$(jq -er .imageName "$manifest")" image_repository_sha="$(jq -er .imageRepositorySha "$manifest")" +image="$(jq -c \ + '{project,imageName,imageId,imageSelfLink,imageCreationTimestamp,imageRepositorySha, + imageOriginWorkflowRunId,imageOriginWorkflowRunAttempt,observedFamily,result}' "$manifest")" rm -rf "$WORK_DIR/handoff" # The standing Launchable resolves the staging family. Give that reference time to @@ -217,9 +231,9 @@ boot_image="$(timeout 300s brev exec "$INSTANCE_NAME" 'set -euo pipefail | sed -n 's/^NEMOCLAW_BOOT_IMAGE=//p' | tail -n 1)" [ -n "$boot_image" ] || die "booted image identity is missing" -jq -n --arg candidateSha "$CANDIDATE_SHA" --arg producerRun "$producer_run" \ +jq -n --arg candidateSha "$CANDIDATE_SHA" --arg producerRun "$producer_run" --argjson image "$image" \ --arg bootImage "$boot_image" --arg workspaceName "$INSTANCE_NAME" --arg workspaceId "$workspace_id" \ - '{candidateSha:$candidateSha,producer:{runId:$producerRun,status:"success"},boot:{bootImage:$bootImage},workspace:{name:$workspaceName,id:$workspaceId},fullE2e:"pending"}' \ + '{candidateSha:$candidateSha,producer:{runId:$producerRun,status:"success"},image:$image,boot:{bootImage:$bootImage},workspace:{name:$workspaceName,id:$workspaceId},fullE2e:"pending"}' \ >"$WORK_DIR/launchable-e2e.json" [ "$boot_image" = "$expected_boot_image" ] \ diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index 12d565143e7..7a5e8feaad0 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -4211,6 +4211,12 @@ function validateStagingBrevLaunchableJob(errors: string[], jobs: WorkflowRecord const dispatchIdentity = requireStep(errors, steps, "Record E2E dispatch identity"); const dispatchEnv = asRecord(dispatchIdentity?.env); for (const [key, expected] of [ + [ + "ALLOW_DGX_SPARK_RUNNER_QUEUE", + "${{ inputs.allow_dgx_spark_runner_queue && 'true' || 'false' }}", + ], + ["ALLOW_JETSON_DISPATCH", "${{ inputs.allow_jetson_dispatch && 'true' || 'false' }}"], + ["ALLOW_JETSON_RUNNER_QUEUE", "false"], ["CANDIDATE_SHA", "${{ env.CANDIDATE_SHA }}"], ["DISPATCH_JOBS", "${{ inputs.jobs }}"], ["DISPATCH_TARGETS", "${{ inputs.targets }}"], @@ -4233,6 +4239,9 @@ function validateStagingBrevLaunchableJob(errors: string[], jobs: WorkflowRecord "eventName: $eventName", "workflowRunId: $workflowRunId", "workflowRunAttempt: $workflowRunAttempt", + "allowDgxSparkRunnerQueue: $allowDgxSparkRunnerQueue", + "allowJetsonDispatch: $allowJetsonDispatch", + "allowJetsonRunnerQueue: $allowJetsonRunnerQueue", "jobs: $jobs", "targets: $targets", "includeStagingBrevLaunchable: $includeStagingBrevLaunchable",