fix(e2e): reuse CLI artifact for failed-job retries

.github/actions/restore-e2e-cli-artifact/action.yaml

@@ -58,8 +58,12 @@ runs:
           { echo "::error::consumer checkout does not match the producer candidate SHA"; exit 1; }
         [[ "$workflow_sha" == "$CALLER_WORKFLOW_SHA" ]] ||
           { echo "::error::consumer and producer workflow SHAs differ"; exit 1; }
-        [[ "$run_id" == "$GITHUB_RUN_ID" && "$run_attempt" == "$GITHUB_RUN_ATTEMPT" ]] ||
-          { echo "::error::consumer and producer workflow run identities differ"; exit 1; }
+        [[ "$GITHUB_RUN_ATTEMPT" =~ ^[1-9][0-9]*$ ]] ||
+          { echo "::error::consumer workflow run attempt is invalid"; exit 1; }
+        [[ "$run_id" == "$GITHUB_RUN_ID" ]] ||
+          { echo "::error::consumer and producer workflow run IDs differ"; exit 1; }
+        (( run_attempt <= GITHUB_RUN_ATTEMPT )) ||
+          { echo "::error::producer workflow attempt is newer than the consumer attempt"; exit 1; }
 
         remote_url="$(git remote get-url origin)"
         case "$remote_url" in
@@ -78,10 +82,12 @@ runs:
           "candidate_repository=" + .candidateRepository,
           "candidate_sha=" + .candidateSha,
           "payload_sha256=" + .payloadSha256,
+          "producer_run_attempt=" + .runAttempt,
           "run_attempt=" + .runAttempt,
           "run_id=" + .runId,
           "workflow_sha=" + .workflowSha
         ' <<<"$PROVENANCE_JSON" >>"$GITHUB_OUTPUT"
+        printf 'consumer_run_attempt=%s\n' "$GITHUB_RUN_ATTEMPT" >>"$GITHUB_OUTPUT"
 
     - name: Download exact-commit CLI artifact
       uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
@@ -96,7 +102,7 @@ runs:
         CANDIDATE_REPOSITORY: ${{ steps.identity.outputs.candidate_repository }}
         CANDIDATE_SHA: ${{ steps.identity.outputs.candidate_sha }}
         PAYLOAD_SHA256: ${{ steps.identity.outputs.payload_sha256 }}
-        RUN_ATTEMPT: ${{ steps.identity.outputs.run_attempt }}
+        PRODUCER_RUN_ATTEMPT: ${{ steps.identity.outputs.producer_run_attempt }}
         RUN_ID: ${{ steps.identity.outputs.run_id }}
         WORKFLOW_SHA: ${{ steps.identity.outputs.workflow_sha }}
       shell: bash
@@ -117,7 +123,7 @@ runs:
           --arg candidateSha "$CANDIDATE_SHA" \
           --arg lockfileSha256 "$lockfile_sha256" \
           --arg payloadSha256 "$PAYLOAD_SHA256" \
-          --arg runAttempt "$RUN_ATTEMPT" \
+          --arg runAttempt "$PRODUCER_RUN_ATTEMPT" \
           --arg runId "$RUN_ID" \
           --arg sourceTree "$source_tree" \
           --arg workflowSha "$WORKFLOW_SHA" \

test/e2e/README.md

@@ -455,9 +455,13 @@ set, incomplete listing, custom or self-hosted label, changed evidence, or
 ambiguous pagination prevents recovery.
 
 For eligible `E2E main` push runs, `E2E / Main Retry` asks GitHub Actions to
-rerun failed jobs after a workflow failure. It can request two retries. The
-controller does not verify that GitHub schedules a different runner, so do not
-treat a retry as evidence of a fresh host. It ignores manual runs and source runs
+rerun failed jobs and their dependent jobs. A successful CLI artifact producer is
+not rerun. Consumers can reuse its immutable, content-addressed artifact from an
+earlier attempt of the same workflow run. Restore validation requires the same
+run, workflow, candidate, immutable artifact ID, and payload digest, and rejects a
+producer attempt newer than the consumer attempt. The controller can request two
+reruns. It does not verify that GitHub schedules a different runner, so do not
+treat a rerun as evidence of a fresh host. It ignores manual runs and source runs
 superseded by a newer `main` push. The controller checks out only trusted
 default-branch code and receives no repository secrets.
 
@@ -727,8 +731,9 @@ them in their issuing services. If cleanup fails, remove the recorded Brev
 workspace. Rotate or revoke each credential to remove later access.
 
 When an eligible `E2E main` push workflow concludes with `failure`,
-`E2E / Main Retry` asks GitHub Actions to rerun the failed jobs. The controller
-permits two retries but does not verify that GitHub schedules a different runner.
+`E2E / Main Retry` asks GitHub Actions to rerun failed jobs and their dependents.
+The controller permits two reruns but does not verify that GitHub schedules a
+different runner.
 After evaluation succeeds, it uploads an artifact named for the current attempt.
 The artifact contains one `attempts` summary for each source attempt through the
 current attempt. The `totalRunnerMinutes` field contains the cumulative runner

test/e2e/docs/README.md

@@ -136,11 +136,15 @@ The aggregate local command rebuilds the CLI before Vitest starts and runs E2E
 test files serially. It does not retry a failed test.
 
 After an eligible `E2E main` push workflow fails, `E2E / Main Retry` asks
-GitHub Actions to rerun its failed jobs. It can request two reruns, for three total
-attempts. The controller does not verify that GitHub schedules a different runner,
-so do not treat a retry as evidence of a fresh host. If a later attempt succeeds,
-the source workflow concludes with `success`. The evidence sets `action` to
-`passed-after-retry` and `flaky` to `true`.
+GitHub Actions to rerun failed jobs and their dependents. A successful CLI
+artifact producer is not rerun. Consumers reuse its immutable, content-addressed
+artifact from the earlier producer attempt in the same workflow run. Restore
+validation rejects another run, workflow, candidate, artifact, payload, or a
+producer attempt newer than the consumer attempt. The controller can request two
+reruns, for three total attempts. It does not verify that GitHub schedules a
+different runner, so do not treat a rerun as evidence of a fresh host. If a later
+attempt succeeds, the source workflow concludes with `success`. The evidence sets
+`action` to `passed-after-retry` and `flaky` to `true`.
 
 After the controller evaluates attempt N, it uploads an artifact named for that
 attempt. The artifact contains one `attempts` entry for each source attempt through

test/e2e/support/cli-artifact-workflow-boundary.test.ts

@@ -29,7 +29,7 @@ const PAYLOAD_SHA256 = "b".repeat(64);
 const CONTENT_ADDRESSED_ARTIFACT_NAME = `artifact_name="nemoclaw-cli-\${CANDIDATE_SHA}-\${payload_sha256}"`;
 const UNBOUND_ARTIFACT_NAME = `artifact_name="nemoclaw-cli-\${CANDIDATE_SHA}"`;
 
-function runIdentityValidation(overrides: Record<string, unknown> = {}) {
+function runIdentityValidation(overrides: Record<string, unknown> = {}, consumerAttempt = "1") {
   const action = readYaml<CompositeAction>(".github/actions/restore-e2e-cli-artifact/action.yaml");
   const workflowSha = "d".repeat(40);
   const outputDirectory = fs.mkdtempSync(path.join(os.tmpdir(), "cli-artifact-identity-"));
@@ -40,7 +40,7 @@ function runIdentityValidation(overrides: Record<string, unknown> = {}) {
         ...process.env,
         CALLER_WORKFLOW_SHA: workflowSha,
         GITHUB_OUTPUT: path.join(outputDirectory, "github-output"),
-        GITHUB_RUN_ATTEMPT: "1",
+        GITHUB_RUN_ATTEMPT: consumerAttempt,
         GITHUB_RUN_ID: "98765",
         PROVENANCE_JSON: JSON.stringify({
           kind: "nemoclaw-e2e-cli-provenance-v1",
@@ -346,7 +346,7 @@ function runRestoreValidation(options: RestoreFixtureOptions = {}) {
       GITHUB_WORKSPACE: workspace,
       PATH: `${toolDirectory}:${process.env.PATH ?? ""}`,
       PAYLOAD_SHA256: expectedPayloadSha256,
-      RUN_ATTEMPT: "1",
+      PRODUCER_RUN_ATTEMPT: "1",
       RUN_ID: "98765",
       RUNNER_TEMP: runnerTemp,
       WORKFLOW_SHA: workflowSha,
@@ -407,6 +407,19 @@ describe("exact-commit CLI artifact workflow boundary", () => {
     expect(result.status, result.stderr).toBe(0);
   });
 
+  it("reuses an immutable producer artifact during a later failed-job rerun", () => {
+    const result = runIdentityValidation({ runAttempt: "1" }, "2");
+    expect(result.status, result.stderr).toBe(0);
+  });
+
+  it("rejects an invalid consumer attempt", () => {
+    const result = runIdentityValidation({}, "0");
+    expect(result.status, `${result.stdout}${result.stderr}`).not.toBe(0);
+    expect(`${result.stdout}${result.stderr}`).toContain(
+      "consumer workflow run attempt is invalid",
+    );
+  });
+
   it.each([
     ["empty artifact ID", { artifactId: "" }, "producer CLI artifact provenance is invalid"],
     [
@@ -457,8 +470,12 @@ describe("exact-commit CLI artifact workflow boundary", () => {
       "consumer checkout repository does not match producer provenance",
     ],
     ["workflow SHA", { workflowSha: "e".repeat(40) }, "consumer and producer workflow SHAs differ"],
-    ["run ID", { runId: "98766" }, "consumer and producer workflow run identities differ"],
-    ["run attempt", { runAttempt: "2" }, "consumer and producer workflow run identities differ"],
+    ["run ID", { runId: "98766" }, "consumer and producer workflow run IDs differ"],
+    [
+      "future producer attempt",
+      { runAttempt: "2" },
+      "producer workflow attempt is newer than the consumer attempt",
+    ],
   ])("rejects a mismatched %s before artifact download", (_case, overrides, expectedError) => {
     const result = runIdentityValidation(overrides);
     expect(result.status, `${result.stdout}${result.stderr}`).not.toBe(0);

test/e2e/support/main-run-retry.test.ts

@@ -92,7 +92,7 @@ async function evaluate(options: { attempt: number; conclusion: string; latestRu
 }
 
 describe("main E2E retry controller", () => {
-  it.each([1, 2])("requests retry %s before the attempt limit", async (attempt) => {
+  it.each([1, 2])("requests failed-job rerun after attempt %s fails", async (attempt) => {
     const { evidence, requests } = await evaluate({ attempt, conclusion: "failure" });
 
     expect(E2E_MAX_RETRIES).toBe(2);
@@ -106,6 +106,11 @@ describe("main E2E retry controller", () => {
       method: "POST",
       path: `repos/${REPOSITORY}/actions/runs/${RUN_ID}/rerun-failed-jobs`,
     });
+    expect(
+      requests.some(
+        (request) => request.path === `repos/${REPOSITORY}/actions/runs/${RUN_ID}/rerun`,
+      ),
+    ).toBe(false);
   });
 
   it("stops after the third failed attempt", async () => {

tools/e2e/cli-artifact-workflow-boundary.mts

@@ -33,7 +33,7 @@ const DEFAULT_RESTORE_ACTION_PATH = join(
   "action.yaml",
 );
 const RESTORE_ACTION_CONTENT_SHA256 =
-  "f90d4532a64473817b183856e225b401b201d71aaafa75bd7cf8d63a3deeb76f";
+  "3a81ad631b839aa938eaaf1ad6777bab247204bf86fbca3c43c326a44dfb9c6c";
 const CLI_ARTIFACT_DOWNLOAD_STEP = "Download exact-commit CLI artifact";
 const CLI_ARTIFACT_VERIFY_STEP = "Verify and restore exact-commit CLI artifact";
 const CLI_ARTIFACT_PROVENANCE_STEP = "Record CLI artifact provenance";
@@ -217,7 +217,9 @@ export function validateCliArtifactRestoreAction(
     '.artifactName == ("nemoclaw-cli-" + .candidateSha + "-" + .payloadSha256)',
     'git rev-parse --verify HEAD)" == "$candidate_sha"',
     '[[ "$workflow_sha" == "$CALLER_WORKFLOW_SHA" ]]',
-    '[[ "$run_id" == "$GITHUB_RUN_ID" && "$run_attempt" == "$GITHUB_RUN_ATTEMPT" ]]',
+    '[[ "$GITHUB_RUN_ATTEMPT" =~ ^[1-9][0-9]*$ ]]',
+    '[[ "$run_id" == "$GITHUB_RUN_ID" ]]',
+    "(( run_attempt <= GITHUB_RUN_ATTEMPT ))",
     '[[ "$remote_repository" == "$candidate_repository" ]]',
     '<<<"$PROVENANCE_JSON" >>"$GITHUB_OUTPUT"',
   ]);
@@ -241,7 +243,7 @@ export function validateCliArtifactRestoreAction(
       CANDIDATE_REPOSITORY: "${{ steps.identity.outputs.candidate_repository }}",
       CANDIDATE_SHA: "${{ steps.identity.outputs.candidate_sha }}",
       PAYLOAD_SHA256: "${{ steps.identity.outputs.payload_sha256 }}",
-      RUN_ATTEMPT: "${{ steps.identity.outputs.run_attempt }}",
+      PRODUCER_RUN_ATTEMPT: "${{ steps.identity.outputs.producer_run_attempt }}",
       RUN_ID: "${{ steps.identity.outputs.run_id }}",
       WORKFLOW_SHA: "${{ steps.identity.outputs.workflow_sha }}",
     })