ci(e2e): retry final-main hosted runner loss

.github/workflows/e2e-main-runner-loss-retry.yaml

@@ -0,0 +1,42 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+name: E2E Main Runner Loss Retry
+run-name: "E2E main runner-loss retry for run ${{ github.event.workflow_run.id }} attempt ${{ github.event.workflow_run.run_attempt }}"
+
+on:
+  workflow_run:
+    workflows: [E2E]
+    types: [completed]
+
+permissions: {}
+
+jobs:
+  classify-and-retry:
+    if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.event.workflow_run.event != 'pull_request' && (github.event.workflow_run.event == 'schedule' || github.event.workflow_run.event == 'workflow_dispatch') && github.event.workflow_run.path == '.github/workflows/e2e.yaml' && github.event.workflow_run.display_title == 'E2E main' && github.event.workflow_run.head_branch == 'main' && github.event.workflow_run.head_repository.full_name == github.repository && github.event.workflow_run.conclusion == 'failure' && github.event.workflow_run.run_attempt == 1 }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      actions: write
+      contents: read
+    concurrency:
+      group: e2e-main-runner-loss-retry-${{ github.event.workflow_run.id }}
+      cancel-in-progress: false
+    steps:
+      - name: Checkout trusted retry controller
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          ref: ${{ github.workflow_sha }}
+          persist-credentials: false
+
+      - name: Setup Node
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0
+        with:
+          node-version: "22"
+
+      - name: Classify runner loss and retry failed jobs once
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+          SUBJECT_RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }}
+          SUBJECT_RUN_ID: ${{ github.event.workflow_run.id }}
+        run: node --experimental-strip-types tools/e2e/main-e2e-runner-loss-retry.mts

test/e2e/README.md

@@ -65,6 +65,17 @@ graph as the live targets:
 - `scorecard` writes the scheduled/manual result summary, compares the trusted
   cloud-onboard timing summary with the latest prior-release `e2e.yaml` run,
   and posts to the daily or full-run Slack route.
+- `E2E Main Runner Loss Retry` authenticates a failed scheduled or full manual
+  `E2E main` attempt after it completes. On attempt 1 only, it reruns failed
+  jobs when every failed job has the GitHub-hosted stranded-step signature and
+  the complete job listing contains no ordinary failure. Assertions, mixed or
+  ambiguous failures, cancelled runs, selective PR dispatches, and later
+  attempts receive no automatic retry. The retry stays on the same workflow
+  run and tested SHA. Validated attempt links remain in the job log to preserve
+  the original evidence, while the job summary stays free of network-derived
+  content. GitHub owns the hosted-runner state, so NemoClaw cannot repair the
+  source of this signature. Remove the retry only after GitHub stops producing
+  it, and keep the classifier fail-closed for replacement terminal evidence.
 - Selective dispatches remain silent unless they run on `main` with
   `post_to_slack=true`, which uses the preview Slack route. Branch-dispatched
   runs never receive Slack webhook secrets.

test/e2e/support/hosted-runner-loss.test.ts

@@ -0,0 +1,135 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { describe, expect, it } from "vitest";
+
+import {
+  hasTrustedHostedRunnerLossMarker,
+  validateWorkflowJobsPage,
+  verifiedRunnerLossEvidence,
+  type WorkflowJob,
+} from "../../../tools/e2e/hosted-runner-loss.mts";
+import { detectRunnerLoss } from "../../../tools/e2e/runner-pressure-core.mts";
+
+function lostRunnerJob(overrides: Partial<WorkflowJob> = {}): WorkflowJob {
+  return {
+    id: 7,
+    name: "rebuild-hermes",
+    status: "completed",
+    conclusion: "failure",
+    runnerId: 9,
+    runnerName: "GitHub Actions 42",
+    labels: ["ubuntu-latest"],
+    steps: [
+      { name: "Prepare", status: "completed", conclusion: "success" },
+      { name: "Run live test", status: "in_progress", conclusion: null },
+      { name: "Upload artifacts", status: "pending", conclusion: null },
+    ],
+    ...overrides,
+  };
+}
+
+describe("hosted-runner-loss workflow evidence (#7146)", () => {
+  it("normalizes the exact GitHub job fields consumed by the classifier", () => {
+    expect(
+      validateWorkflowJobsPage({
+        total_count: 1,
+        jobs: [
+          {
+            id: 7,
+            name: "rebuild-hermes",
+            status: "completed",
+            conclusion: "failure",
+            runner_id: 9,
+            runner_name: "GitHub Actions 42",
+            labels: ["ubuntu-latest"],
+            steps: [{ name: "Run live test", status: "in_progress", conclusion: null }],
+          },
+        ],
+      }),
+    ).toEqual({ totalCount: 1, jobs: [lostRunnerJob({ steps: [lostRunnerJob().steps[1]!] })] });
+  });
+
+  it("accepts one stranded active step only on a GitHub-hosted runner", () => {
+    expect(hasTrustedHostedRunnerLossMarker(lostRunnerJob())).toBe(true);
+    expect(
+      hasTrustedHostedRunnerLossMarker(
+        lostRunnerJob({ labels: ["self-hosted", "linux"], runnerName: "gpu-runner" }),
+      ),
+    ).toBe(false);
+    expect(
+      hasTrustedHostedRunnerLossMarker(
+        lostRunnerJob({
+          steps: [
+            { name: "First", status: "in_progress", conclusion: null },
+            { name: "Second", status: "in_progress", conclusion: null },
+          ],
+        }),
+      ),
+    ).toBe(false);
+  });
+
+  it("rejects an ordinary failed step even when hosted-runner metadata is present", () => {
+    expect(
+      hasTrustedHostedRunnerLossMarker(
+        lostRunnerJob({
+          steps: [{ name: "Assertion", status: "completed", conclusion: "failure" }],
+        }),
+      ),
+    ).toBe(false);
+  });
+
+  it("fails closed when runner loss and an ordinary failure coexist", () => {
+    const evidence = verifiedRunnerLossEvidence({
+      workflowConclusion: "failure",
+      jobs: [
+        lostRunnerJob(),
+        lostRunnerJob({
+          id: 8,
+          name: "full-e2e",
+          steps: [{ name: "Assertion", status: "completed", conclusion: "failure" }],
+        }),
+      ],
+      jobDetailsAvailable: true,
+      jobDetailsComplete: true,
+    });
+
+    expect(evidence).toEqual({
+      jobConclusion: "failure",
+      runnerLostMarkerCount: 1,
+      terminalClassificationPresent: true,
+    });
+    expect(detectRunnerLoss(evidence!)).toBe(false);
+  });
+
+  it("treats a timed-out job beside runner loss as a mixed failure", () => {
+    const evidence = verifiedRunnerLossEvidence({
+      workflowConclusion: "failure",
+      jobs: [
+        lostRunnerJob(),
+        lostRunnerJob({
+          id: 8,
+          name: "timed-out-job",
+          conclusion: "timed_out",
+          steps: [{ name: "Run tests", status: "completed", conclusion: "timed_out" }],
+        }),
+      ],
+      jobDetailsAvailable: true,
+      jobDetailsComplete: true,
+    });
+
+    expect(evidence?.terminalClassificationPresent).toBe(true);
+    expect(detectRunnerLoss(evidence!)).toBe(false);
+  });
+
+  it("requires a complete job listing before confirming runner loss", () => {
+    expect(
+      verifiedRunnerLossEvidence({
+        workflowConclusion: "failure",
+        jobs: [lostRunnerJob()],
+        jobDetailsAvailable: true,
+        jobDetailsComplete: false,
+      }),
+    ).toBeNull();
+  });
+});

test/e2e/support/main-e2e-runner-loss-retry-workflow.test.ts

@@ -0,0 +1,128 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { describe, expect, it } from "vitest";
+
+import {
+  MAIN_E2E_RETRY_CHECKOUT,
+  MAIN_E2E_RETRY_COMMAND,
+  MAIN_E2E_RETRY_JOB_IF,
+  MAIN_E2E_RETRY_SETUP_NODE,
+  validateMainE2eRunnerLossRetryWorkflow,
+  validateMainE2eRunnerLossRetryWorkflowBoundary,
+} from "../../../tools/e2e/main-e2e-runner-loss-retry-workflow-boundary.mts";
+
+function validWorkflow(): Record<string, unknown> {
+  return {
+    name: "E2E Main Runner Loss Retry",
+    "run-name":
+      "E2E main runner-loss retry for run ${{ github.event.workflow_run.id }} attempt ${{ github.event.workflow_run.run_attempt }}",
+    on: { workflow_run: { workflows: ["E2E"], types: ["completed"] } },
+    permissions: {},
+    jobs: {
+      "classify-and-retry": {
+        if: MAIN_E2E_RETRY_JOB_IF,
+        "runs-on": "ubuntu-latest",
+        "timeout-minutes": 10,
+        permissions: { actions: "write", contents: "read" },
+        concurrency: {
+          group: "e2e-main-runner-loss-retry-${{ github.event.workflow_run.id }}",
+          "cancel-in-progress": false,
+        },
+        steps: [
+          {
+            name: "Checkout trusted retry controller",
+            uses: MAIN_E2E_RETRY_CHECKOUT,
+            with: { ref: "${{ github.workflow_sha }}", "persist-credentials": false },
+          },
+          {
+            name: "Setup Node",
+            uses: MAIN_E2E_RETRY_SETUP_NODE,
+            with: { "node-version": "22" },
+          },
+          {
+            name: "Classify runner loss and retry failed jobs once",
+            env: {
+              GITHUB_TOKEN: "${{ github.token }}",
+              SUBJECT_RUN_ATTEMPT: "${{ github.event.workflow_run.run_attempt }}",
+              SUBJECT_RUN_ID: "${{ github.event.workflow_run.id }}",
+            },
+            run: MAIN_E2E_RETRY_COMMAND,
+          },
+        ],
+      },
+    },
+  };
+}
+
+function retryJob(workflow: Record<string, unknown>): Record<string, unknown> {
+  return (workflow.jobs as Record<string, Record<string, unknown>>)["classify-and-retry"]!;
+}
+
+type WorkflowMutation = (workflow: Record<string, unknown>) => void;
+
+const WORKFLOW_MUTATIONS = [
+  [
+    "a pull-request event",
+    (workflow: Record<string, unknown>) => {
+      retryJob(workflow).if = MAIN_E2E_RETRY_JOB_IF.replace(
+        "github.event.workflow_run.event != 'pull_request'",
+        "true",
+      );
+    },
+  ],
+  [
+    "a selective-dispatch title",
+    (workflow: Record<string, unknown>) => {
+      retryJob(workflow).if = MAIN_E2E_RETRY_JOB_IF.replace("'E2E main'", "'E2E PR'");
+    },
+  ],
+  [
+    "a later run attempt",
+    (workflow: Record<string, unknown>) => {
+      retryJob(workflow).if = MAIN_E2E_RETRY_JOB_IF.replace("run_attempt == 1", "run_attempt > 0");
+    },
+  ],
+  [
+    "workflow-wide write permission",
+    (workflow: Record<string, unknown>) => {
+      workflow.permissions = { actions: "write" };
+    },
+  ],
+  [
+    "a subject-SHA checkout",
+    (workflow: Record<string, unknown>) => {
+      const steps = retryJob(workflow).steps as Record<string, unknown>[];
+      steps[0]!.with = {
+        ref: "${{ github.event.workflow_run.head_sha }}",
+        "persist-credentials": false,
+      };
+    },
+  ],
+  [
+    "a repository secret",
+    (workflow: Record<string, unknown>) => {
+      const steps = retryJob(workflow).steps as Record<string, unknown>[];
+      (steps[2]!.env as Record<string, unknown>).EXTRA_TOKEN = "${{ secrets.EXTRA_TOKEN }}";
+    },
+  ],
+  [
+    "an unreviewed command",
+    (workflow: Record<string, unknown>) => {
+      const steps = retryJob(workflow).steps as Record<string, unknown>[];
+      steps[2]!.run = "gh run rerun --failed";
+    },
+  ],
+] satisfies ReadonlyArray<readonly [string, WorkflowMutation]>;
+
+describe("final-main E2E runner-loss retry workflow for item 5 (#7140)", () => {
+  it("accepts the reviewed trusted workflow", () => {
+    expect(validateMainE2eRunnerLossRetryWorkflowBoundary()).toEqual([]);
+  });
+
+  it.each(WORKFLOW_MUTATIONS)("rejects %s", (_label, mutate) => {
+    const workflow = validWorkflow();
+    mutate(workflow);
+    expect(validateMainE2eRunnerLossRetryWorkflow(workflow)).not.toEqual([]);
+  });
+});