From 6e7f129391c48d7378f30074deb34f181ac953fb Mon Sep 17 00:00:00 2001
From: Tony Davis <tony.davis@amd.com>
Date: Tue, 7 Jul 2026 19:36:33 -0500
Subject: [PATCH] ci: auto-rerun nightly CI on failed/flaky jobs

Add a workflow_run follow-up that re-runs the failed jobs of a
"TheRock CI Nightly" run on fresh runners, up to 4 total attempts,
mirroring the web-UI "Re-run failed jobs" button. Most nightly
failures are infra flakes (no GPU, overloaded-machine timeout,
network blip on artifact fetch) that clear on a healthy runner.

Closes #9173

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 .github/workflows/auto-retry-failed.yml | 58 +++++++++++++++++++++++++
 1 file changed, 58 insertions(+)
 create mode 100644 .github/workflows/auto-retry-failed.yml

diff --git a/.github/workflows/auto-retry-failed.yml b/.github/workflows/auto-retry-failed.yml
new file mode 100644
index 000000000000..c7ad57add856
--- /dev/null
+++ b/.github/workflows/auto-retry-failed.yml
@@ -0,0 +1,58 @@
+# Copyright (c) Advanced Micro Devices, Inc., or its affiliates.
+# SPDX-License-Identifier: MIT
+
+# Automatically re-runs the failed jobs of a watched CI run, up to a fixed
+# number of attempts. This mirrors clicking "Re-run failed jobs" in the web UI:
+# `gh run rerun --failed` calls the same rerun-failed-jobs API, so each retry
+# lands on a fresh runner and is recorded as a new attempt on the SAME run
+# (prior attempts stay visible in the run's attempt history).
+#
+# The retry loop is self-capping: `workflow_run` fires whenever an attempt of
+# the watched run completes, so a failed attempt N triggers this workflow with
+# run_attempt == N, which re-runs and produces attempt N+1, and so on until the
+# run_attempt cap below is reached.
+#
+# Motivation: most nightly failures are infrastructure flakes (no GPU assigned,
+# an overloaded/wedged machine, or a network blip during artifact fetch) rather
+# than real test regressions. Those clear on a fresh runner, which a same-runner
+# step retry cannot provide.
+#
+# NOTE: `workflow_run` always uses the copy of this file on the repository's
+# default branch, and only fires for runs there. It has no effect from a PR
+# branch until merged.
+
+name: Auto-retry failed CI
+
+on:
+  workflow_run:
+    workflows:
+      - "TheRock CI Nightly"
+    types:
+      - completed
+
+permissions:
+  # Required to trigger a re-run of the watched workflow run.
+  actions: write
+  contents: read
+
+jobs:
+  rerun-failed:
+    name: Rerun failed jobs
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    # Only genuine failures under the attempt cap. run_attempt < 4 allows up to
+    # 4 total attempts (the original run plus 3 auto-retries). 'cancelled'
+    # (concurrency / superseded runs) and 'startup_failure' are intentionally
+    # excluded so we never retry work that was deliberately abandoned.
+    if: >-
+      github.event.workflow_run.conclusion == 'failure' &&
+      github.event.workflow_run.run_attempt < 4
+    env:
+      GH_TOKEN: ${{ github.token }}
+      RUN_ID: ${{ github.event.workflow_run.id }}
+      ATTEMPT: ${{ github.event.workflow_run.run_attempt }}
+    steps:
+      - name: Re-run failed jobs
+        run: |
+          echo "Run ${RUN_ID} failed on attempt ${ATTEMPT}; re-running failed jobs on fresh runners."
+          gh run rerun "${RUN_ID}" --failed --repo "${{ github.repository }}"