don-petry · Aug 7, 2026 · Mar 28, 2026 · Mar 28, 2026 · Mar 28, 2026 · Mar 29, 2026
@@ -26,6 +26,7 @@ on:
       - 'scripts/canary-rollout.sh'
       - 'scripts/lib/canary-rollout.sh'
       - 'tests/canary_rollout.bats'
+      - '.github/workflows/canary-rollout.yml'
       - '.github/workflows/canary-rollout-tests.yml'
   push:
     branches:
@@ -34,6 +35,7 @@ on:
       - 'scripts/canary-rollout.sh'
       - 'scripts/lib/canary-rollout.sh'
       - 'tests/canary_rollout.bats'
+      - '.github/workflows/canary-rollout.yml'
       - '.github/workflows/canary-rollout-tests.yml'
 
 permissions:

@@ -108,7 +108,12 @@ concurrency:
 jobs:
   canary:
     runs-on: ubuntu-latest
-    timeout-minutes: 15
+    # The scheduled fleet sweep's runtime grows with the fleet (the gate reads
+    # run-history owner-wide across every tier repo). Observed p50 877s / p95 988s
+    # sat right at the old 15-min (900s) ceiling, so the tail of each sweep was
+    # being killed at the timeout (#939). 30 min keeps durable headroom over p95
+    # while still bounding a genuine hang.
+    timeout-minutes: 30
     steps:
       # Mint a scoped GitHub App installation token (Contents/Deployments:write,
       # Actions/Metadata:read). owner-wide so the gate can read run history across all

@@ -87,6 +87,10 @@ Scheduled reports and dashboards post as issues or run summaries for org maintai
 - **[OpenSSF Scorecard](https://github.com/petry-projects/.github/blob/main/.github/workflows/org-scorecard.yml)**
   — Weekly security-posture review across public repos; findings tracked as issues.
 
+- **[Ruleset Remediation Runbook](https://github.com/petry-projects/.github/blob/main/standards/ruleset-remediation-runbook.md)** — Snapshot every ruleset
+  for rollback insurance; bypass actor management; legacy ruleset migration; verify and rollback
+  procedures.
+
 ---
 
 ## Contributing

@@ -1931,6 +1931,7 @@ check_centralized_check_names() {
   done <<< "$contexts"
 }
 
+
 # ---------------------------------------------------------------------------
 # Check: CLAUDE.md exists and references AGENTS.md
 # ---------------------------------------------------------------------------

@@ -21,6 +21,14 @@ set -euo pipefail
 #      concurrent dev-lead runs in any single repo to one, avoiding the rebase
 #      storms and token exhaustion a fleet-wide burst would cause.
 #
+#      Throttling: at most ONE issue per repo is engaged per run (shared across
+#      the primary and legacy-label sweeps), and a repo already active (open PR
+#      or in-progress issue) is skipped.  Issues are processed oldest-first, so
+#      the most-stuck finding in each repo is the one re-engaged; the daily
+#      cadence drains the rest of each repo's backlog one at a time.  This keeps
+#      concurrent dev-lead runs in any single repo to one, avoiding the rebase
+#      storms and token exhaustion a fleet-wide burst would cause.
+#
 # Why re-trigger instead of relying on the original event?
 #   GitHub only fires issues:labeled once per label application.  If dev-lead
 #   had a transient failure at that moment (template error, git-identity bug,
@@ -62,6 +70,14 @@ ISSUES_DEFERRED=0
 # every path that could engage dev-lead.
 declare -A REPO_ENGAGED=()
 
+# Repos that already have an in-flight dev-lead engagement THIS run — either an
+# issue we re-triggered or one we found already active. At most one engagement
+# per repo per run keeps concurrent dev-lead runs in a single repo to one,
+# avoiding rebase storms and token exhaustion from a fleet-wide burst. Shared
+# across BOTH the primary and legacy-label sweeps so the per-repo budget covers
+# every path that could engage dev-lead.
+declare -A REPO_ENGAGED=()
+
 # ---------------------------------------------------------------------------
 # Helpers
 # ---------------------------------------------------------------------------

@@ -549,6 +549,11 @@ deploy_repo() {
   branch="${SYNC_BRANCH_PREFIX}/workflows-$(date -u +%Y%m%d)"
   local title="chore: sync ${n} org-standard workflow stub(s) from ${ORG}/.github"
 
+  local n="${#names[@]}" list branch
+  list=$(IFS=', '; echo "${names[*]}")
+  branch="${SYNC_BRANCH_PREFIX}/workflows-$(date -u +%Y%m%d)"
+  local title="chore: sync ${n} org-standard workflow stub(s) from ${ORG}/.github"
+
   if [[ "$DRY_RUN" == "true" ]]; then
     local i
     for (( i = 0; i < n; i++ )); do

@@ -160,30 +160,35 @@ _should_flag_naming() {
 }
 
 # ===========================================================================
-# Naming suffix check — pure reusable must end in -reusable.yml
+# Naming suffix check — pure reusables must carry -reusable.yml
 # ===========================================================================
 
-@test "naming: compliant suffix (-reusable.yml) is NOT flagged" {
-  run _should_flag_naming "my-repo" "dev-lead-reusable.yml"
+@test "naming: file with -reusable.yml suffix is NOT flagged" {
+  run _should_flag_naming "my-repo" "deploy-reusable.yml"
   [ "$status" -eq 1 ]
 }
 
-@test "naming: missing suffix is flagged" {
-  run _should_flag_naming ".github-private" "some-workflow.yml"
+@test "naming: file without -reusable.yml suffix IS flagged" {
+  run _should_flag_naming "my-repo" "deploy.yml"
   [ "$status" -eq 0 ]
 }
 
-@test "naming: grandfathered pr-review.yml in .github-private is NOT flagged" {
+@test "naming: .github-private/pr-review.yml is grandfathered (NOT flagged)" {
   run _should_flag_naming ".github-private" "pr-review.yml"
   [ "$status" -eq 1 ]
 }
 
-@test "naming: pr-review.yml in a different repo IS flagged (grandfathering is repo-scoped)" {
-  run _should_flag_naming "some-other-repo" "pr-review.yml"
+@test "naming: pr-review.yml in a different repo IS flagged" {
+  run _should_flag_naming "other-repo" "pr-review.yml"
   [ "$status" -eq 0 ]
 }
 
-@test "naming: -reusable.yaml extension (not .yml) is flagged" {
-  run _should_flag_naming "my-repo" "foo-reusable.yaml"
+@test "naming: .yaml extension without -reusable suffix IS flagged" {
+  run _should_flag_naming "my-repo" "deploy.yaml"
   [ "$status" -eq 0 ]
 }
+
+@test "naming: -reusable.yaml is flagged (.yml suffix is the only exempt form)" {
+  run _should_flag_naming "my-repo" "deploy-reusable.yaml"
+  [ "$status" -eq 0 ]
+}
@@ -3738,3 +3738,43 @@ YML
   grep -q "git/tags .*tag=dev-lead/v3.0.0 .*object=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" "$GH_LOG"
   grep -q "PATCH repos/petry-projects/.github-private/git/refs/tags/dev-lead/v3-next .*sha=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" "$GH_LOG"
 }
+
+# ── workflow timeout headroom (#939) ─────────────────────────────────────────
+# Fleet Monitor flagged canary-rollout.yml as DEGRADED (50% failure): the
+# scheduled fleet sweep's p50 was 877s and p95 988s against a 15-min (900s)
+# job timeout, so the tail of every sweep was being killed at the ceiling. The
+# owner-wide run-history reads grow with the fleet, so the bound must sit well
+# above the observed p95 (~16.5 min). This guard fails loud if the canary job's
+# timeout-minutes is dropped back below that headroom floor.
+WORKFLOW="$SCRIPT_DIR/.github/workflows/canary-rollout.yml"
+
+@test "canary-rollout.yml: canary job declares a timeout-minutes" {
+  local minutes
+  minutes="$(awk '
+    /^  canary:[[:space:]]*$/ { in_canary=1; next }
+    in_canary && /^  [^[:space:]]/ { in_canary=0 }
+    in_canary && /^[[:space:]]*timeout-minutes:[[:space:]]*[0-9]+[[:space:]]*(#.*)?$/ {
+      match($0, /[0-9]+/)
+      print substr($0, RSTART, RLENGTH)
+      exit
+    }
+  ' "$WORKFLOW")"
+  [ -n "$minutes" ]
+}
+
+@test "canary-rollout.yml: canary job timeout-minutes keeps headroom over the observed p95 (#939)" {
+  # Observed p95 was 988s (~16.5 min); require >= 20 min so the fleet sweep's
+  # tail is not killed at the ceiling (the old 15-min bound must not return).
+  local minutes
+  minutes="$(awk '
+    /^  canary:[[:space:]]*$/ { in_canary=1; next }
+    in_canary && /^  [^[:space:]]/ { in_canary=0 }
+    in_canary && /^[[:space:]]*timeout-minutes:[[:space:]]*[0-9]+[[:space:]]*(#.*)?$/ {
+      match($0, /[0-9]+/)
+      print substr($0, RSTART, RLENGTH)
+      exit
+    }
+  ' "$WORKFLOW")"
+  [ -n "$minutes" ]
+  [ "$minutes" -ge 20 ]
+}