don-petry · Jun 10, 2026 · Jun 10, 2026 · Jun 10, 2026
@@ -2,7 +2,7 @@ name: Compliance Issue Re-trigger
 
 on:
   schedule:
-    - cron: '0 14 * * *'   # Daily at 14:00 UTC (9 AM CDT)
+    - cron: '0 5 * * *'   # Daily at 05:00 UTC (12:00 AM CDT)
   workflow_dispatch:
     inputs:
       stale_days:

@@ -14,6 +14,14 @@ set -euo pipefail
 #      label (remove + re-add) to re-fire the issues:labeled event and give
 #      dev-lead a fresh chance to create a fix PR.
 #
+#      Throttling: at most ONE issue per repo is engaged per run (shared across
+#      the primary and legacy-label sweeps), and a repo already active (open PR
+#      or in-progress issue) is skipped.  Issues are processed oldest-first, so
+#      the most-stuck finding in each repo is the one re-engaged; the daily
+#      cadence drains the rest of each repo's backlog one at a time.  This keeps
+#      concurrent dev-lead runs in any single repo to one, avoiding the rebase
+#      storms and token exhaustion a fleet-wide burst would cause.
+#
 # Why re-trigger instead of relying on the original event?
 #   GitHub only fires issues:labeled once per label application.  If dev-lead
 #   had a transient failure at that moment (template error, git-identity bug,
@@ -45,9 +53,18 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
 ISSUES_RETRIGGERED=0
 ISSUES_SKIPPED=0
+ISSUES_DEFERRED=0
 WORKFLOWS_DISABLED=0
 WORKFLOWS_ENABLED=0
 
+# Repos that already have an in-flight dev-lead engagement THIS run — either an
+# issue we re-triggered or one we found already active. At most one engagement
+# per repo per run keeps concurrent dev-lead runs in a single repo to one,
+# avoiding rebase storms and token exhaustion from a fleet-wide burst. Shared
+# across BOTH the primary and legacy-label sweeps so the per-repo budget covers
+# every path that could engage dev-lead.
+declare -A REPO_ENGAGED=()
+
 # ---------------------------------------------------------------------------
 # Helpers
 # ---------------------------------------------------------------------------
@@ -131,9 +148,15 @@ retrigger_stale_issues() {
   # producing a single "Skipping null#null" iteration when the token lacked
   # search scope. See: failure mode where ORG_SCORECARD_TOKEN scope did not
   # permit cross-org search but the script still reported success.
+  # is:issue is REQUIRED — GitHub now rejects search/issues queries that omit
+  # is:issue/is:pull-request with HTTP 422 (this was the bug that broke the daily
+  # sweep). sort=created&order=asc surfaces the oldest (most-stuck) issue per repo
+  # first, so the one-per-repo throttle below re-engages the right one. --paginate
+  # walks all result pages: with the throttle, a single repo's large backlog would
+  # otherwise fill the first 100 results and starve every other repo for the run.
   local raw rc
-  raw=$(gh api \
-    "search/issues?q=org:${ORG}+label:${AUDIT_LABEL}+label:${TRIGGER_LABEL}+state:open&per_page=100" \
+  raw=$(gh api --paginate \
+    "search/issues?q=org:${ORG}+label:${AUDIT_LABEL}+label:${TRIGGER_LABEL}+state:open+is:issue&sort=created&order=asc&per_page=100" \
     2>&1) && rc=0 || rc=$?
 
   if [ "$rc" -ne 0 ]; then
@@ -143,15 +166,18 @@ retrigger_stale_issues() {
     return 1
   fi
 
-  # Detect error-response JSON even when exit code was 0 (defensive).
-  if echo "$raw" | jq -e 'has("message") and (.items | not)' >/dev/null 2>&1; then
+  # With --paginate, gh concatenates one JSON object per page, so slurp (-s) and
+  # inspect every page. Detect an error-response object on any page even when the
+  # exit code was 0 (defensive).
+  if echo "$raw" | jq -se 'any(.[]; has("message") and (.items | not))' >/dev/null 2>&1; then
     error "search/issues returned an error response:"
-    echo "$raw" | jq -r '.message' >&2
+    echo "$raw" | jq -rs '.[] | select(has("message")) | .message' >&2
     return 1
   fi
 
+  # total_count is repeated identically on every page; take the first.
   local total
-  total=$(echo "$raw" | jq -r '.total_count // 0')
+  total=$(echo "$raw" | jq -rs '.[0].total_count // 0')
   info "search/issues returned ${total} matching issues"
 
   local issues
@@ -169,20 +195,35 @@ retrigger_stale_issues() {
     created_at=$(echo "$issue_json" | jq -r '.created_at')
     title=$(echo "$issue_json" | jq -r '.title')
 
-    # Check if stale
+    # Issues arrive oldest-first, so once we hit one newer than the cutoff every
+    # remaining issue is also newer (not stale) — stop scanning entirely.
     if [[ "$created_at" > "$cutoff" ]]; then
-      info "Skipping $repo#$number ($title) — created $created_at, not yet stale"
-      ISSUES_SKIPPED=$((ISSUES_SKIPPED + 1))
+      info "Reached first non-stale issue ($repo#$number, created $created_at); no older issues remain."
+      break
+    fi
+
+    # One engagement per repo per run. If this repo already got an issue
+    # re-triggered (or was found active) this run, defer the rest to a later sweep.
+    if [ -n "${REPO_ENGAGED[$repo]:-}" ]; then
+      info "Deferring $repo#$number ($title) — $repo already engaged this run (one issue per repo per run)"
+      ISSUES_DEFERRED=$((ISSUES_DEFERRED + 1))
       continue
     fi
 
     # Skip if dev-lead is already working this issue (open PR or in-progress).
+    # This consumes the repo's slot: a repo already churning on a fix must not
+    # get a second concurrent engagement.
     if dl_dev_lead_active "$ORG" "$repo" "$number"; then
-      info "Skipping $repo#$number — dev-lead already active (open PR or in-progress)"
+      info "Skipping $repo#$number — dev-lead already active (open PR or in-progress); $repo slot taken"
+      REPO_ENGAGED[$repo]=1
       ISSUES_SKIPPED=$((ISSUES_SKIPPED + 1))
       continue
     fi
 
+    # Take the repo's slot for this run regardless of cycle outcome, so a
+    # transient failure cannot let a second issue in the same repo fire and
+    # reintroduce burst behaviour. The next daily sweep retries this repo.
+    REPO_ENGAGED[$repo]=1
     info "Re-triggering $repo#$number: $title (created $created_at)"
     if dl_cycle_trigger_label "$ORG" "$repo" "$number" "$TRIGGER_LABEL" "$DRY_RUN"; then
       ISSUES_RETRIGGERED=$((ISSUES_RETRIGGERED + 1))
@@ -210,9 +251,11 @@ retrigger_stale_issues() {
     info "Sweeping pre-migration '$LEGACY_TRIGGER_LABEL'-labeled issues..."
     local legacy_issues legacy_rc
     # Exclude issues that already carry TRIGGER_LABEL — they were handled above.
-    # --paginate walks all result pages so issues beyond the first 100 are included.
+    # is:issue is required (HTTP 422 otherwise); oldest-first matches the primary
+    # sweep; --paginate walks all result pages so issues beyond the first 100 are
+    # included.
     legacy_issues=$(gh api --paginate \
-      "search/issues?q=org:${ORG}+label:${AUDIT_LABEL}+label:${LEGACY_TRIGGER_LABEL}+-label:${TRIGGER_LABEL}+state:open&per_page=100" \
+      "search/issues?q=org:${ORG}+label:${AUDIT_LABEL}+label:${LEGACY_TRIGGER_LABEL}+-label:${TRIGGER_LABEL}+state:open+is:issue&sort=created&order=asc&per_page=100" \
       --jq '.items[] | {number: .number, repo: (.repository_url | split("/") | last), created_at: .created_at, title: .title}' \
       2>&1) && legacy_rc=0 || legacy_rc=$?
     if [ "$legacy_rc" -ne 0 ]; then
@@ -227,15 +270,24 @@ retrigger_stale_issues() {
         repo=$(echo "$issue_json" | jq -r '.repo')
         created_at=$(echo "$issue_json" | jq -r '.created_at')
         title=$(echo "$issue_json" | jq -r '.title')
+        # Oldest-first: first non-stale issue means no older ones remain.
         if [[ "$created_at" > "$cutoff" ]]; then
-          ISSUES_SKIPPED=$((ISSUES_SKIPPED + 1))
+          break
+        fi
+        # Honour the shared one-engagement-per-repo budget set by the primary
+        # sweep, so a repo already engaged above is not also label-bumped here.
+        if [ -n "${REPO_ENGAGED[$repo]:-}" ]; then
+          info "Deferring legacy $repo#$number ($title) — $repo already engaged this run"
+          ISSUES_DEFERRED=$((ISSUES_DEFERRED + 1))
           continue
         fi
         if dl_dev_lead_active "$ORG" "$repo" "$number"; then
-          info "Skipping legacy $repo#$number — dev-lead already active"
+          info "Skipping legacy $repo#$number — dev-lead already active; $repo slot taken"
+          REPO_ENGAGED[$repo]=1
           ISSUES_SKIPPED=$((ISSUES_SKIPPED + 1))
           continue
         fi
+        REPO_ENGAGED[$repo]=1
         info "Adding '$TRIGGER_LABEL' to pre-migration issue $repo#$number: $title"
         if [ "$DRY_RUN" = "true" ]; then
           info "[dry-run] would ensure '$TRIGGER_LABEL' label exists in $repo and add it to #$number"
@@ -260,7 +312,7 @@ retrigger_stale_issues() {
     fi
   fi
 
-  info "Re-trigger complete: ${ISSUES_RETRIGGERED} retriggered, ${ISSUES_SKIPPED} skipped"
+  info "Re-trigger complete: ${ISSUES_RETRIGGERED} retriggered, ${ISSUES_SKIPPED} skipped, ${ISSUES_DEFERRED} deferred (repo already engaged this run)"
 }
 
 # ---------------------------------------------------------------------------
@@ -276,6 +328,7 @@ print_summary() {
   echo "  Dry run         : ${DRY_RUN}"
   echo "  Issues retriggered  : ${ISSUES_RETRIGGERED}"
   echo "  Issues skipped      : ${ISSUES_SKIPPED}"
+  echo "  Issues deferred     : ${ISSUES_DEFERRED} (repo already engaged this run)"
   echo "  Workflows re-enabled: ${WORKFLOWS_DISABLED}"
   echo "  Workflows already active: ${WORKFLOWS_ENABLED}"
   echo "=========================================="
@@ -288,6 +341,7 @@ print_summary() {
       echo "| ------ | ----- |"
       echo "| Issues retriggered | $ISSUES_RETRIGGERED |"
       echo "| Issues skipped (PR exists or recent) | $ISSUES_SKIPPED |"
+      echo "| Issues deferred (repo already engaged this run) | $ISSUES_DEFERRED |"
       echo "| dev-lead workflows re-enabled | $WORKFLOWS_DISABLED |"
       echo "| dev-lead workflows already active | $WORKFLOWS_ENABLED |"
       echo "| Stale threshold | ${STALE_DAYS} days |"

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+# shellcheck shell=bash
 # dev-lead-retrigger.sh — shared helpers for re-engaging the dev-lead agent on a
 # compliance issue by cycling its trigger label.
 #
@@ -7,6 +7,10 @@
 #                                      persist across runs (issue still open).
 #   scripts/compliance-retrigger.sh  — daily sweep; re-triggers stale issues.
 #
+# Caller contract:
+#   Source this file; do NOT execute it directly.  The library does NOT call
+#   `set` itself, so it is safe to source from scripts that use `set -euo pipefail`.
+#
 # Why a shared lib?
 #   Both entry points need the same two primitives, and both depend on how
 #   dev-lead names its branches (dev-lead/issue-<number>-<timestamp>, see
@@ -59,7 +63,7 @@ dl_dev_lead_active() {
     labeled_at=$(gh api "repos/$org/$repo/issues/$issue/events?per_page=100" \
       --paginate \
       --jq '.[] | select(.event == "labeled" and .label.name == "in-progress") | .created_at' \
-      2>/dev/null | tail -1 || echo "")
+      2>/dev/null | head -1 || echo "")
 
     if [ -z "$labeled_at" ] || [ "$labeled_at" = "null" ]; then
       # Cannot determine when the label was applied; trust it conservatively.
@@ -69,10 +73,14 @@ dl_dev_lead_active() {
     local now_epoch labeled_epoch elapsed_hours
     now_epoch=$(date -u +%s)
     labeled_epoch=$(date -u -d "$labeled_at" +%s 2>/dev/null \
-      || python3 -c "import sys, calendar, datetime; \
-                     ts = sys.stdin.read().strip(); \
-                     print(calendar.timegm(datetime.datetime.strptime(ts, '%Y-%m-%dT%H:%M:%SZ').timetuple()))" \
-      <<< "$labeled_at")
+      || python3 -c "import sys, datetime; \
+                     ts = sys.stdin.read().strip().replace('Z', '+00:00'); \
+                     print(int(datetime.datetime.fromisoformat(ts).timestamp()))" 2>/dev/null \
+      <<< "$labeled_at" || echo "")
+
+    if [ -z "$labeled_epoch" ]; then
+      return 0
+    fi
     elapsed_hours=$(( (now_epoch - labeled_epoch) / 3600 ))
 
     [ "$elapsed_hours" -le "$in_progress_max_hours" ] && return 0