fix(prioritize): CSMA/CD rate-limit resilience for post-prioritize

Makefile

@@ -107,6 +107,7 @@ lint-md-links:
 
 script-test:
 	bash internal/scaffold/fullsend-repo/scripts/post-triage-test.sh
+	bash internal/scaffold/fullsend-repo/scripts/post-prioritize-test.sh
 	bash internal/scaffold/fullsend-repo/scripts/post-code-test.sh
 	bash internal/scaffold/fullsend-repo/scripts/post-review-test.sh
 	bash internal/scaffold/fullsend-repo/scripts/reconcile-repos-test.sh

internal/scaffold/fullsend-repo/scripts/lib/github-api-csma.sh

@@ -0,0 +1,285 @@
+#!/usr/bin/env bash
+# github-api-csma.sh — CSMA/CD-style resilience for GitHub API calls via gh/fullsend.
+#
+# Carrier sense: check rate_limit before transmitting.
+# Slot time: random jitter between calls to desynchronize parallel runners.
+# Collision detection: retry on 429 / secondary rate limit errors with exponential backoff.
+#
+# Source from agent post-scripts:
+#   source "${SCRIPT_DIR}/lib/github-api-csma.sh"
+#
+# Environment (all optional):
+#   GITHUB_CSMA_MAX_ATTEMPTS          — default 8
+#   GITHUB_CSMA_MIN_REMAINING_CORE    — default 100
+#   GITHUB_CSMA_MIN_REMAINING_GRAPHQL — default 100
+#   GITHUB_CSMA_SLOT_MIN_MS           — default 250
+#   GITHUB_CSMA_SLOT_MAX_MS           — default 750 (0 disables jitter)
+#   GITHUB_CSMA_BACKOFF_CAP_SEC       — default 120
+
+# shellcheck shell=bash
+
+[[ -n "${GITHUB_API_CSMA_SH_LOADED:-}" ]] && return 0
+GITHUB_API_CSMA_SH_LOADED=1
+
+_github_csma_max_attempts() {
+  echo "${GITHUB_CSMA_MAX_ATTEMPTS:-8}"
+}
+
+_github_csma_min_remaining() {
+  local resource="$1"
+  case "${resource}" in
+    graphql) echo "${GITHUB_CSMA_MIN_REMAINING_GRAPHQL:-100}" ;;
+    *) echo "${GITHUB_CSMA_MIN_REMAINING_CORE:-100}" ;;
+  esac
+}
+
+_github_csma_slot_min_ms() {
+  echo "${GITHUB_CSMA_SLOT_MIN_MS:-250}"
+}
+
+_github_csma_slot_max_ms() {
+  echo "${GITHUB_CSMA_SLOT_MAX_MS:-750}"
+}
+
+_github_csma_backoff_cap_sec() {
+  echo "${GITHUB_CSMA_BACKOFF_CAP_SEC:-120}"
+}
+
+_github_csma_emit_failure() {
+  printf '%s\n' "$1" >&2
+}
+
+# Wait until the named rate_limit resource has enough quota (carrier sense).
+# Usage: github_csma_sense [core|graphql] [min_remaining]
+github_csma_sense() {
+  local resource="${1:-core}"
+  local min_remaining="${2:-$(_github_csma_min_remaining "${resource}")}"
+
+  local info remaining reset now wait_secs
+  if ! info=$(gh api rate_limit 2>/dev/null); then
+    echo "WARNING: github_csma_sense: could not read rate_limit; proceeding" >&2
+    return 0
+  fi
+
+  remaining=$(echo "${info}" | jq -r --arg r "${resource}" '.resources[$r].remaining // empty')
+  reset=$(echo "${info}" | jq -r --arg r "${resource}" '.resources[$r].reset // empty')
+
+  if [[ -z "${remaining}" || "${remaining}" == "null" ]]; then
+    echo "WARNING: github_csma_sense: no .resources.${resource} in rate_limit; proceeding" >&2
+    return 0
+  fi
+
+  if (( remaining >= min_remaining )); then
+    return 0
+  fi
+
+  now=$(date +%s)
+  wait_secs=$(( reset - now + 1 ))
+  if (( wait_secs < 1 )); then
+    wait_secs=1
+  fi
+  cap=$(_github_csma_backoff_cap_sec)
+  if (( wait_secs > cap )); then
+    wait_secs="${cap}"
+  fi
+
+  echo "Rate limit sense: ${resource} remaining=${remaining} (min=${min_remaining}); waiting ${wait_secs}s until reset..." >&2
+  sleep "${wait_secs}"
+}
+
+# Random inter-call delay (slot time) to reduce synchronized collisions.
+github_csma_slot() {
+  local max_ms min_ms span_ms delay_ms
+  max_ms=$(_github_csma_slot_max_ms)
+  if (( max_ms <= 0 )); then
+    return 0
+  fi
+  min_ms=$(_github_csma_slot_min_ms)
+  if (( min_ms > max_ms )); then
+    min_ms="${max_ms}"
+  fi
+  span_ms=$(( max_ms - min_ms + 1 ))
+  delay_ms=$(( min_ms + RANDOM % span_ms ))
+  sleep "$(awk -v ms="${delay_ms}" 'BEGIN { printf "%.3f", ms / 1000 }')"
+}
+
+# Return 0 if combined output looks like a retryable GitHub rate limit error.
+github_csma_is_rate_limit() {
+  local text="$1"
+  local lower
+  lower=$(echo "${text}" | tr '[:upper:]' '[:lower:]')
+
+  if echo "${lower}" | grep -qE 'http 429|status: 429'; then
+    return 0
+  fi
+  if echo "${lower}" | grep -qE 'secondary rate limit|rate limit exceeded|api rate limit'; then
+    return 0
+  fi
+  if echo "${lower}" | grep -qE 'http 403|status: 403'; then
+    if echo "${lower}" | grep -qE 'secondary|rate limit|abuse|retry.after'; then
+      return 0
+    fi
+  fi
+  return 1
+}
+
+# Compute backoff seconds for attempt (0-based). Writes to stdout.
+github_csma_backoff() {
+  local attempt="$1"
+  local cap base delay
+  cap=$(_github_csma_backoff_cap_sec)
+  base=$(( 1 << attempt ))
+  if (( base > cap )); then
+    base="${cap}"
+  fi
+  delay=$(( RANDOM % (base + 1) ))
+  if (( delay < 1 )); then
+    delay=1
+  fi
+  echo "${delay}"
+}
+
+_github_csma_sleep_after_rate_limit() {
+  local attempt="$1"
+  local resource="${2:-core}"
+  local delay wait_secs now reset info cap
+
+  delay=$(github_csma_backoff "${attempt}")
+  if info=$(gh api rate_limit 2>/dev/null); then
+    now=$(date +%s)
+    reset=$(echo "${info}" | jq -r --arg r "${resource}" '.resources[$r].reset // empty')
+    if [[ -n "${reset}" && "${reset}" != "null" ]]; then
+      wait_secs=$(( reset - now + 1 ))
+      cap=$(_github_csma_backoff_cap_sec)
+      if (( wait_secs > cap )); then
+        wait_secs="${cap}"
+      fi
+      if (( wait_secs > delay && wait_secs > 0 )); then
+        delay="${wait_secs}"
+      fi
+    fi
+  fi
+  echo "GitHub API rate limit (attempt $(( attempt + 1 ))); backing off ${delay}s..." >&2
+  sleep "${delay}"
+}
+
+# Run gh with CSMA/CD. First argument: rate_limit resource (core|graphql).
+# Remaining arguments are passed to gh. Prints gh stdout on success.
+github_csma_run() {
+  local resource="${1:-core}"
+  shift
+
+  local max_attempts attempt outfile errfile combined
+  max_attempts=$(_github_csma_max_attempts)
+  outfile=$(mktemp)
+  errfile=$(mktemp)
+  # shellcheck disable=SC2064
+  trap "rm -f '${outfile}' '${errfile}'" RETURN
+
+  for (( attempt = 0; attempt < max_attempts; attempt++ )); do
+    github_csma_sense "${resource}"
+    github_csma_slot
+
+    : >"${outfile}"
+    : >"${errfile}"
+    if gh "$@" >"${outfile}" 2>"${errfile}"; then
+      cat "${outfile}"
+      return 0
+    fi
+
+    combined=$(cat "${outfile}" "${errfile}")
+    if github_csma_is_rate_limit "${combined}"; then
+      if (( attempt < max_attempts - 1 )); then
+        _github_csma_sleep_after_rate_limit "${attempt}" "${resource}"
+        continue
+      fi
+    fi
+
+    _github_csma_emit_failure "${combined}"
+    return 1
+  done
+
+  return 1
+}
+
+# Run producer | gh with CSMA/CD. First argument: resource; rest are gh args.
+# Reads producer output from stdin (save once for retries).
+github_csma_run_pipe() {
+  local resource="${1:-graphql}"
+  shift
+
+  local max_attempts attempt infile outfile errfile combined
+  max_attempts=$(_github_csma_max_attempts)
+  infile=$(mktemp)
+  outfile=$(mktemp)
+  errfile=$(mktemp)
+  cat >"${infile}"
+  # shellcheck disable=SC2064
+  trap "rm -f '${infile}' '${outfile}' '${errfile}'" RETURN
+
+  for (( attempt = 0; attempt < max_attempts; attempt++ )); do
+    github_csma_sense "${resource}"
+    github_csma_slot
+
+    : >"${outfile}"
+    : >"${errfile}"
+    if gh "$@" <"${infile}" >"${outfile}" 2>"${errfile}"; then
+      cat "${outfile}"
+      return 0
+    fi
+
+    combined=$(cat "${outfile}" "${errfile}")
+    if github_csma_is_rate_limit "${combined}"; then
+      if (( attempt < max_attempts - 1 )); then
+        _github_csma_sleep_after_rate_limit "${attempt}" "${resource}"
+        continue
+      fi
+    fi
+
+    _github_csma_emit_failure "${combined}"
+    return 1
+  done
+
+  return 1
+}
+
+# Run an arbitrary command with stdin from caller; retries on rate-limit errors in output.
+# First argument: rate_limit resource (core|graphql); remaining args are the command.
+github_csma_run_cmd() {
+  local resource="${1:-core}"
+  shift
+
+  local max_attempts attempt infile outfile errfile combined
+  max_attempts=$(_github_csma_max_attempts)
+  infile=$(mktemp)
+  outfile=$(mktemp)
+  errfile=$(mktemp)
+  cat >"${infile}"
+  # shellcheck disable=SC2064
+  trap "rm -f '${infile}' '${outfile}' '${errfile}'" RETURN
+
+  for (( attempt = 0; attempt < max_attempts; attempt++ )); do
+    github_csma_sense "${resource}"
+    github_csma_slot
+
+    : >"${outfile}"
+    : >"${errfile}"
+    if "$@" <"${infile}" >"${outfile}" 2>"${errfile}"; then
+      cat "${outfile}"
+      return 0
+    fi
+
+    combined=$(cat "${outfile}" "${errfile}")
+    if github_csma_is_rate_limit "${combined}"; then
+      if (( attempt < max_attempts - 1 )); then
+        _github_csma_sleep_after_rate_limit "${attempt}" "${resource}"
+        continue
+      fi
+    fi
+
+    _github_csma_emit_failure "${combined}"
+    return 1
+  done
+
+  return 1
+}