diff --git a/internal/scaffold/fullsend-repo/scripts/lib/github-api-csma.sh b/internal/scaffold/fullsend-repo/scripts/lib/github-api-csma.sh deleted file mode 100644 index f3870ad1a5..0000000000 --- a/internal/scaffold/fullsend-repo/scripts/lib/github-api-csma.sh +++ /dev/null @@ -1,321 +0,0 @@ -#!/usr/bin/env bash -# github-api-csma.sh — CSMA/CD-style resilience for GitHub API calls via gh/fullsend. -# -# Carrier sense: check rate_limit before transmitting. -# Slot time: random jitter between calls to desynchronize parallel runners. -# Collision detection: retry on 429 / secondary rate limit errors with exponential backoff. -# -# Source from agent post-scripts: -# source "${SCRIPT_DIR}/lib/github-api-csma.sh" -# -# Environment (all optional): -# GITHUB_CSMA_MAX_ATTEMPTS — default 8 -# GITHUB_CSMA_MIN_REMAINING_CORE — default 100 -# GITHUB_CSMA_MIN_REMAINING_GRAPHQL — default 100 -# GITHUB_CSMA_SLOT_MIN_MS — default 250 -# GITHUB_CSMA_SLOT_MAX_MS — default 750 (0 disables jitter) -# GITHUB_CSMA_SPREAD_MAX_SEC — default 60 (post-reset desync spread) -# GITHUB_CSMA_BACKOFF_CAP_SEC — default 120 - -# shellcheck shell=bash - -[[ -n "${GITHUB_API_CSMA_SH_LOADED:-}" ]] && return 0 -GITHUB_API_CSMA_SH_LOADED=1 - -_github_csma_max_attempts() { - echo "${GITHUB_CSMA_MAX_ATTEMPTS:-8}" -} - -_github_csma_min_remaining() { - local resource="$1" - case "${resource}" in - graphql) echo "${GITHUB_CSMA_MIN_REMAINING_GRAPHQL:-100}" ;; - *) echo "${GITHUB_CSMA_MIN_REMAINING_CORE:-100}" ;; - esac -} - -_github_csma_slot_min_ms() { - echo "${GITHUB_CSMA_SLOT_MIN_MS:-250}" -} - -_github_csma_slot_max_ms() { - echo "${GITHUB_CSMA_SLOT_MAX_MS:-750}" -} - -_github_csma_spread_max_sec() { - echo "${GITHUB_CSMA_SPREAD_MAX_SEC:-60}" -} - -_github_csma_backoff_cap_sec() { - echo "${GITHUB_CSMA_BACKOFF_CAP_SEC:-120}" -} - -# Add a random spread delay after a rate-limit sleep to desynchronize runners. -# Called from both github_csma_sense and _github_csma_sleep_after_rate_limit. -_github_csma_post_reset_spread() { - local spread_max - spread_max=$(_github_csma_spread_max_sec) - if (( spread_max > 0 )); then - local spread_secs=$(( RANDOM % spread_max )) - echo "Rate limit reset — spreading ${spread_secs}s to desync from other runners..." >&2 - sleep "${spread_secs}" - fi -} - -_github_csma_emit_failure() { - printf '%s\n' "$1" >&2 -} - -# Wait until the named rate_limit resource has enough quota (carrier sense). -# Usage: github_csma_sense [core|graphql] [min_remaining] -github_csma_sense() { - local resource="${1:-core}" - local min_remaining="${2:-$(_github_csma_min_remaining "${resource}")}" - - local info remaining reset now wait_secs - if ! info=$(gh api rate_limit 2>/dev/null); then - echo "WARNING: github_csma_sense: could not read rate_limit; proceeding" >&2 - return 0 - fi - - remaining=$(echo "${info}" | jq -r --arg r "${resource}" '.resources[$r].remaining // empty') - reset=$(echo "${info}" | jq -r --arg r "${resource}" '.resources[$r].reset // empty') - - if [[ -z "${remaining}" || "${remaining}" == "null" ]]; then - echo "WARNING: github_csma_sense: no .resources.${resource} in rate_limit; proceeding" >&2 - return 0 - fi - - if (( remaining >= min_remaining )); then - return 0 - fi - - now=$(date +%s) - wait_secs=$(( reset - now + 1 )) - if (( wait_secs < 1 )); then - wait_secs=1 - fi - cap=$(_github_csma_backoff_cap_sec) - if (( wait_secs > cap )); then - wait_secs="${cap}" - fi - - echo "Rate limit sense: ${resource} remaining=${remaining} (min=${min_remaining}); waiting ${wait_secs}s until reset..." >&2 - sleep "${wait_secs}" - - # After a rate-limit sleep, all runners wake at the same reset timestamp. - # Spread them over a wide window to avoid a thundering herd. - _github_csma_post_reset_spread -} - -# Random inter-call delay (slot time) to reduce synchronized collisions. -github_csma_slot() { - local max_ms min_ms span_ms delay_ms - max_ms=$(_github_csma_slot_max_ms) - if (( max_ms <= 0 )); then - return 0 - fi - min_ms=$(_github_csma_slot_min_ms) - if (( min_ms > max_ms )); then - min_ms="${max_ms}" - fi - span_ms=$(( max_ms - min_ms + 1 )) - delay_ms=$(( min_ms + RANDOM % span_ms )) - sleep "$(awk -v ms="${delay_ms}" 'BEGIN { printf "%.3f", ms / 1000 }')" -} - -# Return 0 if combined output looks like a retryable GitHub rate limit error. -github_csma_is_rate_limit() { - local text="$1" - local lower - lower=$(echo "${text}" | tr '[:upper:]' '[:lower:]') - - if echo "${lower}" | grep -qE 'http 429|status: 429'; then - return 0 - fi - if echo "${lower}" | grep -qE 'secondary rate limit|rate limit exceeded|api rate limit'; then - return 0 - fi - if echo "${lower}" | grep -qE 'http 403|status: 403'; then - if echo "${lower}" | grep -qE 'secondary|rate limit|abuse|retry.after'; then - return 0 - fi - fi - return 1 -} - -# Compute backoff seconds for attempt (0-based). Writes to stdout. -github_csma_backoff() { - local attempt="$1" - local cap base delay - cap=$(_github_csma_backoff_cap_sec) - base=$(( 1 << attempt )) - if (( base > cap )); then - base="${cap}" - fi - delay=$(( RANDOM % (base + 1) )) - if (( delay < 1 )); then - delay=1 - fi - echo "${delay}" -} - -_github_csma_sleep_after_rate_limit() { - local attempt="$1" - local resource="${2:-core}" - local delay wait_secs now reset info cap - - delay=$(github_csma_backoff "${attempt}") - if info=$(gh api rate_limit 2>/dev/null); then - now=$(date +%s) - reset=$(echo "${info}" | jq -r --arg r "${resource}" '.resources[$r].reset // empty') - if [[ -n "${reset}" && "${reset}" != "null" ]]; then - wait_secs=$(( reset - now + 1 )) - cap=$(_github_csma_backoff_cap_sec) - if (( wait_secs > cap )); then - wait_secs="${cap}" - fi - if (( wait_secs > delay && wait_secs > 0 )); then - delay="${wait_secs}" - fi - fi - fi - echo "GitHub API rate limit (attempt $(( attempt + 1 ))); backing off ${delay}s..." >&2 - sleep "${delay}" - - # After backing off, spread runners to avoid thundering herd on wake. - _github_csma_post_reset_spread -} - -# Run gh with CSMA/CD. First argument: rate_limit resource (core|graphql). -# Remaining arguments are passed to gh. Prints gh stdout on success. -github_csma_run() { - local resource="${1:-core}" - shift - - local max_attempts attempt outfile errfile combined - max_attempts=$(_github_csma_max_attempts) - outfile=$(mktemp) - errfile=$(mktemp) - # shellcheck disable=SC2064 - trap "rm -f '${outfile}' '${errfile}'" RETURN - - for (( attempt = 0; attempt < max_attempts; attempt++ )); do - github_csma_sense "${resource}" - github_csma_slot - - : >"${outfile}" - : >"${errfile}" - local rc=0 - gh "$@" >"${outfile}" 2>"${errfile}" || rc=$? - - combined=$(cat "${outfile}" "${errfile}") - if github_csma_is_rate_limit "${combined}"; then - if (( attempt < max_attempts - 1 )); then - _github_csma_sleep_after_rate_limit "${attempt}" "${resource}" - continue - fi - _github_csma_emit_failure "${combined}" - return 1 - fi - - if (( rc != 0 )); then - _github_csma_emit_failure "${combined}" - return 1 - fi - cat "${outfile}" - return 0 - done - - return 1 -} - -# Run producer | gh with CSMA/CD. First argument: resource; rest are gh args. -# Reads producer output from stdin (save once for retries). -github_csma_run_pipe() { - local resource="${1:-graphql}" - shift - - local max_attempts attempt infile outfile errfile combined - max_attempts=$(_github_csma_max_attempts) - infile=$(mktemp) - outfile=$(mktemp) - errfile=$(mktemp) - cat >"${infile}" - # shellcheck disable=SC2064 - trap "rm -f '${infile}' '${outfile}' '${errfile}'" RETURN - - for (( attempt = 0; attempt < max_attempts; attempt++ )); do - github_csma_sense "${resource}" - github_csma_slot - - : >"${outfile}" - : >"${errfile}" - local rc=0 - gh "$@" <"${infile}" >"${outfile}" 2>"${errfile}" || rc=$? - - combined=$(cat "${outfile}" "${errfile}") - if github_csma_is_rate_limit "${combined}"; then - if (( attempt < max_attempts - 1 )); then - _github_csma_sleep_after_rate_limit "${attempt}" "${resource}" - continue - fi - _github_csma_emit_failure "${combined}" - return 1 - fi - - if (( rc != 0 )); then - _github_csma_emit_failure "${combined}" - return 1 - fi - cat "${outfile}" - return 0 - done - - return 1 -} - -# Run an arbitrary command with stdin from caller; retries on rate-limit errors in output. -# First argument: rate_limit resource (core|graphql); remaining args are the command. -github_csma_run_cmd() { - local resource="${1:-core}" - shift - - local max_attempts attempt infile outfile errfile combined - max_attempts=$(_github_csma_max_attempts) - infile=$(mktemp) - outfile=$(mktemp) - errfile=$(mktemp) - cat >"${infile}" - # shellcheck disable=SC2064 - trap "rm -f '${infile}' '${outfile}' '${errfile}'" RETURN - - for (( attempt = 0; attempt < max_attempts; attempt++ )); do - github_csma_sense "${resource}" - github_csma_slot - - : >"${outfile}" - : >"${errfile}" - local rc=0 - "$@" <"${infile}" >"${outfile}" 2>"${errfile}" || rc=$? - - combined=$(cat "${outfile}" "${errfile}") - if github_csma_is_rate_limit "${combined}"; then - if (( attempt < max_attempts - 1 )); then - _github_csma_sleep_after_rate_limit "${attempt}" "${resource}" - continue - fi - _github_csma_emit_failure "${combined}" - return 1 - fi - - if (( rc != 0 )); then - _github_csma_emit_failure "${combined}" - return 1 - fi - cat "${outfile}" - return 0 - done - - return 1 -} diff --git a/internal/scaffold/fullsend-repo/scripts/post-prioritize-test.sh b/internal/scaffold/fullsend-repo/scripts/post-prioritize-test.sh index 096e9dd9a3..214b9b353f 100755 --- a/internal/scaffold/fullsend-repo/scripts/post-prioritize-test.sh +++ b/internal/scaffold/fullsend-repo/scripts/post-prioritize-test.sh @@ -266,8 +266,324 @@ run_test_failure_stderr() { } # Unit tests for shared CSMA helpers. -# shellcheck source=lib/github-api-csma.sh -source "${SCRIPT_DIR}/lib/github-api-csma.sh" +# --- Inlined from lib/github-api-csma.sh --- +# CSMA/CD-style resilience for GitHub API calls via gh/fullsend. +# Carrier sense: check rate_limit before transmitting. +# Slot time: random jitter between calls to desynchronize parallel runners. +# Collision detection: retry on 429 / secondary rate limit errors with exponential backoff. +# +# Environment (all optional): +# GITHUB_CSMA_MAX_ATTEMPTS — default 8 +# GITHUB_CSMA_MIN_REMAINING_CORE — default 100 +# GITHUB_CSMA_MIN_REMAINING_GRAPHQL — default 100 +# GITHUB_CSMA_SLOT_MIN_MS — default 250 +# GITHUB_CSMA_SLOT_MAX_MS — default 750 (0 disables jitter) +# GITHUB_CSMA_SPREAD_MAX_SEC — default 60 (post-reset desync spread) +# GITHUB_CSMA_BACKOFF_CAP_SEC — default 120 + +# shellcheck shell=bash + +[[ -n "${GITHUB_API_CSMA_SH_LOADED:-}" ]] && return 0 +GITHUB_API_CSMA_SH_LOADED=1 + +_github_csma_max_attempts() { + echo "${GITHUB_CSMA_MAX_ATTEMPTS:-8}" +} + +_github_csma_min_remaining() { + local resource="$1" + case "${resource}" in + graphql) echo "${GITHUB_CSMA_MIN_REMAINING_GRAPHQL:-100}" ;; + *) echo "${GITHUB_CSMA_MIN_REMAINING_CORE:-100}" ;; + esac +} + +_github_csma_slot_min_ms() { + echo "${GITHUB_CSMA_SLOT_MIN_MS:-250}" +} + +_github_csma_slot_max_ms() { + echo "${GITHUB_CSMA_SLOT_MAX_MS:-750}" +} + +_github_csma_spread_max_sec() { + echo "${GITHUB_CSMA_SPREAD_MAX_SEC:-60}" +} + +_github_csma_backoff_cap_sec() { + echo "${GITHUB_CSMA_BACKOFF_CAP_SEC:-120}" +} + +# Add a random spread delay after a rate-limit sleep to desynchronize runners. +# Called from both github_csma_sense and _github_csma_sleep_after_rate_limit. +_github_csma_post_reset_spread() { + local spread_max + spread_max=$(_github_csma_spread_max_sec) + if (( spread_max > 0 )); then + local spread_secs=$(( RANDOM % spread_max )) + echo "Rate limit reset — spreading ${spread_secs}s to desync from other runners..." >&2 + sleep "${spread_secs}" + fi +} + +_github_csma_emit_failure() { + printf '%s\n' "$1" >&2 +} + +# Wait until the named rate_limit resource has enough quota (carrier sense). +# Usage: github_csma_sense [core|graphql] [min_remaining] +github_csma_sense() { + local resource="${1:-core}" + local min_remaining="${2:-$(_github_csma_min_remaining "${resource}")}" + + local info remaining reset now wait_secs + if ! info=$(gh api rate_limit 2>/dev/null); then + echo "WARNING: github_csma_sense: could not read rate_limit; proceeding" >&2 + return 0 + fi + + remaining=$(echo "${info}" | jq -r --arg r "${resource}" '.resources[$r].remaining // empty') + reset=$(echo "${info}" | jq -r --arg r "${resource}" '.resources[$r].reset // empty') + + if [[ -z "${remaining}" || "${remaining}" == "null" ]]; then + echo "WARNING: github_csma_sense: no .resources.${resource} in rate_limit; proceeding" >&2 + return 0 + fi + + if (( remaining >= min_remaining )); then + return 0 + fi + + now=$(date +%s) + wait_secs=$(( reset - now + 1 )) + if (( wait_secs < 1 )); then + wait_secs=1 + fi + cap=$(_github_csma_backoff_cap_sec) + if (( wait_secs > cap )); then + wait_secs="${cap}" + fi + + echo "Rate limit sense: ${resource} remaining=${remaining} (min=${min_remaining}); waiting ${wait_secs}s until reset..." >&2 + sleep "${wait_secs}" + + # After a rate-limit sleep, all runners wake at the same reset timestamp. + # Spread them over a wide window to avoid a thundering herd. + _github_csma_post_reset_spread +} + +# Random inter-call delay (slot time) to reduce synchronized collisions. +github_csma_slot() { + local max_ms min_ms span_ms delay_ms + max_ms=$(_github_csma_slot_max_ms) + if (( max_ms <= 0 )); then + return 0 + fi + min_ms=$(_github_csma_slot_min_ms) + if (( min_ms > max_ms )); then + min_ms="${max_ms}" + fi + span_ms=$(( max_ms - min_ms + 1 )) + delay_ms=$(( min_ms + RANDOM % span_ms )) + sleep "$(awk -v ms="${delay_ms}" 'BEGIN { printf "%.3f", ms / 1000 }')" +} + +# Return 0 if combined output looks like a retryable GitHub rate limit error. +github_csma_is_rate_limit() { + local text="$1" + local lower + lower=$(echo "${text}" | tr '[:upper:]' '[:lower:]') + + if echo "${lower}" | grep -qE 'http 429|status: 429'; then + return 0 + fi + if echo "${lower}" | grep -qE 'secondary rate limit|rate limit exceeded|api rate limit'; then + return 0 + fi + if echo "${lower}" | grep -qE 'http 403|status: 403'; then + if echo "${lower}" | grep -qE 'secondary|rate limit|abuse|retry.after'; then + return 0 + fi + fi + return 1 +} + +# Compute backoff seconds for attempt (0-based). Writes to stdout. +github_csma_backoff() { + local attempt="$1" + local cap base delay + cap=$(_github_csma_backoff_cap_sec) + base=$(( 1 << attempt )) + if (( base > cap )); then + base="${cap}" + fi + delay=$(( RANDOM % (base + 1) )) + if (( delay < 1 )); then + delay=1 + fi + echo "${delay}" +} + +_github_csma_sleep_after_rate_limit() { + local attempt="$1" + local resource="${2:-core}" + local delay wait_secs now reset info cap + + delay=$(github_csma_backoff "${attempt}") + if info=$(gh api rate_limit 2>/dev/null); then + now=$(date +%s) + reset=$(echo "${info}" | jq -r --arg r "${resource}" '.resources[$r].reset // empty') + if [[ -n "${reset}" && "${reset}" != "null" ]]; then + wait_secs=$(( reset - now + 1 )) + cap=$(_github_csma_backoff_cap_sec) + if (( wait_secs > cap )); then + wait_secs="${cap}" + fi + if (( wait_secs > delay && wait_secs > 0 )); then + delay="${wait_secs}" + fi + fi + fi + echo "GitHub API rate limit (attempt $(( attempt + 1 ))); backing off ${delay}s..." >&2 + sleep "${delay}" + + # After backing off, spread runners to avoid thundering herd on wake. + _github_csma_post_reset_spread +} + +# Run gh with CSMA/CD. First argument: rate_limit resource (core|graphql). +# Remaining arguments are passed to gh. Prints gh stdout on success. +github_csma_run() { + local resource="${1:-core}" + shift + + local max_attempts attempt outfile errfile combined + max_attempts=$(_github_csma_max_attempts) + outfile=$(mktemp) + errfile=$(mktemp) + # shellcheck disable=SC2064 + trap "rm -f '${outfile}' '${errfile}'" RETURN + + for (( attempt = 0; attempt < max_attempts; attempt++ )); do + github_csma_sense "${resource}" + github_csma_slot + + : >"${outfile}" + : >"${errfile}" + local rc=0 + gh "$@" >"${outfile}" 2>"${errfile}" || rc=$? + + combined=$(cat "${outfile}" "${errfile}") + if github_csma_is_rate_limit "${combined}"; then + if (( attempt < max_attempts - 1 )); then + _github_csma_sleep_after_rate_limit "${attempt}" "${resource}" + continue + fi + _github_csma_emit_failure "${combined}" + return 1 + fi + + if (( rc != 0 )); then + _github_csma_emit_failure "${combined}" + return 1 + fi + cat "${outfile}" + return 0 + done + + return 1 +} + +# Run producer | gh with CSMA/CD. First argument: resource; rest are gh args. +# Reads producer output from stdin (save once for retries). +github_csma_run_pipe() { + local resource="${1:-graphql}" + shift + + local max_attempts attempt infile outfile errfile combined + max_attempts=$(_github_csma_max_attempts) + infile=$(mktemp) + outfile=$(mktemp) + errfile=$(mktemp) + cat >"${infile}" + # shellcheck disable=SC2064 + trap "rm -f '${infile}' '${outfile}' '${errfile}'" RETURN + + for (( attempt = 0; attempt < max_attempts; attempt++ )); do + github_csma_sense "${resource}" + github_csma_slot + + : >"${outfile}" + : >"${errfile}" + local rc=0 + gh "$@" <"${infile}" >"${outfile}" 2>"${errfile}" || rc=$? + + combined=$(cat "${outfile}" "${errfile}") + if github_csma_is_rate_limit "${combined}"; then + if (( attempt < max_attempts - 1 )); then + _github_csma_sleep_after_rate_limit "${attempt}" "${resource}" + continue + fi + _github_csma_emit_failure "${combined}" + return 1 + fi + + if (( rc != 0 )); then + _github_csma_emit_failure "${combined}" + return 1 + fi + cat "${outfile}" + return 0 + done + + return 1 +} + +# Run an arbitrary command with stdin from caller; retries on rate-limit errors in output. +# First argument: rate_limit resource (core|graphql); remaining args are the command. +github_csma_run_cmd() { + local resource="${1:-core}" + shift + + local max_attempts attempt infile outfile errfile combined + max_attempts=$(_github_csma_max_attempts) + infile=$(mktemp) + outfile=$(mktemp) + errfile=$(mktemp) + cat >"${infile}" + # shellcheck disable=SC2064 + trap "rm -f '${infile}' '${outfile}' '${errfile}'" RETURN + + for (( attempt = 0; attempt < max_attempts; attempt++ )); do + github_csma_sense "${resource}" + github_csma_slot + + : >"${outfile}" + : >"${errfile}" + local rc=0 + "$@" <"${infile}" >"${outfile}" 2>"${errfile}" || rc=$? + + combined=$(cat "${outfile}" "${errfile}") + if github_csma_is_rate_limit "${combined}"; then + if (( attempt < max_attempts - 1 )); then + _github_csma_sleep_after_rate_limit "${attempt}" "${resource}" + continue + fi + _github_csma_emit_failure "${combined}" + return 1 + fi + + if (( rc != 0 )); then + _github_csma_emit_failure "${combined}" + return 1 + fi + cat "${outfile}" + return 0 + done + + return 1 +} +# --- End inlined CSMA --- if github_csma_is_rate_limit "HTTP 429: Too Many Requests"; then : else diff --git a/internal/scaffold/fullsend-repo/scripts/post-prioritize.sh b/internal/scaffold/fullsend-repo/scripts/post-prioritize.sh index 5c57b2914b..58fa025bd3 100755 --- a/internal/scaffold/fullsend-repo/scripts/post-prioritize.sh +++ b/internal/scaffold/fullsend-repo/scripts/post-prioritize.sh @@ -12,9 +12,324 @@ set -euo pipefail -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=lib/github-api-csma.sh -source "${SCRIPT_DIR}/lib/github-api-csma.sh" +# --- Inlined from lib/github-api-csma.sh --- +# CSMA/CD-style resilience for GitHub API calls via gh/fullsend. +# Carrier sense: check rate_limit before transmitting. +# Slot time: random jitter between calls to desynchronize parallel runners. +# Collision detection: retry on 429 / secondary rate limit errors with exponential backoff. +# +# Environment (all optional): +# GITHUB_CSMA_MAX_ATTEMPTS — default 8 +# GITHUB_CSMA_MIN_REMAINING_CORE — default 100 +# GITHUB_CSMA_MIN_REMAINING_GRAPHQL — default 100 +# GITHUB_CSMA_SLOT_MIN_MS — default 250 +# GITHUB_CSMA_SLOT_MAX_MS — default 750 (0 disables jitter) +# GITHUB_CSMA_SPREAD_MAX_SEC — default 60 (post-reset desync spread) +# GITHUB_CSMA_BACKOFF_CAP_SEC — default 120 + +# shellcheck shell=bash + +[[ -n "${GITHUB_API_CSMA_SH_LOADED:-}" ]] && return 0 +GITHUB_API_CSMA_SH_LOADED=1 + +_github_csma_max_attempts() { + echo "${GITHUB_CSMA_MAX_ATTEMPTS:-8}" +} + +_github_csma_min_remaining() { + local resource="$1" + case "${resource}" in + graphql) echo "${GITHUB_CSMA_MIN_REMAINING_GRAPHQL:-100}" ;; + *) echo "${GITHUB_CSMA_MIN_REMAINING_CORE:-100}" ;; + esac +} + +_github_csma_slot_min_ms() { + echo "${GITHUB_CSMA_SLOT_MIN_MS:-250}" +} + +_github_csma_slot_max_ms() { + echo "${GITHUB_CSMA_SLOT_MAX_MS:-750}" +} + +_github_csma_spread_max_sec() { + echo "${GITHUB_CSMA_SPREAD_MAX_SEC:-60}" +} + +_github_csma_backoff_cap_sec() { + echo "${GITHUB_CSMA_BACKOFF_CAP_SEC:-120}" +} + +# Add a random spread delay after a rate-limit sleep to desynchronize runners. +# Called from both github_csma_sense and _github_csma_sleep_after_rate_limit. +_github_csma_post_reset_spread() { + local spread_max + spread_max=$(_github_csma_spread_max_sec) + if (( spread_max > 0 )); then + local spread_secs=$(( RANDOM % spread_max )) + echo "Rate limit reset — spreading ${spread_secs}s to desync from other runners..." >&2 + sleep "${spread_secs}" + fi +} + +_github_csma_emit_failure() { + printf '%s\n' "$1" >&2 +} + +# Wait until the named rate_limit resource has enough quota (carrier sense). +# Usage: github_csma_sense [core|graphql] [min_remaining] +github_csma_sense() { + local resource="${1:-core}" + local min_remaining="${2:-$(_github_csma_min_remaining "${resource}")}" + + local info remaining reset now wait_secs + if ! info=$(gh api rate_limit 2>/dev/null); then + echo "WARNING: github_csma_sense: could not read rate_limit; proceeding" >&2 + return 0 + fi + + remaining=$(echo "${info}" | jq -r --arg r "${resource}" '.resources[$r].remaining // empty') + reset=$(echo "${info}" | jq -r --arg r "${resource}" '.resources[$r].reset // empty') + + if [[ -z "${remaining}" || "${remaining}" == "null" ]]; then + echo "WARNING: github_csma_sense: no .resources.${resource} in rate_limit; proceeding" >&2 + return 0 + fi + + if (( remaining >= min_remaining )); then + return 0 + fi + + now=$(date +%s) + wait_secs=$(( reset - now + 1 )) + if (( wait_secs < 1 )); then + wait_secs=1 + fi + cap=$(_github_csma_backoff_cap_sec) + if (( wait_secs > cap )); then + wait_secs="${cap}" + fi + + echo "Rate limit sense: ${resource} remaining=${remaining} (min=${min_remaining}); waiting ${wait_secs}s until reset..." >&2 + sleep "${wait_secs}" + + # After a rate-limit sleep, all runners wake at the same reset timestamp. + # Spread them over a wide window to avoid a thundering herd. + _github_csma_post_reset_spread +} + +# Random inter-call delay (slot time) to reduce synchronized collisions. +github_csma_slot() { + local max_ms min_ms span_ms delay_ms + max_ms=$(_github_csma_slot_max_ms) + if (( max_ms <= 0 )); then + return 0 + fi + min_ms=$(_github_csma_slot_min_ms) + if (( min_ms > max_ms )); then + min_ms="${max_ms}" + fi + span_ms=$(( max_ms - min_ms + 1 )) + delay_ms=$(( min_ms + RANDOM % span_ms )) + sleep "$(awk -v ms="${delay_ms}" 'BEGIN { printf "%.3f", ms / 1000 }')" +} + +# Return 0 if combined output looks like a retryable GitHub rate limit error. +github_csma_is_rate_limit() { + local text="$1" + local lower + lower=$(echo "${text}" | tr '[:upper:]' '[:lower:]') + + if echo "${lower}" | grep -qE 'http 429|status: 429'; then + return 0 + fi + if echo "${lower}" | grep -qE 'secondary rate limit|rate limit exceeded|api rate limit'; then + return 0 + fi + if echo "${lower}" | grep -qE 'http 403|status: 403'; then + if echo "${lower}" | grep -qE 'secondary|rate limit|abuse|retry.after'; then + return 0 + fi + fi + return 1 +} + +# Compute backoff seconds for attempt (0-based). Writes to stdout. +github_csma_backoff() { + local attempt="$1" + local cap base delay + cap=$(_github_csma_backoff_cap_sec) + base=$(( 1 << attempt )) + if (( base > cap )); then + base="${cap}" + fi + delay=$(( RANDOM % (base + 1) )) + if (( delay < 1 )); then + delay=1 + fi + echo "${delay}" +} + +_github_csma_sleep_after_rate_limit() { + local attempt="$1" + local resource="${2:-core}" + local delay wait_secs now reset info cap + + delay=$(github_csma_backoff "${attempt}") + if info=$(gh api rate_limit 2>/dev/null); then + now=$(date +%s) + reset=$(echo "${info}" | jq -r --arg r "${resource}" '.resources[$r].reset // empty') + if [[ -n "${reset}" && "${reset}" != "null" ]]; then + wait_secs=$(( reset - now + 1 )) + cap=$(_github_csma_backoff_cap_sec) + if (( wait_secs > cap )); then + wait_secs="${cap}" + fi + if (( wait_secs > delay && wait_secs > 0 )); then + delay="${wait_secs}" + fi + fi + fi + echo "GitHub API rate limit (attempt $(( attempt + 1 ))); backing off ${delay}s..." >&2 + sleep "${delay}" + + # After backing off, spread runners to avoid thundering herd on wake. + _github_csma_post_reset_spread +} + +# Run gh with CSMA/CD. First argument: rate_limit resource (core|graphql). +# Remaining arguments are passed to gh. Prints gh stdout on success. +github_csma_run() { + local resource="${1:-core}" + shift + + local max_attempts attempt outfile errfile combined + max_attempts=$(_github_csma_max_attempts) + outfile=$(mktemp) + errfile=$(mktemp) + # shellcheck disable=SC2064 + trap "rm -f '${outfile}' '${errfile}'" RETURN + + for (( attempt = 0; attempt < max_attempts; attempt++ )); do + github_csma_sense "${resource}" + github_csma_slot + + : >"${outfile}" + : >"${errfile}" + local rc=0 + gh "$@" >"${outfile}" 2>"${errfile}" || rc=$? + + combined=$(cat "${outfile}" "${errfile}") + if github_csma_is_rate_limit "${combined}"; then + if (( attempt < max_attempts - 1 )); then + _github_csma_sleep_after_rate_limit "${attempt}" "${resource}" + continue + fi + _github_csma_emit_failure "${combined}" + return 1 + fi + + if (( rc != 0 )); then + _github_csma_emit_failure "${combined}" + return 1 + fi + cat "${outfile}" + return 0 + done + + return 1 +} + +# Run producer | gh with CSMA/CD. First argument: resource; rest are gh args. +# Reads producer output from stdin (save once for retries). +github_csma_run_pipe() { + local resource="${1:-graphql}" + shift + + local max_attempts attempt infile outfile errfile combined + max_attempts=$(_github_csma_max_attempts) + infile=$(mktemp) + outfile=$(mktemp) + errfile=$(mktemp) + cat >"${infile}" + # shellcheck disable=SC2064 + trap "rm -f '${infile}' '${outfile}' '${errfile}'" RETURN + + for (( attempt = 0; attempt < max_attempts; attempt++ )); do + github_csma_sense "${resource}" + github_csma_slot + + : >"${outfile}" + : >"${errfile}" + local rc=0 + gh "$@" <"${infile}" >"${outfile}" 2>"${errfile}" || rc=$? + + combined=$(cat "${outfile}" "${errfile}") + if github_csma_is_rate_limit "${combined}"; then + if (( attempt < max_attempts - 1 )); then + _github_csma_sleep_after_rate_limit "${attempt}" "${resource}" + continue + fi + _github_csma_emit_failure "${combined}" + return 1 + fi + + if (( rc != 0 )); then + _github_csma_emit_failure "${combined}" + return 1 + fi + cat "${outfile}" + return 0 + done + + return 1 +} + +# Run an arbitrary command with stdin from caller; retries on rate-limit errors in output. +# First argument: rate_limit resource (core|graphql); remaining args are the command. +github_csma_run_cmd() { + local resource="${1:-core}" + shift + + local max_attempts attempt infile outfile errfile combined + max_attempts=$(_github_csma_max_attempts) + infile=$(mktemp) + outfile=$(mktemp) + errfile=$(mktemp) + cat >"${infile}" + # shellcheck disable=SC2064 + trap "rm -f '${infile}' '${outfile}' '${errfile}'" RETURN + + for (( attempt = 0; attempt < max_attempts; attempt++ )); do + github_csma_sense "${resource}" + github_csma_slot + + : >"${outfile}" + : >"${errfile}" + local rc=0 + "$@" <"${infile}" >"${outfile}" 2>"${errfile}" || rc=$? + + combined=$(cat "${outfile}" "${errfile}") + if github_csma_is_rate_limit "${combined}"; then + if (( attempt < max_attempts - 1 )); then + _github_csma_sleep_after_rate_limit "${attempt}" "${resource}" + continue + fi + _github_csma_emit_failure "${combined}" + return 1 + fi + + if (( rc != 0 )); then + _github_csma_emit_failure "${combined}" + return 1 + fi + cat "${outfile}" + return 0 + done + + return 1 +} +# --- End inlined CSMA --- : "${GITHUB_ISSUE_URL:?GITHUB_ISSUE_URL must be set}" : "${GH_TOKEN:?GH_TOKEN must be set}" diff --git a/internal/scaffold/scaffold_test.go b/internal/scaffold/scaffold_test.go index ee0177f63b..c70055f1f9 100644 --- a/internal/scaffold/scaffold_test.go +++ b/internal/scaffold/scaffold_test.go @@ -89,7 +89,6 @@ func TestFullsendRepoFilesExist(t *testing.T) { "scripts/setup-prioritize.sh", "scripts/pre-prioritize.sh", "scripts/post-prioritize.sh", - "scripts/lib/github-api-csma.sh", "scripts/post-prioritize-test.sh", ".github/workflows/prioritize.yml", ".github/workflows/prioritize-scheduler.yml",