From 39335444b65e95442b25e640acecee8870deb711 Mon Sep 17 00:00:00 2001 From: Ralph Bean Date: Sat, 30 May 2026 21:43:57 -0400 Subject: [PATCH] fix: check CSMA output for rate limit errors even on exit 0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gh CLI (notably `gh project view`) can exit 0 while printing a GraphQL rate limit error to stderr. The CSMA wrapper previously only entered the retry loop on non-zero exit codes, so these errors passed through undetected — downstream `jq` would fail to parse the error text as JSON and `set -e` would kill the script with no retries. Now all three CSMA runners (github_csma_run, github_csma_run_pipe, github_csma_run_cmd) always inspect combined stdout+stderr for rate limit patterns regardless of exit code. Fixes: https://github.com/fullsend-ai/.fullsend/actions/runs/26447310264 Assisted-by: Claude Opus 4.6 Signed-off-by: Ralph Bean --- .../scripts/lib/github-api-csma.sh | 48 ++++++++++++------- .../scripts/post-prioritize-test.sh | 24 +++++++++- 2 files changed, 53 insertions(+), 19 deletions(-) diff --git a/internal/scaffold/fullsend-repo/scripts/lib/github-api-csma.sh b/internal/scaffold/fullsend-repo/scripts/lib/github-api-csma.sh index fc10b0d26b..a281397e28 100644 --- a/internal/scaffold/fullsend-repo/scripts/lib/github-api-csma.sh +++ b/internal/scaffold/fullsend-repo/scripts/lib/github-api-csma.sh @@ -182,10 +182,8 @@ github_csma_run() { : >"${outfile}" : >"${errfile}" - if gh "$@" >"${outfile}" 2>"${errfile}"; then - cat "${outfile}" - return 0 - fi + local rc=0 + gh "$@" >"${outfile}" 2>"${errfile}" || rc=$? combined=$(cat "${outfile}" "${errfile}") if github_csma_is_rate_limit "${combined}"; then @@ -193,10 +191,16 @@ github_csma_run() { _github_csma_sleep_after_rate_limit "${attempt}" "${resource}" continue fi + _github_csma_emit_failure "${combined}" + return 1 fi - _github_csma_emit_failure "${combined}" - return 1 + if (( rc != 0 )); then + _github_csma_emit_failure "${combined}" + return 1 + fi + cat "${outfile}" + return 0 done return 1 @@ -223,10 +227,8 @@ github_csma_run_pipe() { : >"${outfile}" : >"${errfile}" - if gh "$@" <"${infile}" >"${outfile}" 2>"${errfile}"; then - cat "${outfile}" - return 0 - fi + local rc=0 + gh "$@" <"${infile}" >"${outfile}" 2>"${errfile}" || rc=$? combined=$(cat "${outfile}" "${errfile}") if github_csma_is_rate_limit "${combined}"; then @@ -234,10 +236,16 @@ github_csma_run_pipe() { _github_csma_sleep_after_rate_limit "${attempt}" "${resource}" continue fi + _github_csma_emit_failure "${combined}" + return 1 fi - _github_csma_emit_failure "${combined}" - return 1 + if (( rc != 0 )); then + _github_csma_emit_failure "${combined}" + return 1 + fi + cat "${outfile}" + return 0 done return 1 @@ -264,10 +272,8 @@ github_csma_run_cmd() { : >"${outfile}" : >"${errfile}" - if "$@" <"${infile}" >"${outfile}" 2>"${errfile}"; then - cat "${outfile}" - return 0 - fi + local rc=0 + "$@" <"${infile}" >"${outfile}" 2>"${errfile}" || rc=$? combined=$(cat "${outfile}" "${errfile}") if github_csma_is_rate_limit "${combined}"; then @@ -275,10 +281,16 @@ github_csma_run_cmd() { _github_csma_sleep_after_rate_limit "${attempt}" "${resource}" continue fi + _github_csma_emit_failure "${combined}" + return 1 fi - _github_csma_emit_failure "${combined}" - return 1 + if (( rc != 0 )); then + _github_csma_emit_failure "${combined}" + return 1 + fi + cat "${outfile}" + return 0 done return 1 diff --git a/internal/scaffold/fullsend-repo/scripts/post-prioritize-test.sh b/internal/scaffold/fullsend-repo/scripts/post-prioritize-test.sh index d966a72d89..435797eb30 100755 --- a/internal/scaffold/fullsend-repo/scripts/post-prioritize-test.sh +++ b/internal/scaffold/fullsend-repo/scripts/post-prioritize-test.sh @@ -47,6 +47,15 @@ if [[ "\${GH_CSMA_FAIL_MODE:-}" == "auth" ]] && [[ "\$1" != "api" || "\$2" != "r exit 1 fi +# Simulate gh exiting 0 but printing a rate limit error to stdout. +# This is how "gh project view" behaves on GraphQL rate limits. +if [[ "\${GH_CSMA_FAIL_MODE:-}" == "exit0-ratelimit" ]] && [[ "\$1" != "api" || "\$2" != "rate_limit" ]]; then + if (( fail_count <= GH_CSMA_FAIL_UNTIL )); then + echo "GraphQL: API rate limit exceeded for installation ID 131739396." >&2 + exit 0 + fi +fi + if [[ -n "\${GH_CSMA_FAIL_UNTIL:-}" ]] && (( fail_count <= GH_CSMA_FAIL_UNTIL )); then echo "You have exceeded a secondary rate limit. Please retry again later." >&2 exit 1 @@ -146,6 +155,7 @@ run_test() { local fail_until="${2:-}" local min_gh_calls="${3:-1}" local expect_failure="${4:-false}" + local fail_mode="${5:-}" local run_dir="${TEST_TMPDIR}/run-${test_name}" mkdir -p "${run_dir}/iteration-1/output" @@ -153,10 +163,13 @@ run_test() { > "${GH_LOG}" rm -f "${GH_FAIL_COUNT}" - unset GH_CSMA_FAIL_UNTIL + unset GH_CSMA_FAIL_UNTIL GH_CSMA_FAIL_MODE if [[ -n "${fail_until}" ]]; then export GH_CSMA_FAIL_UNTIL="${fail_until}" fi + if [[ -n "${fail_mode}" ]]; then + export GH_CSMA_FAIL_MODE="${fail_mode}" + fi local exit_code=0 (cd "${run_dir}" && bash "${POST_SCRIPT}") > "${TEST_TMPDIR}/stdout-${test_name}.log" 2>&1 || exit_code=$? @@ -294,6 +307,15 @@ export GITHUB_CSMA_MAX_ATTEMPTS=3 run_test_failure_stderr "exhausted-retries" "100" "secondary rate limit" unset GITHUB_CSMA_MAX_ATTEMPTS +# gh exits 0 but output contains rate limit error (gh project view behavior). +# First 2 calls fail with exit-0 rate limit, then succeed. +run_test "exit0-rate-limit-retry" "2" 10 "false" "exit0-ratelimit" + +# Exhausted retries when gh keeps exiting 0 with rate limit errors. +export GITHUB_CSMA_MAX_ATTEMPTS=3 +run_test_failure_stderr "exit0-rate-limit-exhausted" "100" "rate limit exceeded" "exit0-ratelimit" +unset GITHUB_CSMA_MAX_ATTEMPTS + if [[ ${FAILURES} -gt 0 ]]; then echo "" echo "${FAILURES} test(s) failed."