From df8a16bc55ccd2cb89a259cc5cf8026f27a4e0c7 Mon Sep 17 00:00:00 2001 From: marcusquinn <6428977+marcusquinn@users.noreply.github.com> Date: Fri, 6 Mar 2026 05:16:26 +0000 Subject: [PATCH] fix: review-bot-gate distinguishes rate-limit notices from real reviews (#2980) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When bots are rate-limited, they post quota/rate-limit notices instead of actual code reviews. The gate was counting these as valid reviews (PASS) because it only checked for comment existence, not content. Now both the helper script and CI workflow: - Base64-encode comment bodies to handle multi-line content correctly - Check each bot comment against known rate-limit patterns - Only count comments as real reviews if they don't match rate-limit patterns - Return WAITING (not PASS) when all bot comments are rate-limit notices - Show rate-limited bots separately in output and CI summary Tested against PR #2978 (rate-limited → WAITING) and PR #2933 (real review → PASS). --- .agents/scripts/review-bot-gate-helper.sh | 87 +++++++++++++++++- .github/workflows/review-bot-gate.yml | 106 +++++++++++++++++++--- 2 files changed, 181 insertions(+), 12 deletions(-) diff --git a/.agents/scripts/review-bot-gate-helper.sh b/.agents/scripts/review-bot-gate-helper.sh index 90228470db..40e9f76c4a 100755 --- a/.agents/scripts/review-bot-gate-helper.sh +++ b/.agents/scripts/review-bot-gate-helper.sh @@ -33,6 +33,17 @@ KNOWN_BOTS=( "copilot" ) +# Patterns that indicate a rate-limit or quota notice (not a real review). +# Case-insensitive grep patterns — one per line. +RATE_LIMIT_PATTERNS=( + "rate limit exceeded" + "rate limited by coderabbit" + "daily quota limit" + "reached your daily quota" + "Please wait up to 24 hours" + "has exceeded the limit for the number of" +) + SKIP_LABEL="skip-review-gate" # --- Functions --- @@ -72,6 +83,57 @@ get_all_bot_commenters() { tr '[:upper:]' '[:lower:]' | sort -u | grep -v '^$' || true } +is_rate_limit_comment() { + # Check if a comment body matches any known rate-limit/quota pattern. + # Returns 0 if the comment IS a rate-limit notice (not a real review). + local body="$1" + + for pattern in "${RATE_LIMIT_PATTERNS[@]}"; do + if echo "$body" | grep -qi "$pattern"; then + return 0 + fi + done + return 1 +} + +bot_has_real_review() { + # Check if a bot has posted at least one comment that is NOT a rate-limit + # notice. Checks all three comment sources (reviews, issue comments, + # review comments). Returns 0 if a real review exists, 1 otherwise. + local pr_number="$1" + local repo="$2" + local bot_login="$3" + + # Build a jq filter that selects comments by this bot (case-insensitive) + # and base64-encodes each body so multi-line content stays on one line. + local jq_filter + jq_filter=".[] | select(.user.login | ascii_downcase | test(\"${bot_login}\")) | .body | @base64" + + local api_endpoints=( + "repos/${repo}/pulls/${pr_number}/reviews" + "repos/${repo}/issues/${pr_number}/comments" + "repos/${repo}/pulls/${pr_number}/comments" + ) + + local endpoint encoded_bodies body + for endpoint in "${api_endpoints[@]}"; do + encoded_bodies=$(gh api "$endpoint" --paginate --jq "$jq_filter" 2>/dev/null || echo "") + if [[ -n "$encoded_bodies" ]]; then + while IFS= read -r encoded; do + [[ -z "$encoded" ]] && continue + body=$(echo "$encoded" | base64 -d 2>/dev/null || echo "") + [[ -z "$body" ]] && continue + if ! is_rate_limit_comment "$body"; then + return 0 + fi + done <<<"$encoded_bodies" + fi + done + + # All comments from this bot were rate-limit notices (or empty) + return 1 +} + check_for_skip_label() { local pr_number="$1" local repo="$2" @@ -117,9 +179,16 @@ do_check() { all_commenters=$(get_all_bot_commenters "$pr_number" "$repo") local found_bots="" + local rate_limited_bots="" for bot in "${KNOWN_BOTS[@]}"; do if echo "$all_commenters" | grep -qi "$bot"; then - found_bots="${found_bots}${bot} " + # Bot commented — but is it a real review or a rate-limit notice? + if bot_has_real_review "$pr_number" "$repo" "$bot"; then + found_bots="${found_bots}${bot} " + else + rate_limited_bots="${rate_limited_bots}${bot} " + echo "rate-limited (not a real review): ${bot}" >&2 + fi fi done @@ -127,6 +196,10 @@ do_check() { echo "PASS" echo "found: ${found_bots}" >&2 return 0 + elif [[ -n "$rate_limited_bots" ]]; then + echo "WAITING" + echo "Bots posted rate-limit notices only (not real reviews): ${rate_limited_bots}" >&2 + return 1 else echo "WAITING" echo "No review bots found yet. Known bots: ${KNOWN_BOTS[*]}" >&2 @@ -176,6 +249,18 @@ do_list() { local result result=$(match_known_bots "$all_commenters") echo "$result" + echo "" + + # Show rate-limit status for each found bot + for bot in "${KNOWN_BOTS[@]}"; do + if echo "$all_commenters" | grep -qi "$bot"; then + if bot_has_real_review "$pr_number" "$repo" "$bot"; then + echo " ${bot}: real review" + else + echo " ${bot}: rate-limited (no real review)" + fi + fi + done return 0 } diff --git a/.github/workflows/review-bot-gate.yml b/.github/workflows/review-bot-gate.yml index 24d210f743..ad669aa1f1 100644 --- a/.github/workflows/review-bot-gate.yml +++ b/.github/workflows/review-bot-gate.yml @@ -62,6 +62,17 @@ jobs: "copilot[bot]" ) + # Patterns indicating rate-limit/quota notices (not real reviews). + # GH#2980: bots post these when rate-limited — must not count as reviews. + RATE_LIMIT_PATTERNS=( + "rate limit exceeded" + "rate limited by coderabbit" + "daily quota limit" + "reached your daily quota" + "Please wait up to 24 hours" + "has exceeded the limit for the number of" + ) + # Minimum wait time (seconds) after PR creation before we check # Gives bots time to start their analysis MIN_WAIT_SECONDS=120 @@ -81,6 +92,46 @@ jobs: echo "::warning::PR is only ${ELAPSED}s old (minimum ${MIN_WAIT_SECONDS}s). Review bots may not have posted yet." fi + # Helper: check if a comment body is a rate-limit notice + is_rate_limit_comment() { + local body="$1" + for pattern in "${RATE_LIMIT_PATTERNS[@]}"; do + if echo "$body" | grep -qi "$pattern"; then + return 0 + fi + done + return 1 + } + + # Helper: check if a bot has at least one real review (not rate-limited) + # Uses jq to select comments by bot login and extract bodies. + # Base64-encodes each body so multi-line content stays on one line. + bot_has_real_review() { + local bot_base="$1" + local jq_filter + jq_filter=".[] | select(.user.login | ascii_downcase | test(\"${bot_base}\")) | .body | @base64" + + local source encoded_bodies encoded body + for source in \ + "repos/${REPO}/pulls/${PR_NUMBER}/reviews" \ + "repos/${REPO}/issues/${PR_NUMBER}/comments" \ + "repos/${REPO}/pulls/${PR_NUMBER}/comments"; do + + encoded_bodies=$(gh api "$source" --paginate --jq "$jq_filter" 2>/dev/null || echo "") + if [[ -n "$encoded_bodies" ]]; then + while IFS= read -r encoded; do + [[ -z "$encoded" ]] && continue + body=$(echo "$encoded" | base64 -d 2>/dev/null || echo "") + [[ -z "$body" ]] && continue + if ! is_rate_limit_comment "$body"; then + return 0 + fi + done <<< "$encoded_bodies" + fi + done + return 1 + } + # Check PR reviews (formal GitHub reviews from bots) echo "" echo "=== Checking PR reviews ===" @@ -105,9 +156,11 @@ jobs: echo "All commenters found:" echo "$ALL_COMMENTERS" | grep -v '^$' | sed 's/^/ - /' - # Check which known bots have posted + # Check which known bots have posted REAL reviews (not rate-limit notices) + # GH#2980: bots posting rate-limit notices were incorrectly counted as reviews FOUND_BOTS="" MISSING_BOTS="" + RATE_LIMITED_BOTS="" for bot in "${KNOWN_BOTS[@]}"; do # Use grep pattern matching (bot patterns may contain [bot] suffix) @@ -116,8 +169,14 @@ jobs: bot_base=$(echo "$bot_lower" | sed 's/\[bot\]$//') if echo "$ALL_COMMENTERS" | grep -qi "$bot_base"; then - FOUND_BOTS="${FOUND_BOTS}${bot} " - echo "FOUND: ${bot}" + # Bot commented — verify it's a real review, not a rate-limit notice + if bot_has_real_review "$bot_base"; then + FOUND_BOTS="${FOUND_BOTS}${bot} " + echo "FOUND (real review): ${bot}" + else + RATE_LIMITED_BOTS="${RATE_LIMITED_BOTS}${bot} " + echo "FOUND (rate-limited, not a real review): ${bot}" + fi else MISSING_BOTS="${MISSING_BOTS}${bot} " echo "MISSING: ${bot}" @@ -125,12 +184,13 @@ jobs: done echo "" - echo "Found bots: ${FOUND_BOTS:-none}" + echo "Found bots (real reviews): ${FOUND_BOTS:-none}" + echo "Rate-limited bots: ${RATE_LIMITED_BOTS:-none}" echo "Missing bots: ${MISSING_BOTS:-none}" - # Gate logic: require at least ONE known bot to have reviewed + # Gate logic: require at least ONE known bot to have posted a REAL review # This is intentionally lenient — not all repos have all bots. - # The goal is to catch the case where NO bots have reviewed at all. + # The goal is to catch the case where NO bots have actually reviewed. # # Race condition fix: on pull_request events (opened/synchronize/reopened), # the workflow fires before bots have had time to analyze. If no bots are @@ -139,10 +199,11 @@ jobs: # Only hard-fail when the PR is old enough that bots should have posted. if [[ -n "$FOUND_BOTS" ]]; then echo "" - echo "PASS: At least one AI review bot has posted feedback." + echo "PASS: At least one AI review bot has posted a real review." echo "gate_passed=true" >> "$GITHUB_OUTPUT" echo "found_bots=${FOUND_BOTS}" >> "$GITHUB_OUTPUT" echo "missing_bots=${MISSING_BOTS}" >> "$GITHUB_OUTPUT" + echo "rate_limited_bots=${RATE_LIMITED_BOTS}" >> "$GITHUB_OUTPUT" elif [[ "$ELAPSED" -lt "$MIN_WAIT_SECONDS" ]]; then echo "" echo "PASS (pending): PR is ${ELAPSED}s old (< ${MIN_WAIT_SECONDS}s minimum)." @@ -151,9 +212,14 @@ jobs: echo "gate_passed=true" >> "$GITHUB_OUTPUT" echo "found_bots=" >> "$GITHUB_OUTPUT" echo "missing_bots=${MISSING_BOTS}" >> "$GITHUB_OUTPUT" + echo "rate_limited_bots=${RATE_LIMITED_BOTS}" >> "$GITHUB_OUTPUT" else echo "" - echo "WAITING: No AI review bots have posted yet." + if [[ -n "$RATE_LIMITED_BOTS" ]]; then + echo "WAITING: Bots posted rate-limit notices only (not real reviews): ${RATE_LIMITED_BOTS}" + else + echo "WAITING: No AI review bots have posted yet." + fi echo "This check will re-run when a bot posts a review or comment." echo "" echo "Expected bots: ${KNOWN_BOTS[*]}" @@ -163,6 +229,7 @@ jobs: echo "gate_passed=false" >> "$GITHUB_OUTPUT" echo "found_bots=" >> "$GITHUB_OUTPUT" echo "missing_bots=${MISSING_BOTS}" >> "$GITHUB_OUTPUT" + echo "rate_limited_bots=${RATE_LIMITED_BOTS}" >> "$GITHUB_OUTPUT" fi - name: Check for skip label @@ -186,10 +253,18 @@ jobs: - name: Gate result if: steps.check.outputs.gate_passed != 'true' && steps.skip.outputs.skip != 'true' run: | - echo "::error::No AI review bots have posted on this PR yet." + if [[ -n "${{ steps.check.outputs.rate_limited_bots }}" ]]; then + echo "::error::Review bots posted rate-limit notices only — no real reviews." + echo "" + echo "Rate-limited bots: ${{ steps.check.outputs.rate_limited_bots }}" + echo "" + echo "The bots are rate-limited and did not perform actual code review." + else + echo "::error::No AI review bots have posted on this PR yet." + fi echo "" echo "This PR cannot be merged until at least one AI code review bot" - echo "(CodeRabbit, Gemini Code Assist, etc.) has posted its review." + echo "(CodeRabbit, Gemini Code Assist, etc.) has posted a real review." echo "" echo "What to do:" echo " 1. Wait a few minutes for bots to complete their analysis" @@ -209,6 +284,10 @@ jobs: echo "**Status**: PASSED" echo "" echo "**Bots that reviewed**: ${{ steps.check.outputs.found_bots }}" + if [[ -n "${{ steps.check.outputs.rate_limited_bots }}" ]]; then + echo "" + echo "**Bots rate-limited (not counted)**: ${{ steps.check.outputs.rate_limited_bots }}" + fi if [[ -n "${{ steps.check.outputs.missing_bots }}" ]]; then echo "" echo "**Bots not yet reviewed**: ${{ steps.check.outputs.missing_bots }}" @@ -223,7 +302,12 @@ jobs: else echo "**Status**: WAITING" echo "" - echo "No AI review bots have posted yet. This check will re-run" + if [[ -n "${{ steps.check.outputs.rate_limited_bots }}" ]]; then + echo "Bots posted rate-limit notices only (not real reviews):" + echo "${{ steps.check.outputs.rate_limited_bots }}" + echo "" + fi + echo "No real AI review bot feedback yet. This check will re-run" echo "automatically when a bot posts a review or comment." fi } >> "$GITHUB_STEP_SUMMARY"