diff --git a/docs/reviewer-report.md b/docs/reviewer-report.md index 881b2c309..2dbdd7918 100644 --- a/docs/reviewer-report.md +++ b/docs/reviewer-report.md @@ -61,7 +61,7 @@ reviews on a single PR). | Metric | Definition | |---|---| | **Total PRs** | Review-eligible (non-draft) PRs active in the window; the denominator each row is measured against. | -| **Reviews** | Count of real review submissions by the bot, **each occurrence** (multiple per PR from multiple commits all count). Rate-limit notices are never counted as reviews. | +| **Reviews** | Count of reviews the bot submitted, **each occurrence** (multiple per PR from multiple commits all count). A bot that posts no formal review but delivers its verdict as a top-level comment (e.g. SonarCloud's quality-gate comment) has that comment counted as its review; bots that do submit formal reviews are unaffected, so their extra summary comments are never double-counted. Rate-limit notices are never counted. | | **✅ / 🔄** | Of those reviews, how many carried state APPROVED / CHANGES_REQUESTED. | | **Rate-limited** | Count of out-of-quota / rate-limit refusal events (detected by body text via the shared gate pattern), each occurrence. | | **No response** | Eligible PRs the bot never engaged with at all — no review, comment, or refusal. | @@ -72,8 +72,6 @@ reviews on a single PR). Per-PR bucket counts (`reviewed_prs` / `refused_prs` / `no_response_prs`, which partition the eligible PRs), plus thread-resolution and reaction counts, are still computed and stored in the snapshot for downstream use — they're just kept out of the headline table to keep it legible. -SonarCloud posts status-check comments rather than GitHub reviews, so its **Reviews** reads 0 -by design. ### Week-over-week deltas diff --git a/scripts/reviewer_report.sh b/scripts/reviewer_report.sh index 8cb3702f4..537809aa7 100755 --- a/scripts/reviewer_report.sh +++ b/scripts/reviewer_report.sh @@ -187,6 +187,7 @@ _NORMALIZE_JQ=' | ($mine | map(select(.refusal | not))) as $real | ($mine | map(select(.refusal))) as $refd | ($real | map(.at) | min) as $first_real + | ($real | map(select(.kind=="review")) | length) as $rev_events | { kind: "bot_pr", repo: $repo, pr: ($pr.url), bot: $grp[0].bot, created: $pr.createdAt, real_responses: ($real | length), @@ -195,7 +196,14 @@ _NORMALIZE_JQ=' # so refusals never pollute the latency percentiles). latency_s: (if ($real | length) == 0 then null else (($first_real | fromdateiso8601) - ($pr.createdAt | fromdateiso8601)) end), - reviews: ($real | map(select(.kind=="review")) | length), + # Reviews = formal review submissions. A bot that posts NO formal review + # but delivers its verdict as a top-level comment (e.g. SonarCloud'"'"'s + # quality-gate comment) has that comment counted as its review, so its + # work is not invisible. Bots that DO submit reviews are unaffected, so + # their extra summary comments are never double-counted. Refusals are + # already excluded ($real holds only non-refusal responses). + reviews: (if $rev_events > 0 then $rev_events + else ($real | map(select(.kind=="issue")) | length) end), approved: ($real | map(select(.kind=="review" and .state=="APPROVED")) | length), changes_req: ($real | map(select(.kind=="review" and .state=="CHANGES_REQUESTED")) | length), inline_comments: ($real | map(select(.kind=="inline")) | length), @@ -335,11 +343,10 @@ render_reviewer_report() { done printf '\n' printf -- '- **Total PRs** — review-eligible (non-draft) PRs in the window; the denominator each row is measured against.\n' - printf -- '- **Reviews** — count of real reviews the bot submitted, **each occurrence** (a PR re-reviewed on 5 commits = 5). Rate-limit notices are excluded.\n' - printf -- '- **✅ / 🔄** — of those reviews, how many were APPROVED / CHANGES_REQUESTED.\n' + printf -- '- **Reviews** — count of reviews the bot submitted, **each occurrence** (a PR re-reviewed on 5 commits = 5). A bot that posts no formal review but delivers its verdict as a comment (e.g. SonarCloud'"'"'s quality-gate comment) has that comment counted as its review. Rate-limit notices are never counted.\n' + printf -- '- **✅ / 🔄** — of those reviews, how many were APPROVED / CHANGES_REQUESTED (comment-only responses carry no verdict).\n' printf -- '- **Rate-limited** — count of out-of-quota / rate-limit refusals, each occurrence.\n' - printf -- '- **No response** — eligible PRs the bot never engaged with at all (no review, comment, or refusal).\n' - printf -- '- _Note: SonarCloud posts status-check comments rather than GitHub reviews, so its Reviews count reads 0 by design._\n\n' + printf -- '- **No response** — eligible PRs the bot never engaged with at all (no review, comment, or refusal).\n\n' # ---- Coverage / overlap -------------------------------------------------- local co_review multi_bot diff --git a/tests/reviewer_report.bats b/tests/reviewer_report.bats index b6f0bf93c..9ec76cd60 100644 --- a/tests/reviewer_report.bats +++ b/tests/reviewer_report.bats @@ -97,6 +97,31 @@ JSON echo "$output" | jq -e '.[] | select(.bot=="copilot-pull-request-reviewer") | .thumbs_up==2 and .thumbs_down==1 and .threads_resolved==1' } +@test "reviews: a comment-only responder (SonarCloud) has its comment counted as a review" { + tmp="$(mktemp "$BATS_TEST_TMPDIR/tmp.XXXXXX")" + cat > "$tmp" <<'JSON' +{"url":"u","createdAt":"2026-07-10T10:00:00Z","updatedAt":"2026-07-10T10:00:00Z","mergedAt":null,"isDraft":false,"author":{"login":"h"}, + "reviews":{"nodes":[]}, + "reviewThreads":{"nodes":[]}, + "comments":{"nodes":[{"author":{"login":"sonarqubecloud"},"createdAt":"2026-07-10T10:02:00Z","bodyText":"Quality Gate passed"}]}} +JSON + run jq -c --arg repo "r" --argjson bots "$BOTS" --arg rl "$RATE_LIMIT_RE" "[ $_NORMALIZE_JQ ]" "$tmp" + echo "$output" | jq -e '.[] | select(.bot=="sonarqubecloud") | .reviews==1 and .real_responses==1' +} + +@test "reviews: a bot with a formal review does NOT also count its summary comment (no double-count)" { + tmp="$(mktemp "$BATS_TEST_TMPDIR/tmp.XXXXXX")" + cat > "$tmp" <<'JSON' +{"url":"u","createdAt":"2026-07-10T10:00:00Z","updatedAt":"2026-07-10T10:00:00Z","mergedAt":null,"isDraft":false,"author":{"login":"h"}, + "reviews":{"nodes":[{"author":{"login":"coderabbitai"},"state":"COMMENTED","submittedAt":"2026-07-10T10:05:00Z","bodyText":"formal review"}]}, + "reviewThreads":{"nodes":[]}, + "comments":{"nodes":[{"author":{"login":"coderabbitai"},"createdAt":"2026-07-10T10:01:00Z","bodyText":"Walkthrough summary comment"}]}} +JSON + run jq -c --arg repo "r" --argjson bots "$BOTS" --arg rl "$RATE_LIMIT_RE" "[ $_NORMALIZE_JQ ]" "$tmp" + # 1 formal review only — the summary issue comment is not counted as a second review. + echo "$output" | jq -e '.[] | select(.bot=="coderabbitai") | .reviews==1' +} + # --------------------------------------------------------------------------- # aggregate_snapshot # ---------------------------------------------------------------------------