Skip to content

feat: support multiple AI code reviewers in PR loop #295

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
alex-solovyev merged 6 commits into from
Feb 3, 2026
Merged

feat: support multiple AI code reviewers in PR loop #295

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
da23954
feat: support multiple AI code reviewers in PR loop
alex-solovyev Feb 3, 2026
caa20a1
fix: address AI reviewer feedback on PR #295
alex-solovyev Feb 3, 2026
0563e51
fix: address second round of AI reviewer feedback
alex-solovyev Feb 3, 2026
c45acee
fix: use GraphQL reviewThreads.isResolved for accurate thread resolution
alex-solovyev Feb 3, 2026
69c0b53
fix: add shellcheck directive for GraphQL query false positive
alex-solovyev Feb 3, 2026
4e604f3
fix: treat ACTION_REQUIRED check conclusion as blocking
alex-solovyev Feb 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
116 changes: 97 additions & 19 deletions .agent/scripts/quality-loop-helper.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,10 @@ readonly FAST_SERVICES="codefactor|version|framework"
readonly MEDIUM_SERVICES="sonarcloud|codacy|qlty|code-review-monitoring"
readonly SLOW_SERVICES="coderabbit|coderabbitai"

# AI code reviewers (regex pattern for jq test() - anchored to prevent false positives)
# Supported: CodeRabbit, Gemini Code Assist, Augment Code, GitHub Copilot
readonly AI_REVIEWERS="^coderabbit|^gemini-code-assist\\[bot\\]$|^augment-code\\[bot\\]$|^augmentcode\\[bot\\]$|^copilot\\[bot\\]$"

# Timing constants (seconds)
readonly WAIT_FAST=10
readonly WAIT_MEDIUM=60
Expand Down Expand Up @@ -604,14 +608,15 @@ check_pr_status() {
check_count=$(echo "$pr_info" | jq '.statusCheckRollup | length')

if [[ "$check_count" -gt 0 ]]; then
local pending_count failed_count
pending_count=$(echo "$pr_info" | jq '[.statusCheckRollup[] | select(.status == "PENDING" or .status == "IN_PROGRESS")] | length')
failed_count=$(echo "$pr_info" | jq '[.statusCheckRollup[] | select(.conclusion == "FAILURE")] | length')
local pending_count failed_count action_required_count
pending_count=$(printf '%s' "$pr_info" | jq '[.statusCheckRollup[] | select(.status == "PENDING" or .status == "IN_PROGRESS")] | length')
failed_count=$(printf '%s' "$pr_info" | jq '[.statusCheckRollup[] | select(.conclusion == "FAILURE")] | length')
action_required_count=$(printf '%s' "$pr_info" | jq '[.statusCheckRollup[] | select(.conclusion == "ACTION_REQUIRED")] | length')

print_info " CI Checks: $check_count total, $pending_count pending, $failed_count failed"
print_info " CI Checks: $check_count total, $pending_count pending, $failed_count failed, $action_required_count action required"

[[ "$pending_count" -gt 0 ]] && checks_pending=true
[[ "$failed_count" -gt 0 ]] && checks_failed=true
[[ "$failed_count" -gt 0 || "$action_required_count" -gt 0 ]] && checks_failed=true
fi

# Determine overall status
Expand Down Expand Up @@ -640,13 +645,21 @@ get_pr_feedback() {

print_step "Getting PR feedback..."

# Get CodeRabbit comments
local coderabbit_comments
coderabbit_comments=$(gh api "repos/{owner}/{repo}/pulls/${pr_number}/comments" --jq '.[] | select(.user.login | contains("coderabbit")) | .body' 2>/dev/null | head -10 || echo "")
# Get AI reviewer comments (CodeRabbit, Gemini Code Assist, Augment Code, Copilot)
local ai_review_comments api_response
api_response=$(gh api "repos/{owner}/{repo}/pulls/${pr_number}/comments" 2>/dev/null)

if [[ -n "$coderabbit_comments" ]]; then
print_info "CodeRabbit feedback found"
echo "$coderabbit_comments"
if [[ -z "$api_response" ]]; then
print_warning "Failed to fetch PR comments from GitHub API"
else
ai_review_comments=$(printf '%s' "$api_response" | jq -r --arg bots "$AI_REVIEWERS" \
'.[] | select(.user.login | test($bots; "i")) | "\(.user.login): \(.body)"' \
2>/dev/null | head -20)

if [[ -n "$ai_review_comments" ]]; then
print_info "AI reviewer feedback found"
echo "$ai_review_comments"
fi
fi

# Get check run annotations
Expand Down Expand Up @@ -685,10 +698,16 @@ check_and_trigger_review() {
return 1
fi

# Get last CodeRabbit review time
local last_review_time
last_review_time=$(gh api "repos/{owner}/{repo}/pulls/${pr_number}/reviews" \
--jq '[.[] | select(.user.login | contains("coderabbit"))] | sort_by(.submitted_at) | last | .submitted_at // ""' 2>/dev/null || echo "")
# Get last AI reviewer review time (any supported reviewer)
local last_review_time api_response
api_response=$(gh api "repos/{owner}/{repo}/pulls/${pr_number}/reviews" 2>/dev/null)

if [[ -n "$api_response" ]]; then
last_review_time=$(printf '%s' "$api_response" | jq -r --arg bots "$AI_REVIEWERS" \
'[.[] | select(.user.login | test($bots; "i"))] | sort_by(.submitted_at) | last | .submitted_at // ""' 2>/dev/null)
else
last_review_time=""
fi

# Convert times to epoch for comparison
local now_epoch last_push_epoch last_review_epoch
Expand Down Expand Up @@ -728,6 +747,52 @@ check_and_trigger_review() {
return 1
}

# Check for unresolved review threads on a PR using GraphQL
# Arguments: $1 - PR number
# Returns: 0 if no unresolved threads, 1 if unresolved threads exist, 2 on API error
# Output: Warning message if unresolved threads found
check_unresolved_review_comments() {
Copy link

@augmentcode augmentcode bot Feb 3, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

check_unresolved_review_comments is defined but not called anywhere in the PR loop, so it won’t currently prevent /pr-loop from reporting success even when AI review comments exist.

Severity: medium

Fix This in Augment

🤖 Was this useful? React with 👍 or 👎, or 🚀 if it prevented an incident/outage.

local pr_number="$1"

local repo_owner repo_name api_response unresolved_count
repo_owner=$(gh repo view --json owner -q '.owner.login' 2>/dev/null || echo "")
repo_name=$(gh repo view --json name -q '.name' 2>/dev/null || echo "")

if [[ -z "$repo_owner" || -z "$repo_name" ]]; then
print_error "Failed to resolve repo owner/name - cannot verify review status"
return 2
fi

# shellcheck disable=SC2016 # GraphQL variables, not shell - single quotes intentional
api_response=$(gh api graphql -f query='
query($owner:String!, $repo:String!, $number:Int!) {
repository(owner:$owner, name:$repo) {
pullRequest(number:$number) {
reviewThreads(first:100) { nodes { isResolved } }
}
}
}' -f owner="$repo_owner" -f repo="$repo_name" -F number="$pr_number" 2>/dev/null)

if [[ -z "$api_response" ]]; then
print_error "Failed to fetch PR review threads from GitHub API - cannot verify review status"
return 2
fi

unresolved_count=$(printf '%s' "$api_response" | jq -r \
'[.data.repository.pullRequest.reviewThreads.nodes[] | select(.isResolved == false)] | length' 2>/dev/null)

if ! [[ "$unresolved_count" =~ ^[0-9]+$ ]]; then
print_error "Failed to parse unresolved thread count - cannot proceed safely"
return 2
fi

if [[ "$unresolved_count" -gt 0 ]]; then
print_warning "Found $unresolved_count unresolved review threads"
return 1
fi
return 0
}
Comment on lines +750 to +794
Copy link
Contributor

@coderabbitai coderabbitai bot Feb 3, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🔴 Critical

🧩 Analysis chain

🏁 Script executed:

# Get broader context around the function
head -n 800 .agent/scripts/quality-loop-helper.sh | tail -n +730 | cat -n

# Search for references to AI_REVIEWERS in the file
rg -n "AI_REVIEWERS" .agent/scripts/quality-loop-helper.sh -B 2 -A 2

# Search for any comments about unresolved review filtering
rg -n "unresolved" .agent/scripts/quality-loop-helper.sh -B 3 -A 3

Repository: marcusquinn/aidevops

Length of output: 6409

Filter unresolved threads by AI reviewer author to match stated intent.

The function checks for unresolved review threads from any source, but the calling context (line 869) and comment state "Check for unresolved AI review comments." Other functions in this file filter by $AI_REVIEWERS using jq's test() function—apply the same pattern here.

Update the GraphQL query to include comments { nodes { author { login } } } and filter the jq selection to exclude non-AI reviewers, matching the behavior at lines 654-655 and 705-706:

[.data.repository.pullRequest.reviewThreads.nodes[] | select(.isResolved == false and (.comments.nodes[0].author.login | test($bots; "i")))] | length

This ensures the automation only blocks on AI reviewer feedback, maintaining the intended gate before marking PRs as ready.

🤖 Prompt for AI Agents 
In @.agent/scripts/quality-loop-helper.sh around lines 749 - 793, The
check_unresolved_review_comments function currently counts all unresolved review
threads but should only consider AI reviewer threads; update the GraphQL
selection to include comments' author login (add comments { nodes { author {
login } } } to the pullRequest.reviewThreads query) and change the jq extraction
(unresolved_count) to filter unresolved threads where the first comment author
login matches the AI reviewer pattern variable ($AI_REVIEWERS) using jq's test()
(same pattern used elsewhere in this file), ensuring unresolved_count uses that
filtered length; keep references to check_unresolved_review_comments,
api_response, and $AI_REVIEWERS when making the change.


# Monitor PR until approved or merged
# Arguments: --pr NUMBER, --wait-for-ci, --max-iterations N, --auto-trigger-review
# Returns: 0 on approval/merge, 1 if max iterations reached
Expand Down Expand Up @@ -802,10 +867,23 @@ pr_review_loop() {
return 0
;;
READY)
print_success "PR is approved and ready to merge!"
rm -f "$STATE_FILE"
echo "<promise>PR_APPROVED</promise>"
return 0
# Check for unresolved AI review comments before declaring ready
local unresolved_check_result
check_unresolved_review_comments "$pr_number"
unresolved_check_result=$?

if [[ $unresolved_check_result -eq 2 ]]; then
print_warning "Could not verify AI review status (API error) - proceeding with caution"
elif [[ $unresolved_check_result -eq 1 ]]; then
print_warning "PR approved but has unresolved AI review comments"
get_pr_feedback "$pr_number"
print_info "Address the AI reviewer feedback and push updates."
else
print_success "PR is approved and ready to merge!"
rm -f "$STATE_FILE"
echo "<promise>PR_APPROVED</promise>"
return 0
fi
;;
PENDING)
# Get pending checks and calculate adaptive wait
Expand Down
13 changes: 13 additions & 0 deletions .agent/workflows/pr.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,19 @@ Calls remote quality services via APIs:
- Codacy - Code quality analysis
- SonarCloud - Security and maintainability

### Supported AI Code Reviewers

The PR loop monitors comments from multiple AI code review services:

| Reviewer | Bot Username Pattern | Purpose |
|----------|---------------------|---------|
| CodeRabbit | `coderabbit*` | AI-powered code review with suggestions |
| Gemini Code Assist | `gemini-code-assist[bot]` | Google's AI code review |
| Augment Code | `augment-code[bot]`, `augmentcode[bot]` | AI-powered code review and improvement |
| GitHub Copilot | `copilot[bot]` | GitHub's AI assistant |

The `/pr-loop` command automatically detects and surfaces comments from all these reviewers, ensuring no feedback is missed regardless of which AI services are configured on the repository.

### 3. Standards Compliance (`/code-standards`)

Checks against our documented quality standards:
Expand Down