diff --git a/.github/scripts/Invoke-RerunReviewTrigger.Tests.ps1 b/.github/scripts/Invoke-RerunReviewTrigger.Tests.ps1 index 860094f5afdc..aad263ac8545 100644 --- a/.github/scripts/Invoke-RerunReviewTrigger.Tests.ps1 +++ b/.github/scripts/Invoke-RerunReviewTrigger.Tests.ps1 @@ -333,14 +333,17 @@ Describe 'Get-RerunActions' { $result.Actions.Count | Should -Be 0 } - It 'refuses to trigger when the candidate has no rerun comment id' { + It 'triggers when the candidate has no proven current-cycle rerun comment id' { $items = @(New-TestDecision -PRNumber '5' -Decision 'trigger' -ExpectedHeadSha 'x') $candidates = @(New-TestCandidate -PRNumber 5 -HeadSha 'x' -RerunCommentId 0) $result = Get-RerunActions -Items $items -Candidates $candidates - $result.HadFailure | Should -BeTrue - $result.Actions.Count | Should -Be 0 + $result.HadFailure | Should -BeFalse + $result.Actions.Count | Should -Be 1 + $result.Actions[0].prNumber | Should -Be 5 + $result.Actions[0].decision | Should -Be 'trigger' + $result.Actions[0].rerunCommentId | Should -Be 0 } It 'continues processing valid decisions after a failed one' { diff --git a/.github/scripts/Invoke-RerunReviewTrigger.ps1 b/.github/scripts/Invoke-RerunReviewTrigger.ps1 index dc9562d02411..8fcb2f3ef17a 100644 --- a/.github/scripts/Invoke-RerunReviewTrigger.ps1 +++ b/.github/scripts/Invoke-RerunReviewTrigger.ps1 @@ -211,8 +211,11 @@ function Get-RerunActions { $platform = Get-PlatformFromLabels -Labels @() -Fallback ([string]$candidate.platform) $pipelineRef = Normalize-PipelineRef -Value ([string]$candidate.pipelineRef) -Fallback $DefaultPipelineRef + # The queue label does not prove which command created the current cycle, + # so source-ambiguous candidates intentionally carry no reaction target. + # review-trigger.yml does not need a comment to dispatch. if ($decision -eq 'trigger' -and $rerunCommentId -le 0) { - throw "Candidate for PR #$prNumber has no rerun comment id; cannot trigger." + Write-Host "PR #$prNumber trigger has no proven current-cycle rerun comment; dispatching without a reaction target." } $actions.Add([pscustomobject]@{ diff --git a/.github/scripts/Query-AutoRerunCandidates.Tests.ps1 b/.github/scripts/Query-AutoRerunCandidates.Tests.ps1 new file mode 100644 index 000000000000..2eef271cb85c --- /dev/null +++ b/.github/scripts/Query-AutoRerunCandidates.Tests.ps1 @@ -0,0 +1,373 @@ +#!/usr/bin/env pwsh +#Requires -Modules Pester + +BeforeAll { + $scriptPath = Join-Path $PSScriptRoot 'Query-AutoRerunCandidates.ps1' + $workflowPath = Join-Path $PSScriptRoot '../workflows/pr-review-queue.yml' + $scannerPath = Join-Path $PSScriptRoot '../workflows/rerun-review-scanner.md' + $reviewTriggerPath = Join-Path $PSScriptRoot '../workflows/review-trigger.yml' + $labelHelperPath = Join-Path $PSScriptRoot 'shared/Update-AgentLabels.ps1' + $outputDir = Join-Path $PSScriptRoot '../../CustomAgentLogsTmp/QueryAutoRerunTests' + New-Item -ItemType Directory -Force -Path $outputDir | Out-Null + + . $scriptPath -Owner 'test-owner' -Repo 'test-repo' + + function New-TestPR { + param( + [int]$Number, + [bool]$IsDraft = $false, + [string[]]$Labels = @(), + [string]$HeadSha = '2222222abcdef' + ) + + [pscustomobject]@{ + number = $Number + title = "PR $Number" + url = "https://example.test/$Number" + headRefOid = $HeadSha + isDraft = $IsDraft + labels = @($Labels | ForEach-Object { [pscustomobject]@{ name = $_ } }) + author = [pscustomobject]@{ login = 'dev-user' } + } + } + + function ConvertTo-GhLines { + param([object[]]$Items) + return @($Items | ForEach-Object { $_ | ConvertTo-Json -Depth 10 -Compress }) + } + + function New-DeclineMarkerComment { + param( + [int64]$Id, + [string]$CreatedAt, + [string]$HeadSha, + [string]$Login = 'github-actions[bot]' + ) + + [pscustomobject]@{ + id = $Id + body = "" + created_at = $CreatedAt + updated_at = $CreatedAt + user = [pscustomobject]@{ login = $Login; type = 'Bot' } + author_association = 'NONE' + } + } + + function Invoke-TestScan { + Invoke-AutoRerunCandidateScan ` + -ScanOwner 'test-owner' ` + -ScanRepo 'test-repo' ` + -ScanLimit $script:Limit ` + -ScanDryRun:$script:DryRun ` + -ScanOutputPath $script:OutputPath + } +} + +AfterAll { + Remove-Item -LiteralPath $outputDir -Recurse -Force -ErrorAction SilentlyContinue +} + +Describe 'Query-AutoRerunCandidates' { + BeforeEach { + $script:Owner = 'test-owner' + $script:Repo = 'test-repo' + $script:Limit = 5 + $script:DryRun = $true + $script:OutputPath = Join-Path $outputDir "$([Guid]::NewGuid().ToString('N')).json" + $script:prList = @() + $script:issueComments = @() + $script:reviews = @() + $script:reviewComments = @() + $script:commits = @() + $script:failedPRs = @() + $script:ghCalls = @() + + Mock gh { + param( + [Parameter(ValueFromRemainingArguments = $true)] + [string[]]$GhArgs + ) + + $command = $GhArgs -join ' ' + $script:ghCalls += $command + $global:LASTEXITCODE = 0 + + if ($command -match '^pr list ') { + return ($script:prList | ConvertTo-Json -Depth 10 -Compress) + } + + $numberMatch = [regex]::Match($command, '/(?:issues|pulls)/(\d+)/') + $number = if ($numberMatch.Success) { [int]$numberMatch.Groups[1].Value } else { 0 } + if ($script:failedPRs -contains $number) { + $global:LASTEXITCODE = 1 + return @() + } + + if ($command -match '/issues/\d+/comments') { return ConvertTo-GhLines $script:issueComments } + if ($command -match '/pulls/\d+/reviews') { return ConvertTo-GhLines $script:reviews } + if ($command -match '/pulls/\d+/comments') { return ConvertTo-GhLines $script:reviewComments } + if ($command -match '/pulls/\d+/commits') { return ConvertTo-GhLines $script:commits } + throw "Unexpected gh call: $command" + } + } + +Context 'bounded scan metadata' { + It 'processes only Limit items and reports exact truncation using one sentinel item' { + $script:prList = @(1..6 | ForEach-Object { New-TestPR -Number $_ -IsDraft $true }) + + $messages = Invoke-TestScan 6>&1 | Out-String + + $summary = Get-Content -Raw -LiteralPath $script:OutputPath | ConvertFrom-Json + $messages | Should -Match '::warning::Open PR scan truncated' + $summary.scan.limit | Should -Be 5 + $summary.scan.fetchedCount | Should -Be 5 + $summary.scan.observedCount | Should -Be 6 + $summary.scan.truncated | Should -BeTrue + $summary.decisions.Count | Should -Be 5 + @($script:ghCalls | Where-Object { $_ -match '^pr list .*--limit 6' }).Count | Should -Be 1 + } + + It 'pins pull-request dry-run validation to Limit 5' { + $workflow = Get-Content -Raw -LiteralPath $workflowPath + $workflow | Should -Match '(?s)Validate auto-rerun labeler \(dry-run\).*?-DryRun\s+\\\s*\r?\n\s*-Limit 5\s+\\' + } + + It 'keeps the scheduled queue job at pull-request read permission' { + $workflow = Get-Content -Raw -LiteralPath $workflowPath + $generateJob = [regex]::Match($workflow, '(?s) generate-report:.*?^ validate:', 'Multiline').Value + $generateJob | Should -Match 'pull-requests:\s+read' + $generateJob | Should -Not -Match 'pull-requests:\s+write' + } +} + +Context 'API request bounding' { + It 'does not fetch review or commit history for a PR without an AI Summary' { + $script:prList = @(New-TestPR -Number 10) + + Invoke-TestScan + + $summary = Get-Content -Raw -LiteralPath $script:OutputPath | ConvertFrom-Json + $summary.decisions[0].reason | Should -Be 'no-ai-summary' + @($script:ghCalls | Where-Object { $_ -match '/issues/10/comments' }).Count | Should -Be 1 + @($script:ghCalls | Where-Object { $_ -match '/pulls/10/(reviews|comments|commits)' }).Count | Should -Be 0 + } +} + +Context 'error aggregation' { + It 'fails a dry-run after writing structured error details' { + $script:prList = @(1..2 | ForEach-Object { New-TestPR -Number $_ }) + $script:failedPRs = @(1, 2) + + { Invoke-TestScan } | Should -Throw '*2 of 2 evaluated PR(s) had errors*' + + $summary = Get-Content -Raw -LiteralPath $script:OutputPath | ConvertFrom-Json + $summary.errors | Should -Be 2 + $summary.evaluated | Should -Be 2 + $summary.systemicFailure | Should -BeTrue + @($summary.decisions | Where-Object reason -like 'error:*').Count | Should -Be 2 + } + + It 'keeps an isolated scheduled error non-fatal but visible' { + $script:DryRun = $false + $script:prList = @(1..4 | ForEach-Object { New-TestPR -Number $_ }) + $script:failedPRs = @(1) + + { Invoke-TestScan } | Should -Not -Throw + + $summary = Get-Content -Raw -LiteralPath $script:OutputPath | ConvertFrom-Json + $summary.errors | Should -Be 1 + $summary.systemicFailure | Should -BeFalse + } + + It 'fails a scheduled scan when evaluation failures are systemic' { + $script:DryRun = $false + $script:prList = @(1..3 | ForEach-Object { New-TestPR -Number $_ }) + $script:failedPRs = @(1, 2, 3) + + { Invoke-TestScan } | Should -Throw '*3 of 3 evaluated PR(s) had errors*' + + $summary = Get-Content -Raw -LiteralPath $script:OutputPath | ConvertFrom-Json + $summary.systemicFailure | Should -BeTrue + } + + It 'fails after writing the summary when an eligible label cannot be applied or verified' { + $script:DryRun = $false + $script:prList = @(New-TestPR -Number 1) + $script:issueComments = @( + [pscustomobject]@{ + id = 1 + body = "`n" + created_at = '2026-05-31T09:00:00Z' + updated_at = '2026-05-31T09:00:00Z' + user = [pscustomobject]@{ login = 'MauiBot'; type = 'User' } + author_association = 'MEMBER' + } + ) + Mock Ensure-LabelExists {} + Mock Add-Label { $false } + Mock Get-IssueLabels { @() } + + { Invoke-TestScan } | Should -Throw '*1 label application failure(s)*' + + $summary = Get-Content -Raw -LiteralPath $script:OutputPath | ConvertFrom-Json + $summary.applyFailures | Should -Be 1 + $summary.applied | Should -Be 0 + $summary.decisions[0].eligible | Should -BeTrue + $summary.decisions[0].applied | Should -BeFalse + } +} + +Context 'scanner decline checkpoint' { + It 'does not treat generic ready-label removals as scanner declines' { + $script:prList = @(New-TestPR -Number 6) + $script:issueComments = @( + [pscustomobject]@{ + id = 1 + body = "`n" + created_at = '2026-05-31T09:00:00Z' + updated_at = '2026-05-31T09:00:00Z' + user = [pscustomobject]@{ login = 'MauiBot'; type = 'User' } + author_association = 'MEMBER' + } + ) + + Invoke-TestScan + + $summary = Get-Content -Raw -LiteralPath $script:OutputPath | ConvertFrom-Json + $summary.decisions[0].eligible | Should -BeTrue + $summary.decisions[0].reason | Should -Be 'new-head-commit' + } + + It 'sorts explicit scanner markers newest-first and re-evaluates eligibility against that checkpoint' { + $script:prList = @( + New-TestPR -Number 7 -Labels @('s/agent-rerun-declined') -HeadSha '2222222222222222222222222222222222222222' + ) + $script:issueComments = @( + [pscustomobject]@{ + id = 1 + body = "`n" + created_at = '2026-05-31T09:00:00Z' + updated_at = '2026-05-31T09:00:00Z' + user = [pscustomobject]@{ login = 'MauiBot'; type = 'User' } + author_association = 'MEMBER' + }, + [pscustomobject]@{ + id = 2 + body = 'I pushed the update.' + created_at = '2026-05-31T09:45:00Z' + updated_at = '2026-05-31T09:45:00Z' + user = [pscustomobject]@{ login = 'dev-user'; type = 'User' } + author_association = 'CONTRIBUTOR' + }, + (New-DeclineMarkerComment -Id 3 -CreatedAt '2026-05-31T08:00:00Z' -HeadSha 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'), + (New-DeclineMarkerComment -Id 4 -CreatedAt '2026-05-31T10:00:00Z' -HeadSha '2222222222222222222222222222222222222222'), + (New-DeclineMarkerComment -Id 5 -CreatedAt '2026-05-31T09:30:00Z' -HeadSha 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb') + ) + + Invoke-TestScan + + $summary = Get-Content -Raw -LiteralPath $script:OutputPath | ConvertFrom-Json + $summary.decisions[0].eligible | Should -BeFalse + $summary.decisions[0].reason | Should -Be 'declined-state-unchanged' + } + + It 're-qualifies a head that changed while the scanner was persisting its decline marker' { + $script:prList = @( + New-TestPR -Number 8 -Labels @('s/agent-rerun-declined') -HeadSha '3333333333333333333333333333333333333333' + ) + $script:issueComments = @( + [pscustomobject]@{ + id = 1 + body = "`n" + created_at = '2026-05-31T09:00:00Z' + updated_at = '2026-05-31T09:00:00Z' + user = [pscustomobject]@{ login = 'MauiBot'; type = 'User' } + author_association = 'MEMBER' + }, + (New-DeclineMarkerComment -Id 2 -CreatedAt '2026-05-31T10:00:00Z' -HeadSha '2222222222222222222222222222222222222222') + ) + + Invoke-TestScan + + $summary = Get-Content -Raw -LiteralPath $script:OutputPath | ConvertFrom-Json + $summary.decisions[0].eligible | Should -BeTrue + $summary.decisions[0].reason | Should -Be 'new-head-commit' + } + + It 'persists skip markers before consuming ready labels and clears them before trigger dispatch' { + $scanner = Get-Content -Raw -LiteralPath $scannerPath + $scanner.IndexOf('await markDeclined(prNumber, liveHeadSha);') | + Should -BeLessThan $scanner.IndexOf("await react(a.rerunCommentId, '-1');") + $scanner.IndexOf("await react(a.rerunCommentId, '-1');") | + Should -BeLessThan $scanner.IndexOf('await removeReadyLabel(prNumber);') + $scanner.IndexOf('await clearDeclined(prNumber);') | + Should -BeLessThan $scanner.IndexOf('await github.rest.actions.createWorkflowDispatch({') + } + + It 'makes decline marking idempotent across partial failures' { + $scanner = Get-Content -Raw -LiteralPath $scannerPath + $markDeclined = [regex]::Match( + $scanner, + '(?s)async function markDeclined\(prNumber, headSha\).*?async function clearDeclined' + ).Value + + $markDeclined.IndexOf('if (alreadyLabelled && existingMarker)') | + Should -BeLessThan $markDeclined.IndexOf('issues.createComment({') + $markDeclined.IndexOf('if (existingMarker)') | + Should -BeLessThan $markDeclined.IndexOf('issues.createComment({') + $markDeclined | Should -Match 'rest\.issues\.addLabels' + } + + It 'recovers when concurrent decline-label creation returns 422' { + $scanner = Get-Content -Raw -LiteralPath $scannerPath + $ensureDeclinedLabel = [regex]::Match( + $scanner, + '(?s)async function ensureDeclinedLabel\(\).*?async function markDeclined' + ).Value + + $ensureDeclinedLabel | Should -Match 'if \(createError\.status !== 422\) \{ throw createError; \}' + ([regex]::Matches($ensureDeclinedLabel, 'rest\.issues\.getLabel')).Count | Should -Be 2 + $ensureDeclinedLabel.LastIndexOf('rest.issues.getLabel') | + Should -BeGreaterThan $ensureDeclinedLabel.IndexOf('rest.issues.createLabel') + $ensureDeclinedLabel.LastIndexOf('await syncDeclinedLabel(existing);') | + Should -BeGreaterThan $ensureDeclinedLabel.LastIndexOf('rest.issues.getLabel') + } + + It 'keeps advisory decline cleanup best-effort before dispatch' { + $scanner = Get-Content -Raw -LiteralPath $scannerPath + $clearDeclined = [regex]::Match( + $scanner, + '(?s)async function clearDeclined\(prNumber\).*?\n\s+for \(const a of actions\)' + ).Value + + $clearDeclined | Should -Match 'core\.warning' + $clearDeclined | Should -Not -Match 'throw new Error' + $scanner.IndexOf('await clearDeclined(prNumber);') | + Should -BeLessThan $scanner.IndexOf('await github.rest.actions.createWorkflowDispatch({') + } + + It 'clears the decline label in the shared review entrypoint before acquiring the review lock' { + $reviewTrigger = Get-Content -Raw -LiteralPath $reviewTriggerPath + $reviewTrigger.IndexOf('$declineCleared = Clear-AgentRerunDeclined') | + Should -BeLessThan $reviewTrigger.IndexOf('$locked = Set-AgentReviewInProgress') + $reviewTrigger | Should -Match '::warning::Could not clear s/agent-rerun-declined' + $reviewTrigger | Should -Not -Match 'throw "Failed to clear s/agent-rerun-declined' + } + + It 'keeps scanner decline-label metadata aligned with the shared definition' { + $scanner = Get-Content -Raw -LiteralPath $scannerPath + $labelHelper = Get-Content -Raw -LiteralPath $labelHelperPath + + $scanner | Should -Match ([regex]::Escape('agent-rerun-declined:')) + foreach ($value in @( + 's/agent-rerun-declined', + 'AI rerun scanner declined the current PR state; new author activity is required', + 'D4C5F9' + )) { + $scanner | Should -Match ([regex]::Escape($value)) + $labelHelper | Should -Match ([regex]::Escape($value)) + } + } +} +} diff --git a/.github/scripts/Query-AutoRerunCandidates.ps1 b/.github/scripts/Query-AutoRerunCandidates.ps1 new file mode 100644 index 000000000000..851919b5a6eb --- /dev/null +++ b/.github/scripts/Query-AutoRerunCandidates.ps1 @@ -0,0 +1,374 @@ +#!/usr/bin/env pwsh +<# +.SYNOPSIS + Autonomously applies the s/agent-ready-for-rerun label to open PRs that have + genuinely new PR-author activity since their last AI review. + +.DESCRIPTION + Driver for the PR Review Queue workflow. Enumerates open, non-draft PRs and + evaluates each with Resolve-AutonomousRerunEligibility — the deterministic, + AI-free counterpart of the `/review rerun` eligibility check. A PR qualifies + only when it already carries a MauiBot AI Summary AND has a new commit, a new + non-command author comment, or a head SHA that differs from the last reviewed + SHA since that summary. Eligible PRs are labelled s/agent-ready-for-rerun, + which the hourly rerun-review-scanner then picks up and re-reviews. + + No AI is used and untrusted text is never inspected semantically. PRs that + were never AI-reviewed do not qualify. PRs that already carry the label, or + that have a non-stale s/agent-review-in-progress label, are skipped. + +.PARAMETER Owner + Repository owner (default: dotnet). + +.PARAMETER Repo + Repository name (default: maui). + +.PARAMETER Limit + Maximum number of open PRs to inspect (default: 300). + +.PARAMETER DryRun + Evaluate and report without applying any labels. + +.PARAMETER OutputPath + Optional path to write a JSON summary of the decisions. + +.EXAMPLE + ./Query-AutoRerunCandidates.ps1 -DryRun + # Read-only: classify open PRs without applying labels. +#> + +param( + [string]$Owner = 'dotnet', + [string]$Repo = 'maui', + [ValidateRange(1, 10000)] + [int]$Limit = 300, + [switch]$DryRun, + [string]$OutputPath +) + +$ErrorActionPreference = 'Stop' + +$ReadyForRerunLabel = 's/agent-ready-for-rerun' +$RerunDeclinedLabel = 's/agent-rerun-declined' +$ReviewInProgressLabel = 's/agent-review-in-progress' + +# Pass -Owner/-Repo through: Resolve-RerunEligibility.ps1 has its own $Owner/$Repo params +# defaulting to dotnet/maui, so dot-sourcing it without arguments would reset THIS script's +# $Owner/$Repo to the defaults (mirrors the correct pattern in Query-RerunReadyPRs.ps1). +. "$PSScriptRoot/Resolve-RerunEligibility.ps1" -Owner $Owner -Repo $Repo +. "$PSScriptRoot/shared/Update-AgentLabels.ps1" + +# Derive the label description/color from the shared canonical definition so this script +# and Update-AgentLabels.ps1 can't drift and repeatedly re-PATCH each other's metadata. +$rerunLabelDef = $AllLabelDefs[$ReadyForRerunLabel] +$ReadyForRerunLabelDescription = $rerunLabelDef.Description +$ReadyForRerunLabelColor = $rerunLabelDef.Color + +function Get-IssueLabels { + param([int]$Number) + + # Don't silently treat an API failure as "no labels" — that would drop a real + # s/agent-ready-for-rerun / in-progress label and cause a spurious re-label or + # skip. Surface the failure (including gh's stderr, which we no longer suppress) + # so the per-PR try/catch records it as an error with actionable detail. + $names = gh api "repos/$Owner/$Repo/issues/$Number/labels" --jq '.[].name' + if ($LASTEXITCODE -ne 0) { + throw "Failed to fetch labels for #$Number (gh api exited $LASTEXITCODE)." + } + return @($names) +} + +function Get-IssueCommentsForPR { + param([int]$Number) + + $issueCommentsRaw = gh api "repos/$Owner/$Repo/issues/$Number/comments?per_page=100" --paginate --jq '.[]' + if ($LASTEXITCODE -ne 0) { throw "Failed to fetch issue comments for #$Number (gh api exited $LASTEXITCODE)." } + return @($issueCommentsRaw | ForEach-Object { ConvertTo-RerunActivityItem -Item ($_ | ConvertFrom-Json) -Kind 'issue-comment' }) +} + +function Get-ReviewActivityForPR { + param([int]$Number) + + # Fetch review history only after an issue comment proves this PR has an AI + # Summary. Most open PRs have never been AI-reviewed, so this avoids three + # paginated API families for every ineligible PR. + $reviewsRaw = gh api "repos/$Owner/$Repo/pulls/$Number/reviews?per_page=100" --paginate --jq '.[]' + if ($LASTEXITCODE -ne 0) { throw "Failed to fetch reviews for #$Number (gh api exited $LASTEXITCODE)." } + $reviewCommentsRaw = gh api "repos/$Owner/$Repo/pulls/$Number/comments?per_page=100" --paginate --jq '.[]' + if ($LASTEXITCODE -ne 0) { throw "Failed to fetch review comments for #$Number (gh api exited $LASTEXITCODE)." } + + $reviews = @($reviewsRaw | ForEach-Object { ConvertTo-RerunActivityItem -Item ($_ | ConvertFrom-Json) -Kind 'review' }) + $reviewComments = @($reviewCommentsRaw | ForEach-Object { ConvertTo-RerunActivityItem -Item ($_ | ConvertFrom-Json) -Kind 'review-comment' }) + return @($reviews + $reviewComments) +} + +function Get-CommitsForPR { + param([int]$Number) + + # Fail loud (see Get-ActivityForPR) — a dropped commit list would mislead the + # new-head-commit eligibility check. + $commitsRaw = gh api "repos/$Owner/$Repo/pulls/$Number/commits?per_page=100" --paginate --jq '.[]' + if ($LASTEXITCODE -ne 0) { throw "Failed to fetch commits for #$Number (gh api exited $LASTEXITCODE)." } + return @($commitsRaw | ForEach-Object { $_ | ConvertFrom-Json }) +} + +function Get-LatestScannerDecline { + param([object[]]$Comments) + + # The scanner records the exact head it declined in a minimized bot comment. + # The SHA identity, not just the marker timestamp, closes the TOCTOU window + # between the safe-output job's live-head read and its subsequent writes. + $markerPattern = '' + $markers = @($Comments | Where-Object { + $_.kind -eq 'issue-comment' -and + $_.user -and + (Normalize-GitHubActorLogin ([string]$_.user.login)) -eq 'github-actions[bot]' -and + ([string]$_.body) -match $markerPattern + } | ForEach-Object { + $match = [regex]::Match([string]$_.body, $markerPattern) + [pscustomobject]@{ + CommentId = [Int64]$_.id + DeclinedAt = Get-ObjectDate $_ 'created_at' + HeadSha = $match.Groups[1].Value.ToLowerInvariant() + } + } | Sort-Object @{ Expression = { $_.DeclinedAt }; Descending = $true }, @{ Expression = { $_.CommentId }; Descending = $true }) + + return @($markers | Select-Object -First 1) +} + +function Invoke-AutoRerunCandidateScan { + param( + [string]$ScanOwner = $Owner, + [string]$ScanRepo = $Repo, + [int]$ScanLimit = $Limit, + [switch]$ScanDryRun = $DryRun, + [string]$ScanOutputPath = $OutputPath + ) + + $Owner = $ScanOwner + $Repo = $ScanRepo + $Limit = $ScanLimit + $DryRun = $ScanDryRun + $OutputPath = $ScanOutputPath + + # Fetch one sentinel item beyond the processing ceiling. This preserves the + # safety bound while making truncation exact instead of guessing when Count == Limit. + $fetchLimit = $Limit + 1 + $searchJson = gh pr list ` + --repo "$Owner/$Repo" ` + --state open ` + --limit $fetchLimit ` + --json number,title,url,headRefOid,isDraft,labels,author + if ($LASTEXITCODE -ne 0) { + throw "Failed to list open PRs (gh pr list exited with code $LASTEXITCODE)." + } + $listedPRs = @($searchJson | ConvertFrom-Json) + $truncated = $listedPRs.Count -gt $Limit + $openPRs = @($listedPRs | Select-Object -First $Limit) + + if ($truncated) { + Write-Host "::warning::Open PR scan truncated to the newest $Limit PR(s); at least $($listedPRs.Count) were returned. Older PRs were not evaluated." + } + + Write-Host "Inspecting $($openPRs.Count) open PR(s) for autonomous rerun eligibility..." + + $labelEnsured = $false + $decisions = @() + $appliedCount = 0 + $applyFailureCount = 0 + + foreach ($pr in $openPRs) { + $number = [int]$pr.number + $title = [string]$pr.title + + if ($pr.isDraft) { + $decisions += [pscustomobject]@{ prNumber = $number; title = $title; eligible = $false; reason = 'draft'; applied = $false } + continue + } + + # Per-PR error isolation: a single malformed PR or transient API failure + # must not abort the whole daily scan. Aggregate errors after the loop so + # systemic failures cannot produce a misleading green run. + try { + # gh pr list already fetched labels, so avoid another Issues API call. + $labels = @(@($pr.labels) | Where-Object { $_ } | ForEach-Object { $_.name }) + + # Treat a stale in-progress label as absent so a wedged review can recover. + $effectiveLabels = @($labels) + if ($labels -contains $ReviewInProgressLabel -and (Test-AgentReviewInProgressIsStale -PRNumber $number -Owner $Owner -Repo $Repo)) { + $effectiveLabels = @($labels | Where-Object { $_ -ne $ReviewInProgressLabel }) + } + + $issueComments = @(Get-IssueCommentsForPR -Number $number) + $activity = @($issueComments) + $commits = @() + if (Get-LatestAISummaryComment -Comments $issueComments) { + $reviewActivity = @(Get-ReviewActivityForPR -Number $number) + $activity = @($issueComments + $reviewActivity) + $commits = @(Get-CommitsForPR -Number $number) + } + $rawAuthorLogin = if ($pr.author -and $pr.author.login) { [string]$pr.author.login } else { '' } + $authorLogin = Normalize-GitHubActorLogin $rawAuthorLogin + + $result = Resolve-AutonomousRerunEligibility ` + -Comments $activity ` + -Commits $commits ` + -CurrentHeadSha $pr.headRefOid ` + -PRAuthorLogin $authorLogin ` + -CurrentLabels $effectiveLabels + + $alreadyPresent = @($labels | Where-Object { $_ -eq $ReadyForRerunLabel }).Count -gt 0 + $hasDeclinedMarker = @($labels | Where-Object { $_ -eq $RerunDeclinedLabel }).Count -gt 0 + + # Only a trusted scanner marker advances the checkpoint. It carries the + # exact declined head so a push racing the marker write always re-qualifies, + # even when the commit timestamp predates the marker timestamp. + if (-not $alreadyPresent -and $hasDeclinedMarker) { + $lastDecline = Get-LatestScannerDecline -Comments $issueComments + if ($lastDecline) { + $result = Resolve-AutonomousRerunEligibility ` + -Comments $activity ` + -Commits $commits ` + -CurrentHeadSha $pr.headRefOid ` + -PRAuthorLogin $authorLogin ` + -CurrentLabels $effectiveLabels ` + -LastDeclinedAt $lastDecline.DeclinedAt.ToString('o') ` + -LastDeclinedHeadSha $lastDecline.HeadSha + } else { + Write-Host "::warning::PR #$number has $RerunDeclinedLabel but no trusted head marker; ignoring the stale marker." + } + } + + $applied = $false + + if ($result.Eligible -and -not $alreadyPresent) { + if ($DryRun) { + Write-Host " [dry-run] Would label #$number ($($result.Reason)): $title" + } else { + if (-not $labelEnsured) { + Ensure-LabelExists ` + -LabelName $ReadyForRerunLabel ` + -Description $ReadyForRerunLabelDescription ` + -Color $ReadyForRerunLabelColor ` + -Owner $Owner ` + -Repo $Repo + $labelEnsured = $true + } + + $addSucceeded = Add-Label -PRNumber $number -LabelName $ReadyForRerunLabel -Owner $Owner -Repo $Repo + $labelIsPresent = $false + if (-not $addSucceeded) { + try { + $updatedLabels = @(Get-IssueLabels -Number $number) + $labelIsPresent = @($updatedLabels | Where-Object { $_ -eq $ReadyForRerunLabel }).Count -gt 0 + } catch { + Write-Host " ⚠️ Could not verify label state for #$($number): $($_.Exception.Message)" -ForegroundColor Yellow + } + } + if ($addSucceeded -or $labelIsPresent) { + $applied = $true + $appliedCount++ + if ($hasDeclinedMarker -and -not (Remove-Label -PRNumber $number -LabelName $RerunDeclinedLabel -Owner $Owner -Repo $Repo)) { + Write-Host "::warning::Applied $ReadyForRerunLabel to #$number but could not clear $RerunDeclinedLabel." + } + Write-Host " ✅ Applied $ReadyForRerunLabel to #$number ($($result.Reason)): $title" -ForegroundColor Green + } else { + Write-Host " ⚠️ Failed to apply $ReadyForRerunLabel to #$number" -ForegroundColor Yellow + Write-Host "::warning::Auto-rerun label application failed for PR #$number." + $applyFailureCount++ + } + } + } elseif ($result.Eligible -and $alreadyPresent) { + Write-Host " ⏭️ #$number already has $ReadyForRerunLabel — skipping" + } + + $decisions += [pscustomobject]@{ + prNumber = $number + title = $title + eligible = [bool]$result.Eligible + reason = [string]$result.Reason + alreadyPresent = $alreadyPresent + applied = $applied + } + } catch { + Write-Host " ⚠️ Skipping #$number due to evaluation error: $($_.Exception.Message)" -ForegroundColor Yellow + Write-Host "::warning::Auto-rerun evaluation failed for PR #$number; see the preceding log line." + $decisions += [pscustomobject]@{ + prNumber = $number + title = $title + eligible = $false + reason = "error: $($_.Exception.Message)" + alreadyPresent = $false + applied = $false + } + } + } + + $eligibleCount = @($decisions | Where-Object { $_.eligible -and -not $_.alreadyPresent }).Count + $errorCount = @($decisions | Where-Object { $_.reason -like 'error:*' }).Count + $evaluatedCount = @($decisions | Where-Object { $_.reason -ne 'draft' }).Count + $systemicThreshold = [Math]::Max(3, [Math]::Ceiling($evaluatedCount * 0.10)) + $systemicFailure = $evaluatedCount -gt 0 -and ( + $errorCount -eq $evaluatedCount -or + $errorCount -ge $systemicThreshold + ) + $shouldFail = ($DryRun -and $errorCount -gt 0) -or $systemicFailure -or $applyFailureCount -gt 0 + + if ($errorCount -gt 0) { + Write-Host "::warning::Autonomous rerun scan encountered $errorCount evaluation error(s) across $evaluatedCount evaluated PR(s)." + } + if ($applyFailureCount -gt 0) { + Write-Host "::warning::Autonomous rerun scan failed to apply $ReadyForRerunLabel to $applyFailureCount eligible PR(s)." + } + + if ($DryRun) { + Write-Host "Autonomous rerun scan complete: $eligibleCount PR(s) eligible (dry-run, no labels applied)." + } else { + Write-Host "Autonomous rerun scan complete: applied $ReadyForRerunLabel to $appliedCount PR(s)." + } + + if ($OutputPath) { + $outputDir = Split-Path -Parent $OutputPath + if ($outputDir) { + New-Item -ItemType Directory -Force -Path $outputDir | Out-Null + } + $summary = @{ + generatedAt = (Get-Date).ToUniversalTime().ToString('o') + dryRun = [bool]$DryRun + applied = $appliedCount + applyFailures = $applyFailureCount + eligible = $eligibleCount + errors = $errorCount + evaluated = $evaluatedCount + systemicThreshold = $systemicThreshold + systemicFailure = [bool]$systemicFailure + scan = @{ + limit = $Limit + fetchedCount = $openPRs.Count + observedCount = $listedPRs.Count + truncated = [bool]$truncated + } + decisions = @($decisions) + } | ConvertTo-Json -Depth 10 + $summary | Set-Content -LiteralPath $OutputPath -Encoding UTF8 + Write-Host "Wrote decision summary to $OutputPath" + } + + if ($shouldFail) { + if ($applyFailureCount -gt 0) { + throw "Autonomous rerun scan failed: $applyFailureCount label application failure(s); $errorCount of $evaluatedCount evaluated PR(s) had errors." + } + throw "Autonomous rerun scan failed: $errorCount of $evaluatedCount evaluated PR(s) had errors." + } +} + +if ($MyInvocation.InvocationName -eq '.') { + return +} + +Invoke-AutoRerunCandidateScan ` + -ScanOwner $Owner ` + -ScanRepo $Repo ` + -ScanLimit $Limit ` + -ScanDryRun:$DryRun ` + -ScanOutputPath $OutputPath diff --git a/.github/scripts/Query-RerunReadyPRs.Tests.ps1 b/.github/scripts/Query-RerunReadyPRs.Tests.ps1 new file mode 100644 index 000000000000..593d4aa05d6f --- /dev/null +++ b/.github/scripts/Query-RerunReadyPRs.Tests.ps1 @@ -0,0 +1,98 @@ +#!/usr/bin/env pwsh +#Requires -Modules Pester + +BeforeAll { + $scriptPath = Join-Path $PSScriptRoot 'Query-RerunReadyPRs.ps1' + $outputDir = Join-Path $PSScriptRoot '../../CustomAgentLogsTmp/QueryRerunReadyTests' + New-Item -ItemType Directory -Force -Path $outputDir | Out-Null + + . $scriptPath -Owner 'test-owner' -Repo 'test-repo' + + function ConvertTo-GhLines { + param([object[]]$Items) + return @($Items | ForEach-Object { $_ | ConvertTo-Json -Depth 10 -Compress }) + } +} + +AfterAll { + Remove-Item -LiteralPath $outputDir -Recurse -Force -ErrorAction SilentlyContinue +} + +Describe 'Query-RerunReadyPRs' { + BeforeEach { + $script:OutputPath = Join-Path $outputDir "$([Guid]::NewGuid().ToString('N')).json" + $script:issueComments = @( + [pscustomobject]@{ + id = 100 + body = "`n" + created_at = '2026-05-31T09:00:00Z' + updated_at = '2026-05-31T09:00:00Z' + user = [pscustomobject]@{ login = 'MauiBot'; type = 'User' } + author_association = 'MEMBER' + }, + [pscustomobject]@{ + id = 200 + body = '/review rerun' + created_at = '2026-05-31T10:00:00Z' + updated_at = '2026-05-31T10:00:00Z' + user = [pscustomobject]@{ login = 'maintainer'; type = 'User' } + author_association = 'MEMBER' + } + ) + + Mock Get-LatestReviewCommandOptions { + [pscustomobject]@{ + Platform = '' + PipelineRef = 'main' + CommentId = 200 + Body = '/review rerun' + } + } + + Mock gh { + param( + [Parameter(ValueFromRemainingArguments = $true)] + [string[]]$GhArgs + ) + + $command = $GhArgs -join ' ' + $global:LASTEXITCODE = 0 + + if ($command -match '^pr list ') { + return ([pscustomobject]@{ + number = 1 + title = 'Autonomously queued PR' + url = 'https://example.test/1' + headRefOid = '2222222abcdef' + isDraft = $false + labels = @([pscustomobject]@{ name = 's/agent-ready-for-rerun' }) + author = [pscustomobject]@{ login = 'dev-user' } + } | ConvertTo-Json -Depth 10 -Compress) + } + if ($command -match '/issues/1/labels') { + return 's/agent-ready-for-rerun' + } + if ($command -match '/issues/1/comments') { + return ConvertTo-GhLines $script:issueComments + } + if ($command -match '/pulls/1/(reviews|comments|commits)') { + return @() + } + + throw "Unexpected gh call: $command" + } + } + + It 'does not reuse a historical rerun command as the current queue reaction target' { + Invoke-RerunReadyPRQuery ` + -QueryMaxPRs 5 ` + -QueryOwner 'test-owner' ` + -QueryRepo 'test-repo' ` + -QueryOutputPath $script:OutputPath | Out-Null + + $result = Get-Content -Raw -LiteralPath $script:OutputPath | ConvertFrom-Json + $result.candidates.Count | Should -Be 1 + $result.candidates[0].reviewCommandId | Should -Be 200 + $result.candidates[0].rerunCommentId | Should -Be 0 + } +} diff --git a/.github/scripts/Query-RerunReadyPRs.ps1 b/.github/scripts/Query-RerunReadyPRs.ps1 index b5f66cc63dd1..b347fe75b541 100644 --- a/.github/scripts/Query-RerunReadyPRs.ps1 +++ b/.github/scripts/Query-RerunReadyPRs.ps1 @@ -59,6 +59,19 @@ function Get-PlatformFromLabels { return 'android' } +function Invoke-RerunReadyPRQuery { + param( + [int]$QueryMaxPRs = $MaxPRs, + [string]$QueryOwner = $Owner, + [string]$QueryRepo = $Repo, + [string]$QueryOutputPath = $OutputPath + ) + + $MaxPRs = $QueryMaxPRs + $Owner = $QueryOwner + $Repo = $QueryRepo + $OutputPath = $QueryOutputPath + $searchJson = gh pr list ` --repo "$Owner/$Repo" ` --state open ` @@ -80,10 +93,8 @@ foreach ($pr in @($searchResult)) { if ($labels -contains $ReviewInProgressLabel -and -not (Test-AgentReviewInProgressIsStale -PRNumber $number -Owner $Owner -Repo $Repo)) { continue } - $activity = @(Get-ActivityForPR -Number $number) $commits = @(Get-CommitsForPR -Number $number) - $latestRerun = Get-LatestRerunComment -Comments $activity $reviewOptions = Get-LatestReviewCommandOptions -Comments $activity -Owner $Owner -Repo $Repo $rawAuthorLogin = if ($pr.author -and $pr.author.login) { [string]$pr.author.login } else { '' } $authorLogin = Normalize-GitHubActorLogin $rawAuthorLogin @@ -103,7 +114,10 @@ foreach ($pr in @($searchResult)) { reviewCommandId = $reviewOptions.CommentId reviewCommand = $reviewOptions.Body labels = $labels - rerunCommentId = if ($latestRerun) { [Int64]$latestRerun.id } else { $null } + # The ready label does not encode whether this queue cycle came from a + # specific command or the autonomous labeler. Never reuse a historical + # /review rerun comment as this cycle's reaction target. + rerunCommentId = [Int64]0 contextMarkdown = $contextMarkdown } } @@ -118,3 +132,8 @@ $json | Set-Content -LiteralPath $OutputPath -Encoding UTF8 Write-Host "Wrote $($candidates.Count) rerun-ready candidate(s) to $OutputPath" Write-Output $json +} + +if ($MyInvocation.InvocationName -ne '.') { + Invoke-RerunReadyPRQuery +} diff --git a/.github/scripts/Resolve-RerunEligibility.Tests.ps1 b/.github/scripts/Resolve-RerunEligibility.Tests.ps1 index 984d09ab8bdf..02ef85bc6f20 100644 --- a/.github/scripts/Resolve-RerunEligibility.Tests.ps1 +++ b/.github/scripts/Resolve-RerunEligibility.Tests.ps1 @@ -540,3 +540,186 @@ new $context | Should -Match 'New non-command author comments: 0' } } + +Describe 'Resolve-AutonomousRerunEligibility' { + It 'rejects a PR that was never AI-reviewed (no AI Summary)' { + $comments = @( + New-TestComment -Id 1 -Body 'Some author update.' -CreatedAt '2026-05-31T09:00:00Z' + ) + + $result = Resolve-AutonomousRerunEligibility -Comments $comments -Commits @() -CurrentHeadSha 'abcdef123' -PRAuthorLogin 'dev-user' + $result.Eligible | Should -BeFalse + $result.Reason | Should -Be 'no-ai-summary' + } + + It 'accepts a new PR-author comment after the latest AI Summary' { + $comments = @( + New-TestComment -Id 1 -Body (New-AISummaryBody) -CreatedAt '2026-05-31T09:00:00Z' -UpdatedAt '2026-05-31T09:30:00Z' -Login 'MauiBot' -Type 'User' + New-TestComment -Id 2 -Body 'I pushed the requested update.' -CreatedAt '2026-05-31T09:45:00Z' + ) + + $result = Resolve-AutonomousRerunEligibility -Comments $comments -Commits @() -CurrentHeadSha 'abcdef123' -PRAuthorLogin 'dev-user' + $result.Eligible | Should -BeTrue + $result.Reason | Should -Be 'new-author-comment-after-ai-summary' + } + + It 'accepts a new commit after the latest AI Summary' { + $comments = @( + New-TestComment -Id 1 -Body (New-AISummaryBody) -CreatedAt '2026-05-31T09:00:00Z' -UpdatedAt '2026-05-31T09:30:00Z' -Login 'MauiBot' -Type 'User' + ) + $commits = @( + New-TestCommit -Sha 'aaaaaaa' -Date '2026-05-31T09:45:00Z' + ) + + $result = Resolve-AutonomousRerunEligibility -Comments $comments -Commits $commits -CurrentHeadSha 'abcdef123' -PRAuthorLogin 'dev-user' + $result.Eligible | Should -BeTrue + $result.Reason | Should -Be 'new-commit-after-ai-summary' + } + + It 'accepts a head SHA that differs from the last reviewed SHA' { + $comments = @( + New-TestComment -Id 1 -Body (New-AISummaryBody -Sha '1111111') -CreatedAt '2026-05-31T09:00:00Z' -UpdatedAt '2026-05-31T09:30:00Z' -Login 'MauiBot' -Type 'User' + ) + + $result = Resolve-AutonomousRerunEligibility -Comments $comments -Commits @() -CurrentHeadSha '2222222abcdef' -PRAuthorLogin 'dev-user' + $result.Eligible | Should -BeTrue + $result.Reason | Should -Be 'new-head-commit' + } + + It 'reports eligible with reason label-already-present when the ready-for-rerun label is already applied' { + $comments = @( + New-TestComment -Id 1 -Body (New-AISummaryBody) -CreatedAt '2026-05-31T09:00:00Z' -UpdatedAt '2026-05-31T09:30:00Z' -Login 'MauiBot' -Type 'User' + New-TestComment -Id 2 -Body 'I pushed the requested update.' -CreatedAt '2026-05-31T09:45:00Z' + ) + + $result = Resolve-AutonomousRerunEligibility -Comments $comments -Commits @() -CurrentHeadSha 'abcdef123' -PRAuthorLogin 'dev-user' -CurrentLabels @('s/agent-ready-for-rerun') + $result.Eligible | Should -BeTrue + $result.Reason | Should -Be 'label-already-present' + } + + It 'skips when a review is already in progress' { + $comments = @( + New-TestComment -Id 1 -Body (New-AISummaryBody) -CreatedAt '2026-05-31T09:00:00Z' -UpdatedAt '2026-05-31T09:30:00Z' -Login 'MauiBot' -Type 'User' + New-TestComment -Id 2 -Body 'I pushed the requested update.' -CreatedAt '2026-05-31T09:45:00Z' + ) + + $result = Resolve-AutonomousRerunEligibility -Comments $comments -Commits @() -CurrentHeadSha 'abcdef123' -PRAuthorLogin 'dev-user' -CurrentLabels @('s/agent-review-in-progress') + $result.Eligible | Should -BeFalse + $result.Reason | Should -Be 'review-in-progress' + } + + It 'rejects when there is no new activity since the AI Summary' { + $comments = @( + New-TestComment -Id 1 -Body (New-AISummaryBody) -CreatedAt '2026-05-31T09:00:00Z' -UpdatedAt '2026-05-31T09:30:00Z' -Login 'MauiBot' -Type 'User' + ) + + $result = Resolve-AutonomousRerunEligibility -Comments $comments -Commits @() -CurrentHeadSha 'abcdef123' -PRAuthorLogin 'dev-user' + $result.Eligible | Should -BeFalse + $result.Reason | Should -Be 'no-new-comments-or-commits' + } + + It 'ignores a maintainer (non-author) comment after the AI Summary' { + $comments = @( + New-TestComment -Id 1 -Body (New-AISummaryBody) -CreatedAt '2026-05-31T09:00:00Z' -UpdatedAt '2026-05-31T09:30:00Z' -Login 'MauiBot' -Type 'User' + New-TestComment -Id 2 -Body 'Please address the AI feedback.' -CreatedAt '2026-05-31T09:45:00Z' -Login 'maintainer' + ) + + $result = Resolve-AutonomousRerunEligibility -Comments $comments -Commits @() -CurrentHeadSha 'abcdef123' -PRAuthorLogin 'dev-user' + $result.Eligible | Should -BeFalse + $result.Reason | Should -Be 'no-new-comments-or-commits' + } + + Context 'anti-flap: last-declined checkpoint (scanner skip)' { + It 'does not re-qualify a head that only differs from the summary when the scanner already declined it and nothing new landed' { + # AI Summary reviewed 1111111; head is 2222222 (differs). An author comment + # landed after the summary but BEFORE the scanner removed the label. With the + # decline checkpoint that state is already-declined, so it must not flap back on. + $comments = @( + New-TestComment -Id 1 -Body (New-AISummaryBody -Sha '1111111') -CreatedAt '2026-05-31T09:00:00Z' -UpdatedAt '2026-05-31T09:30:00Z' -Login 'MauiBot' -Type 'User' + New-TestComment -Id 2 -Body 'I pushed the update.' -CreatedAt '2026-05-31T09:45:00Z' + ) + + $result = Resolve-AutonomousRerunEligibility -Comments $comments -Commits @() -CurrentHeadSha '2222222abcdef' -PRAuthorLogin 'dev-user' -LastDeclinedAt '2026-05-31T10:00:00Z' + $result.Eligible | Should -BeFalse + $result.Reason | Should -Be 'declined-state-unchanged' + } + + It 're-qualifies (new-head-commit) when a commit lands after the decline' { + $comments = @( + New-TestComment -Id 1 -Body (New-AISummaryBody -Sha '1111111') -CreatedAt '2026-05-31T09:00:00Z' -UpdatedAt '2026-05-31T09:30:00Z' -Login 'MauiBot' -Type 'User' + ) + $commits = @( + New-TestCommit -Sha '2222222abcdef' -Date '2026-05-31T10:30:00Z' + ) + + $result = Resolve-AutonomousRerunEligibility -Comments $comments -Commits $commits -CurrentHeadSha '2222222abcdef' -PRAuthorLogin 'dev-user' -LastDeclinedAt '2026-05-31T10:00:00Z' + $result.Eligible | Should -BeTrue + $result.Reason | Should -Be 'new-head-commit' + } + + It 'does not re-qualify when the differing-head commit predates the decline' { + $comments = @( + New-TestComment -Id 1 -Body (New-AISummaryBody -Sha '1111111') -CreatedAt '2026-05-31T09:00:00Z' -UpdatedAt '2026-05-31T09:30:00Z' -Login 'MauiBot' -Type 'User' + ) + $commits = @( + New-TestCommit -Sha '2222222abcdef' -Date '2026-05-31T09:45:00Z' + ) + + $result = Resolve-AutonomousRerunEligibility -Comments $comments -Commits $commits -CurrentHeadSha '2222222abcdef' -PRAuthorLogin 'dev-user' -LastDeclinedAt '2026-05-31T10:00:00Z' + $result.Eligible | Should -BeFalse + $result.Reason | Should -Be 'declined-state-unchanged' + } + + It 're-qualifies when the current head differs from the exact declined head even if the marker is newer' { + $comments = @( + New-TestComment -Id 1 -Body (New-AISummaryBody -Sha '1111111') -CreatedAt '2026-05-31T09:00:00Z' -UpdatedAt '2026-05-31T09:30:00Z' -Login 'MauiBot' -Type 'User' + ) + + $result = Resolve-AutonomousRerunEligibility ` + -Comments $comments ` + -Commits @() ` + -CurrentHeadSha '3333333333333333333333333333333333333333' ` + -PRAuthorLogin 'dev-user' ` + -LastDeclinedAt '2026-05-31T10:00:00Z' ` + -LastDeclinedHeadSha '2222222222222222222222222222222222222222' + + $result.Eligible | Should -BeTrue + $result.Reason | Should -Be 'new-head-commit' + } + + It 're-qualifies on a fresh PR-author comment posted after the decline' { + $comments = @( + New-TestComment -Id 1 -Body (New-AISummaryBody -Sha '1111111') -CreatedAt '2026-05-31T09:00:00Z' -UpdatedAt '2026-05-31T09:30:00Z' -Login 'MauiBot' -Type 'User' + New-TestComment -Id 2 -Body 'Any update on this?' -CreatedAt '2026-05-31T10:30:00Z' + ) + + $result = Resolve-AutonomousRerunEligibility -Comments $comments -Commits @() -CurrentHeadSha '1111111abcdef' -PRAuthorLogin 'dev-user' -LastDeclinedAt '2026-05-31T10:00:00Z' + $result.Eligible | Should -BeTrue + $result.Reason | Should -Be 'new-author-comment-after-ai-summary' + } + + It 'ignores a decline that predates the latest AI Summary (trigger-path removal superseded by the fresh summary)' { + $comments = @( + New-TestComment -Id 1 -Body (New-AISummaryBody -Sha '1111111') -CreatedAt '2026-05-31T09:00:00Z' -UpdatedAt '2026-05-31T09:30:00Z' -Login 'MauiBot' -Type 'User' + New-TestComment -Id 2 -Body 'I pushed the update.' -CreatedAt '2026-05-31T09:45:00Z' + ) + + # Label removed at 08:00 (before the 09:00 summary) — a completed review's removal, + # not a skip — so it must not suppress the genuine post-summary author comment. + $result = Resolve-AutonomousRerunEligibility -Comments $comments -Commits @() -CurrentHeadSha '1111111abcdef' -PRAuthorLogin 'dev-user' -LastDeclinedAt '2026-05-31T08:00:00Z' + $result.Eligible | Should -BeTrue + $result.Reason | Should -Be 'new-author-comment-after-ai-summary' + } + + It 'falls back to the summary checkpoint when LastDeclinedAt is malformed' { + $comments = @( + New-TestComment -Id 1 -Body (New-AISummaryBody -Sha '1111111') -CreatedAt '2026-05-31T09:00:00Z' -UpdatedAt '2026-05-31T09:30:00Z' -Login 'MauiBot' -Type 'User' + New-TestComment -Id 2 -Body 'I pushed the update.' -CreatedAt '2026-05-31T09:45:00Z' + ) + + $result = Resolve-AutonomousRerunEligibility -Comments $comments -Commits @() -CurrentHeadSha '1111111abcdef' -PRAuthorLogin 'dev-user' -LastDeclinedAt 'not-a-real-date' + $result.Eligible | Should -BeTrue + $result.Reason | Should -Be 'new-author-comment-after-ai-summary' + } + } +} diff --git a/.github/scripts/Resolve-RerunEligibility.ps1 b/.github/scripts/Resolve-RerunEligibility.ps1 index 4811cdbfe8d9..8541fd305999 100644 --- a/.github/scripts/Resolve-RerunEligibility.ps1 +++ b/.github/scripts/Resolve-RerunEligibility.ps1 @@ -626,6 +626,100 @@ function New-RerunContextMarkdown { return ($lines -join "`n") } +function Resolve-AutonomousRerunEligibility { + <# + .SYNOPSIS + Deterministically decides whether an already-AI-reviewed PR should be + auto-marked ready for rerun WITHOUT a `/review rerun` comment. + + .DESCRIPTION + This is the autonomous counterpart of Resolve-RerunEligibility used by + the PR Review Queue workflow. It applies the SAME deterministic signal — + genuinely new PR-author activity (a new non-command author comment, a new + commit, or a head SHA that differs from the last reviewed SHA) since the + latest AI Summary — but is not gated on a maintainer's `/review rerun` + comment. No AI is used and untrusted text is never inspected + semantically. A prior MauiBot AI Summary is REQUIRED: PRs that were never + AI-reviewed do not qualify. + #> + param( + [object[]]$Comments, + [object[]]$Commits, + [string]$CurrentHeadSha, + [string]$PRAuthorLogin, + [object[]]$CurrentLabels = @(), + # ISO-8601 timestamp of the most recent explicit scanner `skip` marker. + # When newer than the latest AI Summary it advances the checkpoint so the + # same declined state is not re-labelled on every daily run (anti-flap). + [string]$LastDeclinedAt, + # Exact PR head SHA stored by the scanner when it made that skip decision. + # A different current head always represents post-decline activity, even + # when the push raced the marker write and has an earlier commit timestamp. + [string]$LastDeclinedHeadSha + ) + + if (@($CurrentLabels | Where-Object { $_ -eq $ReviewInProgressLabel }).Count -gt 0) { + return [pscustomobject]@{ Eligible = $false; Reason = 'review-in-progress'; Label = $ReadyForRerunLabel } + } + + $latestSummary = Get-LatestAISummaryComment -Comments $Comments + if (-not $latestSummary) { + return [pscustomobject]@{ Eligible = $false; Reason = 'no-ai-summary'; Label = $ReadyForRerunLabel } + } + + if (@($CurrentLabels | Where-Object { $_ -eq $ReadyForRerunLabel }).Count -gt 0) { + return [pscustomobject]@{ Eligible = $true; Reason = 'label-already-present'; Label = $ReadyForRerunLabel } + } + + $summaryCreatedAt = Get-ObjectDate $latestSummary 'created_at' + $latestReviewedSha = Get-LatestReviewedSha -AISummaryBody $latestSummary.body + + # Anti-flap checkpoint: if the scanner explicitly declined this state more recently + # than the latest AI Summary, re-labelling requires genuinely NEW activity after + # that decline. Trigger-path and manual ready-label removals are not decline markers. + $effectiveCheckpoint = $summaryCreatedAt + $declinedAt = $null + if (-not [string]::IsNullOrWhiteSpace($LastDeclinedAt)) { + try { $declinedAt = ConvertTo-DateTimeOffset $LastDeclinedAt } catch { $declinedAt = $null } + if ($declinedAt -and $declinedAt -gt $effectiveCheckpoint) { + $effectiveCheckpoint = $declinedAt + } + } + $isDeclineGated = [bool]($declinedAt -and $declinedAt -gt $summaryCreatedAt) + $headDiffersFromDeclined = $isDeclineGated -and + (Test-HeadDiffersFromReviewedSha -CurrentHeadSha $CurrentHeadSha -LatestReviewedSha $LastDeclinedHeadSha) + + if ($headDiffersFromDeclined) { + return [pscustomobject]@{ Eligible = $true; Reason = 'new-head-commit'; Label = $ReadyForRerunLabel } + } + + $normalizedPRAuthorLogin = Normalize-GitHubActorLogin $PRAuthorLogin + $hasNewComment = Test-HasEvidenceCommentAfter -Comments $Comments -Checkpoint $effectiveCheckpoint -CurrentCommentId 0 -PRAuthorLogin $normalizedPRAuthorLogin + $hasNewCommit = Test-HasCommitAfter -Commits $Commits -Checkpoint $effectiveCheckpoint + $headDiffers = Test-HeadDiffersFromReviewedSha -CurrentHeadSha $CurrentHeadSha -LatestReviewedSha $latestReviewedSha + + # A head SHA that differs from the last-reviewed SHA only re-qualifies when it is + # backed by a commit that landed after the checkpoint. Absent a decline this is + # always true (the differing head IS that post-summary push), so behaviour is + # unchanged; once a decline advances the checkpoint, a head that merely still + # differs from the summary's SHA (the exact state the scanner declined) no longer + # counts — only a fresh push after the decline does. + if ($headDiffers -and (-not $isDeclineGated -or $hasNewCommit)) { + return [pscustomobject]@{ Eligible = $true; Reason = 'new-head-commit'; Label = $ReadyForRerunLabel } + } + + if ($hasNewComment) { + return [pscustomobject]@{ Eligible = $true; Reason = 'new-author-comment-after-ai-summary'; Label = $ReadyForRerunLabel } + } + + if ($hasNewCommit) { + return [pscustomobject]@{ Eligible = $true; Reason = 'new-commit-after-ai-summary'; Label = $ReadyForRerunLabel } + } + + $noNewReason = if ($isDeclineGated) { 'declined-state-unchanged' } else { 'no-new-comments-or-commits' } + return [pscustomobject]@{ Eligible = $false; Reason = $noNewReason; Label = $ReadyForRerunLabel } +} + function Resolve-RerunEligibility { param( [object[]]$Comments, @@ -756,10 +850,16 @@ if ($env:GITHUB_OUTPUT) { if ($ApplyLabel -and $result.Eligible) { . "$PSScriptRoot/shared/Update-AgentLabels.ps1" + # Derive the label description/color from the shared canonical definition (same pattern as + # Query-AutoRerunCandidates.ps1) so this script and Update-AgentLabels.ps1 can't drift and + # repeatedly re-PATCH each other's metadata back and forth depending on which ran last. + $rerunLabelDef = $AllLabelDefs[$ReadyForRerunLabel] + $rerunLabelDescription = if ($rerunLabelDef) { $rerunLabelDef.Description } else { $ReadyForRerunLabelDescription } + $rerunLabelColor = if ($rerunLabelDef) { $rerunLabelDef.Color } else { $ReadyForRerunLabelColor } Ensure-LabelExists ` -LabelName $ReadyForRerunLabel ` - -Description $ReadyForRerunLabelDescription ` - -Color $ReadyForRerunLabelColor ` + -Description $rerunLabelDescription ` + -Color $rerunLabelColor ` -Owner $Owner ` -Repo $Repo @@ -768,10 +868,17 @@ if ($ApplyLabel -and $result.Eligible) { Write-Host " ✅ Already present: $ReadyForRerunLabel" -ForegroundColor Green } else { $addSucceeded = Add-Label -PRNumber $PRNumber -LabelName $ReadyForRerunLabel -Owner $Owner -Repo $Repo - $updatedLabels = @(gh api "repos/$Owner/$Repo/issues/$PRNumber/labels" --jq '.[].name' 2>$null) - $labelIsPresent = @($updatedLabels | Where-Object { $_ -eq $ReadyForRerunLabel }).Count -gt 0 + # Best-effort re-read to confirm. Surface gh's stderr (no 2>$null) and check the exit + # code so a rate-limited/unauthorized/transient verification failure isn't misread as + # "label absent" — that would throw a misleading "Failed to apply label" even though + # Add-Label may have succeeded. + $updatedLabels = @(gh api "repos/$Owner/$Repo/issues/$PRNumber/labels" --jq '.[].name') + $verificationSucceeded = ($LASTEXITCODE -eq 0) + $labelIsPresent = $verificationSucceeded -and (@($updatedLabels | Where-Object { $_ -eq $ReadyForRerunLabel }).Count -gt 0) if ($addSucceeded -or $labelIsPresent) { Write-Host " ✅ Applied: $ReadyForRerunLabel" -ForegroundColor Green + } elseif (-not $verificationSucceeded) { + throw "Could not verify label '$ReadyForRerunLabel' after applying it (gh api re-read exited $LASTEXITCODE)." } else { throw "Failed to apply label: $ReadyForRerunLabel" } diff --git a/.github/scripts/shared/Update-AgentLabels.Tests.ps1 b/.github/scripts/shared/Update-AgentLabels.Tests.ps1 index e6803840a0b9..591c884e849b 100644 --- a/.github/scripts/shared/Update-AgentLabels.Tests.ps1 +++ b/.github/scripts/shared/Update-AgentLabels.Tests.ps1 @@ -20,7 +20,7 @@ BeforeAll { throw ($parseErrors | ForEach-Object { $_.Message }) -join [Environment]::NewLine } - # Extract only the pure-function we are testing (it reads files, makes no gh/network calls). + # Extract only the functions under test. $function = $ast.Find({ $args[0] -is [System.Management.Automation.Language.FunctionDefinitionAst] -and $args[0].Name -eq 'Parse-PhaseOutcomes' @@ -28,6 +28,24 @@ BeforeAll { if (-not $function) { throw "Function 'Parse-PhaseOutcomes' not found" } Invoke-Expression $function.Extent.Text + $clearDeclinedFunction = $ast.Find({ + $args[0] -is [System.Management.Automation.Language.FunctionDefinitionAst] -and + $args[0].Name -eq 'Clear-AgentRerunDeclined' + }, $true) + if (-not $clearDeclinedFunction) { throw "Function 'Clear-AgentRerunDeclined' not found" } + Invoke-Expression $clearDeclinedFunction.Extent.Text + + $removeLabelFunction = $ast.Find({ + $args[0] -is [System.Management.Automation.Language.FunctionDefinitionAst] -and + $args[0].Name -eq 'Remove-Label' + }, $true) + if (-not $removeLabelFunction) { throw "Function 'Remove-Label' not found" } + Invoke-Expression ($removeLabelFunction.Extent.Text -replace '^function Remove-Label', 'function Invoke-RemoveLabelUnderTest') + + function Remove-Label { + param([string]$PRNumber, [string]$LabelName, [string]$Owner, [string]$Repo) + } + # Helper: build a fake repo root with a PRAgent artifact dir and optional files. function New-FixtureRoot { param( @@ -49,6 +67,56 @@ BeforeAll { } } +Describe 'Remove-Label' { + BeforeEach { + Mock gh { + $global:LASTEXITCODE = 0 + } + } + + It 'treats an already-absent label as a successful idempotent removal' { + Mock gh { + $global:LASTEXITCODE = 1 + 'gh: Label does not exist (HTTP 404)' + } + + Invoke-RemoveLabelUnderTest -PRNumber 1 -LabelName 's/agent-rerun-declined' | + Should -BeTrue + } + + It 'reports a non-404 API failure' { + Mock gh { + $global:LASTEXITCODE = 1 + 'gh: API rate limit exceeded (HTTP 403)' + } + + Invoke-RemoveLabelUnderTest -PRNumber 1 -LabelName 's/agent-rerun-declined' | + Should -BeFalse + } +} + +Describe 'Clear-AgentRerunDeclined' { + BeforeEach { + Mock Remove-Label { $true } + } + + It 'unconditionally performs an idempotent removal' { + Clear-AgentRerunDeclined -PRNumber 1 -Owner dotnet -Repo maui | + Should -BeTrue + Should -Invoke Remove-Label -Times 1 -ParameterFilter { + $PRNumber -eq '1' -and $LabelName -eq 's/agent-rerun-declined' -and + $Owner -eq 'dotnet' -and $Repo -eq 'maui' + } + } + + It 'reports a failed marker removal' { + Mock Remove-Label { $false } + + Clear-AgentRerunDeclined -PRNumber 1 | + Should -BeFalse + } +} + Describe 'Parse-PhaseOutcomes — Fix result from winner.json' { It 'maps isPRFix=false (alternative won) to win => s/agent-fix-win' { $root = New-FixtureRoot -WinnerJson '{ "winner": "try-fix-1", "isPRFix": false }' diff --git a/.github/scripts/shared/Update-AgentLabels.ps1 b/.github/scripts/shared/Update-AgentLabels.ps1 index acda6e483412..ff9226476ac9 100644 --- a/.github/scripts/shared/Update-AgentLabels.ps1 +++ b/.github/scripts/shared/Update-AgentLabels.ps1 @@ -38,6 +38,7 @@ $script:SignalLabels = @{ $script:ManualLabels = @{ 's/agent-fix-implemented' = @{ Description = 'PR author implemented the agent suggested fix'; Color = '7B1FA2' } 's/agent-ready-for-rerun' = @{ Description = 'AI review has new PR activity and is ready for rerun'; Color = '5319E7' } + 's/agent-rerun-declined' = @{ Description = 'AI rerun scanner declined the current PR state; new author activity is required'; Color = 'D4C5F9' } 's/agent-review-in-progress' = @{ Description = 'AI review is currently running for this PR'; Color = 'FBCA04' } } @@ -167,9 +168,26 @@ function Remove-Label { [string]$Repo = 'maui' ) - & gh api "repos/$Owner/$Repo/issues/$PRNumber/labels/$([uri]::EscapeDataString($LabelName))" ` - --method DELETE 1>$null 2>$null - return $LASTEXITCODE -eq 0 + $output = & gh api "repos/$Owner/$Repo/issues/$PRNumber/labels/$([uri]::EscapeDataString($LabelName))" ` + --method DELETE 2>&1 + $exitCode = $LASTEXITCODE + if ($exitCode -eq 0) { + return $true + } + + $message = ($output | Out-String).Trim() + if ($message -match '(?i)\bHTTP\s+404\b|\bstatus(?:\s+code)?\s*:?\s*404\b|\blabel does not exist\b') { + return $true + } + + if ([string]::IsNullOrWhiteSpace($message)) { + $message = "gh api exited with code $exitCode." + } elseif ($message.Length -gt 1000) { + $message = $message.Substring(0, 1000) + '...' + } + + Write-Host " ⚠️ Failed to remove label '$LabelName' from PR #$PRNumber (gh api exit code $exitCode): $message" -ForegroundColor Yellow + return $false } # ============================================================ @@ -239,6 +257,26 @@ function Clear-AgentReviewInProgress { return $false } +# ============================================================ +# Clear-AgentRerunDeclined +# ============================================================ +function Clear-AgentRerunDeclined { + param( + [Parameter(Mandatory)] [string]$PRNumber, + [string]$Owner = 'dotnet', + [string]$Repo = 'maui' + ) + + $label = 's/agent-rerun-declined' + $ok = Remove-Label -PRNumber $PRNumber -LabelName $label -Owner $Owner -Repo $Repo + if ($ok) { + Write-Host " ✅ Cleared or already absent: $label" -ForegroundColor Green + } else { + Write-Host " ⚠️ Failed to remove: $label" -ForegroundColor Yellow + } + return $ok +} + # ============================================================ # Test-AgentReviewInProgressIsStale # ============================================================ diff --git a/.github/workflows/pr-review-queue.yml b/.github/workflows/pr-review-queue.yml index 4f3f500fa614..60bcfd20cb2d 100644 --- a/.github/workflows/pr-review-queue.yml +++ b/.github/workflows/pr-review-queue.yml @@ -9,6 +9,9 @@ on: paths: - '.github/workflows/pr-review-queue.yml' - '.github/skills/find-reviewable-pr/**' + - '.github/scripts/Query-AutoRerunCandidates.ps1' + - '.github/scripts/Resolve-RerunEligibility.ps1' + - '.github/scripts/shared/Update-AgentLabels.ps1' permissions: contents: read @@ -85,6 +88,37 @@ jobs: --label "report" \ --label "s/triaged" + # Autonomously apply s/agent-ready-for-rerun to PRs with genuinely new + # PR-author activity since their last AI review — the same deterministic + # signal as a maintainer's `/review rerun`. The hourly rerun-review-scanner + # then picks these up and re-reviews them. Failures here are surfaced as a + # warning and never block the queue issue that was already created above. + - name: Auto-label rerun-ready PRs + env: + GH_TOKEN: ${{ github.token }} + REPO_OWNER: ${{ github.repository_owner }} + REPO_NAME: ${{ github.event.repository.name }} + run: | + set +e + pwsh .github/scripts/Query-AutoRerunCandidates.ps1 \ + -Owner "$REPO_OWNER" \ + -Repo "$REPO_NAME" \ + -OutputPath auto-rerun-decisions.json + rc=$? + set -e + if [ $rc -ne 0 ]; then + echo "::warning::Auto-rerun labeling step failed (rc=$rc); the queue issue was still created." + fi + + - name: Upload auto-rerun decisions + if: always() + uses: actions/upload-artifact@v4 + with: + name: auto-rerun-decisions + path: auto-rerun-decisions.json + if-no-files-found: warn + retention-days: 7 + # Dry-run on PRs: validate the script works without creating issues validate: runs-on: ubuntu-latest @@ -92,8 +126,13 @@ jobs: permissions: contents: read pull-requests: read + issues: read # Query-AutoRerunCandidates reads issue labels (gh api repos/.../issues/N/labels) steps: - uses: actions/checkout@v4 + with: + # This job runs on pull_request (untrusted PR code); it only reads and dry-runs the + # scripts, so it never needs the workflow token in .git/config. Match review-trigger.yml. + persist-credentials: false - name: Validate PR review queue script env: @@ -123,3 +162,25 @@ jobs: echo "✅ Validation passed — report preview:" cat pr-review-queue-body.md + + - name: Validate auto-rerun labeler (dry-run) + env: + GH_TOKEN: ${{ github.token }} + REPO_OWNER: ${{ github.repository_owner }} + REPO_NAME: ${{ github.event.repository.name }} + run: | + set +e + pwsh .github/scripts/Query-AutoRerunCandidates.ps1 \ + -Owner "$REPO_OWNER" \ + -Repo "$REPO_NAME" \ + -DryRun \ + -Limit 5 \ + -OutputPath auto-rerun-decisions.json + rc=$? + set -e + if [ $rc -ne 0 ]; then + echo "::error::Auto-rerun labeler dry-run failed" + exit 1 + fi + echo "✅ Auto-rerun labeler dry-run passed:" + cat auto-rerun-decisions.json diff --git a/.github/workflows/rerun-review-scanner.lock.yml b/.github/workflows/rerun-review-scanner.lock.yml index 2d0e94147378..119fc69da372 100644 --- a/.github/workflows/rerun-review-scanner.lock.yml +++ b/.github/workflows/rerun-review-scanner.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"d2392173a145bf7aaff1c3d9adee8e445d8eb995355b12131286b43910f5625c","body_hash":"1d0b4c7935e9c0baaf8f4e375a00c9eaaf58dae4b2fb2d128989f1f4c93e7561","compiler_version":"v0.85.4","strict":true,"agent_id":"copilot","agent_model":"gpt-5.6-sol","engine_versions":{"copilot":"1.0.78"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"5f6ad1b8c667f53fc70d6b3217cab6857e7348de3620eb13e65ab30713dbad80","body_hash":"3d7e66002c3647cf82b380c6625f9e9b3de327517d4d644b15e06ac464671033","compiler_version":"v0.85.4","strict":true,"agent_id":"copilot","agent_model":"gpt-5.6-sol","engine_versions":{"copilot":"1.0.78"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_PAT_0","COPILOT_PAT_1","COPILOT_PAT_2","COPILOT_PAT_3","COPILOT_PAT_4","COPILOT_PAT_5","COPILOT_PAT_6","COPILOT_PAT_7","COPILOT_PAT_8","COPILOT_PAT_9","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"2709137ea6c5b0e19aa621454dc643ea8dc526b1","version":"v0.85.4"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44","digest":"sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44@sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44","digest":"sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44@sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44","digest":"sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44@sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.8","digest":"sha256:38bbea36cdb46a3c9d04d1db05e672966f5239b431a2022eb35881688e5721d8","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.8@sha256:38bbea36cdb46a3c9d04d1db05e672966f5239b431a2022eb35881688e5721d8"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196"},{"image":"ghcr.io/github/github-mcp-server:v1.8.0","digest":"sha256:d5a18c04b92714c309eb46a2305087e91a4dbd80420f6e462656699f95093520","pinned_image":"ghcr.io/github/github-mcp-server:v1.8.0@sha256:d5a18c04b92714c309eb46a2305087e91a4dbd80420f6e462656699f95093520"}]} # This file was automatically generated by gh-aw (v0.85.4). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -581,9 +581,9 @@ jobs: mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_eb2f1c68c5e4565d_EOF' - {"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"false"},"report_incomplete":{},"trigger-rerun-review":{"description":"Apply validated rerun scanner decisions. Call EXACTLY ONCE per run, passing a `decisions` JSON array with one entry per candidate PR.","inputs":{"decisions":{"default":null,"description":"JSON array of decision objects, one per candidate PR. Each object: pr_number (string), decision ('trigger'|'skip'), rerun_comment_id (string), expected_head_sha (string), reason (short string), and optional platform and pipeline_ref strings.","required":true,"type":"string"}},"output":"Rerun scanner decisions processed."}} - GH_AW_SAFE_OUTPUTS_CONFIG_eb2f1c68c5e4565d_EOF + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_a773574eb8306318_EOF' + {"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"false"},"report_incomplete":{},"trigger-rerun-review":{"description":"Apply validated rerun scanner decisions. Call EXACTLY ONCE per run, passing a `decisions` JSON array with one entry per candidate PR.","inputs":{"decisions":{"default":null,"description":"JSON array of decision objects, one per candidate PR. Each object: pr_number (string), decision ('trigger'|'skip'), rerun_comment_id ('0' unless a current-cycle source is proven), expected_head_sha (string), reason (short string), and optional platform and pipeline_ref strings.","required":true,"type":"string"}},"output":"Rerun scanner decisions processed."}} + GH_AW_SAFE_OUTPUTS_CONFIG_a773574eb8306318_EOF - name: Generate Safe Outputs Tools env: GH_AW_TOOLS_META_JSON: | @@ -597,7 +597,7 @@ jobs: "additionalProperties": false, "properties": { "decisions": { - "description": "JSON array of decision objects, one per candidate PR. Each object: pr_number (string), decision ('trigger'|'skip'), rerun_comment_id (string), expected_head_sha (string), reason (short string), and optional platform and pipeline_ref strings.", + "description": "JSON array of decision objects, one per candidate PR. Each object: pr_number (string), decision ('trigger'|'skip'), rerun_comment_id ('0' unless a current-cycle source is proven), expected_head_sha (string), reason (short string), and optional platform and pipeline_ref strings.", "type": "string" } }, @@ -1866,6 +1866,11 @@ jobs: script: | const fs = require('fs'); const readyLabel = 's/agent-ready-for-rerun'; + const declinedLabel = { + name: 's/agent-rerun-declined', + description: 'AI rerun scanner declined the current PR state; new author activity is required', + color: 'D4C5F9', + }; const dryRun = process.env.DRY_RUN === 'true'; const actionsPath = process.env.RERUN_ACTIONS_PATH; const { owner, repo } = context.repo; @@ -1904,7 +1909,101 @@ jobs: core.info(`Removed ${readyLabel} from PR #${prNumber}`); } catch (e) { if (e.status === 404) { core.info(`${readyLabel} already absent on PR #${prNumber}`); } - else { core.warning(`Failed to remove ${readyLabel} from PR #${prNumber}: ${e.message}`); } + else { throw new Error(`Failed to remove ${readyLabel} from PR #${prNumber}: ${e.message}`); } + } + } + + async function syncDeclinedLabel(existing) { + if (existing.data.description !== declinedLabel.description || existing.data.color.toUpperCase() !== declinedLabel.color) { + await github.rest.issues.updateLabel({ + owner, repo, + name: declinedLabel.name, + new_name: declinedLabel.name, + description: declinedLabel.description, + color: declinedLabel.color, + }); + } + } + + async function ensureDeclinedLabel() { + try { + const existing = await github.rest.issues.getLabel({ owner, repo, name: declinedLabel.name }); + await syncDeclinedLabel(existing); + } catch (e) { + if (e.status !== 404) { throw e; } + try { + await github.rest.issues.createLabel({ owner, repo, ...declinedLabel }); + } catch (createError) { + if (createError.status !== 422) { throw createError; } + const existing = await github.rest.issues.getLabel({ owner, repo, name: declinedLabel.name }); + await syncDeclinedLabel(existing); + core.info(`${declinedLabel.name} was created concurrently; using the existing label.`); + } + } + } + + async function markDeclined(prNumber, headSha) { + if (dryRun) { core.info(`[dry-run] Would record ${declinedLabel.name} for PR #${prNumber} at ${headSha}`); return; } + await ensureDeclinedLabel(); + const issue = await github.rest.issues.get({ owner, repo, issue_number: prNumber }); + const alreadyLabelled = issue.data.labels.some( + label => (typeof label === 'string' ? label : label.name) === declinedLabel.name, + ); + + const markerText = ``; + const existing = await github.graphql( + `query($owner:String!,$repo:String!,$number:Int!){ + repository(owner:$owner,name:$repo){ + pullRequest(number:$number){ + comments(last:100){nodes{id body}} + } + } + }`, + { owner, repo, number: prNumber }, + ); + const existingMarker = existing.repository.pullRequest.comments.nodes.find( + comment => comment.body.includes(markerText), + ); + + if (alreadyLabelled && existingMarker) { + core.info(`${declinedLabel.name} already records unchanged head ${headSha} for PR #${prNumber}`); + return; + } else if (existingMarker) { + core.info(`Decline marker for PR #${prNumber} at ${headSha} already exists; restoring its label only.`); + } else { + const marker = await github.rest.issues.createComment({ + owner, repo, + issue_number: prNumber, + body: `${markerText}\nAutomated rerun scanner checkpoint for declined head \`${headSha.slice(0, 12)}\`.`, + }); + try { + await github.graphql( + 'mutation($id:ID!){minimizeComment(input:{subjectId:$id,classifier:RESOLVED}){minimizedComment{isMinimized}}}', + { id: marker.data.node_id }, + ); + } catch (e) { + core.warning(`Recorded decline head for PR #${prNumber}, but could not minimize marker comment: ${e.message}`); + } + } + await github.rest.issues.addLabels({ + owner, repo, + issue_number: prNumber, + labels: [declinedLabel.name], + }); + core.info(`Applied ${declinedLabel.name} to PR #${prNumber}`); + } + + async function clearDeclined(prNumber) { + if (dryRun) { core.info(`[dry-run] Would remove ${declinedLabel.name} from PR #${prNumber}`); return; } + try { + await github.rest.issues.removeLabel({ owner, repo, issue_number: prNumber, name: declinedLabel.name }); + core.info(`Removed ${declinedLabel.name} from PR #${prNumber}`); + } catch (e) { + if (e.status === 404) { + core.info(`${declinedLabel.name} already absent on PR #${prNumber}`); + } else { + core.warning(`Could not remove advisory label ${declinedLabel.name} from PR #${prNumber}; continuing dispatch: ${e.message}`); + } } } @@ -1915,6 +2014,9 @@ jobs: if (dryRun) { core.info(`[dry-run] Would dispatch review-trigger.yml for PR #${prNumber} (platform=${a.platform}, pipeline_ref=${a.pipelineRef})`); } else { + // A previous semantic skip must not suppress recovery if this + // dispatch or the downstream review fails before posting a summary. + await clearDeclined(prNumber); await github.rest.actions.createWorkflowDispatch({ owner, repo, workflow_id: 'review-trigger.yml', @@ -1958,6 +2060,9 @@ jobs: } else if (a.headSha && liveHeadSha !== a.headSha) { core.info(`Skip: PR #${prNumber} head advanced ${a.headSha} -> ${liveHeadSha} since the scan; leaving ${readyLabel} for re-evaluation.`); } else { + // Persist an explicit semantic-skip checkpoint before consuming + // the queue label. Generic ready-label removals are not declines. + await markDeclined(prNumber, liveHeadSha); await react(a.rerunCommentId, '-1'); await removeReadyLabel(prNumber); } diff --git a/.github/workflows/rerun-review-scanner.md b/.github/workflows/rerun-review-scanner.md index 8a2c75e1d636..49a5d9e201c5 100644 --- a/.github/workflows/rerun-review-scanner.md +++ b/.github/workflows/rerun-review-scanner.md @@ -115,7 +115,7 @@ safe-outputs: # scanner to a single PR per run. Batching all decisions into one array # field lets a single invocation carry every candidate's decision. decisions: - description: "JSON array of decision objects, one per candidate PR. Each object: pr_number (string), decision ('trigger'|'skip'), rerun_comment_id (string), expected_head_sha (string), reason (short string), and optional platform and pipeline_ref strings." + description: "JSON array of decision objects, one per candidate PR. Each object: pr_number (string), decision ('trigger'|'skip'), rerun_comment_id ('0' unless a current-cycle source is proven), expected_head_sha (string), reason (short string), and optional platform and pipeline_ref strings." required: true type: string steps: @@ -154,6 +154,11 @@ safe-outputs: script: | const fs = require('fs'); const readyLabel = 's/agent-ready-for-rerun'; + const declinedLabel = { + name: 's/agent-rerun-declined', + description: 'AI rerun scanner declined the current PR state; new author activity is required', + color: 'D4C5F9', + }; const dryRun = process.env.DRY_RUN === 'true'; const actionsPath = process.env.RERUN_ACTIONS_PATH; const { owner, repo } = context.repo; @@ -192,7 +197,101 @@ safe-outputs: core.info(`Removed ${readyLabel} from PR #${prNumber}`); } catch (e) { if (e.status === 404) { core.info(`${readyLabel} already absent on PR #${prNumber}`); } - else { core.warning(`Failed to remove ${readyLabel} from PR #${prNumber}: ${e.message}`); } + else { throw new Error(`Failed to remove ${readyLabel} from PR #${prNumber}: ${e.message}`); } + } + } + + async function syncDeclinedLabel(existing) { + if (existing.data.description !== declinedLabel.description || existing.data.color.toUpperCase() !== declinedLabel.color) { + await github.rest.issues.updateLabel({ + owner, repo, + name: declinedLabel.name, + new_name: declinedLabel.name, + description: declinedLabel.description, + color: declinedLabel.color, + }); + } + } + + async function ensureDeclinedLabel() { + try { + const existing = await github.rest.issues.getLabel({ owner, repo, name: declinedLabel.name }); + await syncDeclinedLabel(existing); + } catch (e) { + if (e.status !== 404) { throw e; } + try { + await github.rest.issues.createLabel({ owner, repo, ...declinedLabel }); + } catch (createError) { + if (createError.status !== 422) { throw createError; } + const existing = await github.rest.issues.getLabel({ owner, repo, name: declinedLabel.name }); + await syncDeclinedLabel(existing); + core.info(`${declinedLabel.name} was created concurrently; using the existing label.`); + } + } + } + + async function markDeclined(prNumber, headSha) { + if (dryRun) { core.info(`[dry-run] Would record ${declinedLabel.name} for PR #${prNumber} at ${headSha}`); return; } + await ensureDeclinedLabel(); + const issue = await github.rest.issues.get({ owner, repo, issue_number: prNumber }); + const alreadyLabelled = issue.data.labels.some( + label => (typeof label === 'string' ? label : label.name) === declinedLabel.name, + ); + + const markerText = ``; + const existing = await github.graphql( + `query($owner:String!,$repo:String!,$number:Int!){ + repository(owner:$owner,name:$repo){ + pullRequest(number:$number){ + comments(last:100){nodes{id body}} + } + } + }`, + { owner, repo, number: prNumber }, + ); + const existingMarker = existing.repository.pullRequest.comments.nodes.find( + comment => comment.body.includes(markerText), + ); + + if (alreadyLabelled && existingMarker) { + core.info(`${declinedLabel.name} already records unchanged head ${headSha} for PR #${prNumber}`); + return; + } else if (existingMarker) { + core.info(`Decline marker for PR #${prNumber} at ${headSha} already exists; restoring its label only.`); + } else { + const marker = await github.rest.issues.createComment({ + owner, repo, + issue_number: prNumber, + body: `${markerText}\nAutomated rerun scanner checkpoint for declined head \`${headSha.slice(0, 12)}\`.`, + }); + try { + await github.graphql( + 'mutation($id:ID!){minimizeComment(input:{subjectId:$id,classifier:RESOLVED}){minimizedComment{isMinimized}}}', + { id: marker.data.node_id }, + ); + } catch (e) { + core.warning(`Recorded decline head for PR #${prNumber}, but could not minimize marker comment: ${e.message}`); + } + } + await github.rest.issues.addLabels({ + owner, repo, + issue_number: prNumber, + labels: [declinedLabel.name], + }); + core.info(`Applied ${declinedLabel.name} to PR #${prNumber}`); + } + + async function clearDeclined(prNumber) { + if (dryRun) { core.info(`[dry-run] Would remove ${declinedLabel.name} from PR #${prNumber}`); return; } + try { + await github.rest.issues.removeLabel({ owner, repo, issue_number: prNumber, name: declinedLabel.name }); + core.info(`Removed ${declinedLabel.name} from PR #${prNumber}`); + } catch (e) { + if (e.status === 404) { + core.info(`${declinedLabel.name} already absent on PR #${prNumber}`); + } else { + core.warning(`Could not remove advisory label ${declinedLabel.name} from PR #${prNumber}; continuing dispatch: ${e.message}`); + } } } @@ -203,6 +302,9 @@ safe-outputs: if (dryRun) { core.info(`[dry-run] Would dispatch review-trigger.yml for PR #${prNumber} (platform=${a.platform}, pipeline_ref=${a.pipelineRef})`); } else { + // A previous semantic skip must not suppress recovery if this + // dispatch or the downstream review fails before posting a summary. + await clearDeclined(prNumber); await github.rest.actions.createWorkflowDispatch({ owner, repo, workflow_id: 'review-trigger.yml', @@ -246,6 +348,9 @@ safe-outputs: } else if (a.headSha && liveHeadSha !== a.headSha) { core.info(`Skip: PR #${prNumber} head advanced ${a.headSha} -> ${liveHeadSha} since the scan; leaving ${readyLabel} for re-evaluation.`); } else { + // Persist an explicit semantic-skip checkpoint before consuming + // the queue label. Generic ready-label removals are not declines. + await markDeclined(prNumber, liveHeadSha); await react(a.rerunCommentId, '-1'); await removeReadyLabel(prNumber); } @@ -264,7 +369,7 @@ safe-outputs: # Rerun Review Scanner -You are scanning queued .NET MAUI PRs that already have the label `s/agent-ready-for-rerun`. +You are scanning queued .NET MAUI PRs that already have the label `s/agent-ready-for-rerun`. This label is applied either by a maintainer's `/review rerun` command or autonomously by the daily PR Review Queue workflow when it detects genuinely new PR-author activity since the last AI review. Both sources are valid and treated identically here. ## Concurrency, locking, and duplicate prevention @@ -286,8 +391,8 @@ OIDC exchange, and triggers the AzDO `maui-copilot` pipeline (which removes the lock in its final cleanup stage). `review-trigger.yml` also has a per-PR concurrency group and refuses to start when the in-progress lock is already present, so a dispatched rerun can never double-trigger a review that is already -running. For a `skip`, the safe-output job reacts `-1` and removes the queue -label itself. +running. For a `skip`, the safe-output job applies `s/agent-rerun-declined`, reacts +`-1`, and removes the queue label itself. Because `review-trigger.yml` consumes `s/agent-ready-for-rerun` when it locks+triggers, a queued PR is removed from the candidate set after its first @@ -305,11 +410,28 @@ so the scanner path behaves exactly like a maintainer `/review`, which has no such limit. Volume is instead bounded structurally: 1. A PR only becomes a candidate when `Resolve-RerunEligibility.ps1` finds - genuinely *new* author activity (a new commit or a new non-command comment) - since the last AI Summary / rerun checkpoint — the same deterministic gate the + genuinely *new* author activity (a new commit, a new non-command comment, or a + head commit SHA that differs from the last reviewed SHA) since the last AI Summary / + rerun checkpoint — the same deterministic gate the `/review rerun` command uses. The identical PR state cannot be re-queued. -2. Re-entry is not autonomous: a human (or the PR author's new push) must produce - that new activity and `/review rerun` must re-apply the queue label each cycle. +2. Re-entry requires genuinely new activity each cycle. The queue label is applied + either by a maintainer's `/review rerun` or autonomously by the PR Review Queue + workflow — but in both cases only when the deterministic gate + (`Resolve-RerunEligibility.ps1` / `Resolve-AutonomousRerunEligibility`) finds + new activity since the last AI Summary. When a review **completes** (the `trigger` + path) it posts a fresh AI Summary that advances the checkpoint, so the identical + PR state cannot re-qualify and autonomous re-entry cannot loop. + + > **Skip-path checkpoint (anti-flap):** the `trigger` path advances the checkpoint by + > posting a fresh AI Summary, but a `skip` decision does not. Before consuming + > `s/agent-ready-for-rerun`, the safe-output job applies the explicit + > `s/agent-rerun-declined` marker and writes a minimized bot comment containing the + > exact head SHA that was declined. `Query-AutoRerunCandidates.ps1` passes both that + > SHA and the comment timestamp to `Resolve-AutonomousRerunEligibility`. Re-labelling + > then requires genuinely new activity after the semantic skip, while any different + > current head always re-qualifies even if a push raced the marker write. Trigger-path + > and manual removals of the ready label are not decline checkpoints. Every successful + > review entrypoint clears the visible decline label. 3. The per-PR in-progress lock prevents overlapping reviews of the same PR. This is an accepted, documented cost trade-off: it matches manual `/review` @@ -337,12 +459,12 @@ Each object in the `decisions` array must use: - `pr_number`: the candidate `prNumber`. - `decision`: `trigger` or `skip`. -- `rerun_comment_id`: the candidate `rerunCommentId`. If it is missing, choose `skip` and use `"0"`. +- `rerun_comment_id`: use the candidate `rerunCommentId`, currently `"0"`. The queue label does not preserve which `/review rerun` command, if any, created the current cycle, so historical comments are never reused as reaction targets. A missing id is **not** a reason to skip; the dispatch does not require a comment. - `expected_head_sha`: the candidate `headSha`. - `platform`: the candidate `platform`. - `pipeline_ref`: the candidate `pipelineRef`. - `reason`: one short sentence. -Example: `decisions = "[{\"pr_number\":\"123\",\"decision\":\"trigger\",\"rerun_comment_id\":\"456\",\"expected_head_sha\":\"abc123\",\"platform\":\"android\",\"pipeline_ref\":\"main\",\"reason\":\"New commit addresses review feedback.\"}]"` +Example: `decisions = "[{\"pr_number\":\"123\",\"decision\":\"trigger\",\"rerun_comment_id\":\"0\",\"expected_head_sha\":\"abc123\",\"platform\":\"android\",\"pipeline_ref\":\"main\",\"reason\":\"New commit addresses review feedback.\"}]"` Do not call any other write tool. Do not create comments, labels, issues, or pull requests directly. The safe-output job will handle reactions, queue-label removal, and dispatching `review-trigger.yml` deterministically. diff --git a/.github/workflows/review-trigger.yml b/.github/workflows/review-trigger.yml index 2ca45b6ff37b..437aae754d36 100644 --- a/.github/workflows/review-trigger.yml +++ b/.github/workflows/review-trigger.yml @@ -255,6 +255,16 @@ jobs: } } + # Every review entrypoint best-effort clears a prior semantic decline, + # not only the scanner path. This bookkeeping must never block a review. + $declineCleared = Clear-AgentRerunDeclined ` + -PRNumber $env:PR_NUMBER ` + -Owner '${{ github.repository_owner }}' ` + -Repo '${{ github.event.repository.name }}' + if (-not $declineCleared) { + Write-Host "::warning::Could not clear s/agent-rerun-declined from PR #$($env:PR_NUMBER); continuing with the review." + } + $locked = Set-AgentReviewInProgress -PRNumber $env:PR_NUMBER -Owner '${{ github.repository_owner }}' -Repo '${{ github.event.repository.name }}' if (-not $locked) { throw "Failed to apply s/agent-review-in-progress to PR #$($env:PR_NUMBER)."