Repository navigation
fix(#1559): preserve prior successful review on agent failure #1560
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 1 commit
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -218,6 +218,9 @@ The review agent reviewed commit `+"`%s`"+` but the PR HEAD is now `+"`%s`"+`. T | |
| } | ||
|
|
||
| // postFailureNotice posts a failure comment as a sticky comment. | ||
| // When a prior successful review exists, the failure is shown as a | ||
| // warning banner while preserving the previous review findings | ||
| // prominently (not collapsed into history). | ||
| func postFailureNotice(ctx context.Context, client forge.Client, owner, repo string, pr int, parsed ReviewResult, cfg sticky.Config, printer *ui.Printer) error { | ||
| printer.StepStart("Review agent reported failure, posting notice") | ||
|
|
||
|
|
@@ -239,13 +242,36 @@ func postFailureNotice(ctx context.Context, client forge.Client, owner, repo str | |
| This PR was NOT reviewed. Do not count this as an approval.`, reason) | ||
| } | ||
|
|
||
| // When a prior successful review exists, preserve it prominently | ||
| // and show the failure as a warning banner instead of replacing the | ||
| // review findings with "NOT reviewed". See #1559. | ||
| botUser, _ := client.GetAuthenticatedUser(ctx) | ||
| comments, listErr := client.ListIssueComments(ctx, owner, repo, pr) | ||
| if listErr == nil { | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [low] redundant-api-calls postFailureNotice now calls GetAuthenticatedUser and ListIssueComments to check for prior successful reviews, then unconditionally calls sticky.Post() which makes the same two API calls internally (sticky.go lines 43-53). This doubles the API round-trips on every failure path. Suggested fix: Consider extracting the comment lookup into a shared helper or passing the found comment/bot-user to sticky.Post to avoid redundant calls. |
||
| if existing := sticky.FindMarkedComment(comments, cfg.Marker, botUser); existing != nil { | ||
| priorContent := sticky.ExtractCurrentContent(existing.Body, cfg) | ||
| if priorContent != "" && !isFailureOrStaleContent(priorContent) { | ||
| body = fmt.Sprintf("## Review\n\n> **⚠️ Latest review run failed** (%s). Showing results from previous successful review.\n\n%s", reason, priorContent) | ||
| printer.StepInfo("Preserving prior successful review") | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [high] correctness isFailureOrStaleContent does not match the format string produced by postFailureNotice. The format string wraps the emoji inside bold markers ( Suggested fix: Align the search string with the format string — either change the format to place the emoji outside bold, or change the search to |
||
| } | ||
| } | ||
| } | ||
|
|
||
| if _, err := sticky.Post(ctx, client, owner, repo, pr, body, cfg, printer); err != nil { | ||
| return fmt.Errorf("posting failure notice: %w", err) | ||
| } | ||
| printer.StepDone("Failure notice posted") | ||
| return nil | ||
| } | ||
|
|
||
| // isFailureOrStaleContent returns true if the content looks like a | ||
| // failure notice or stale-head notice rather than a successful review. | ||
| func isFailureOrStaleContent(content string) bool { | ||
| return strings.Contains(content, "This PR was NOT reviewed") || | ||
| strings.Contains(content, "**Reason:** stale-head") || | ||
| strings.Contains(content, "⚠️ **Latest review run failed**") | ||
| } | ||
|
|
||
| // submitFormalReview minimizes stale reviews by the same user, then | ||
| // submits a new GitHub PR review. When commitSHA is non-empty, the | ||
| // review is pinned to that commit via the commit_id field, closing | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -190,6 +190,122 @@ func TestPostFailureNotice_EmptyReason(t *testing.T) { | |
| assert.Contains(t, comments[0].Body, "NOT reviewed") | ||
| } | ||
|
|
||
| func TestPostFailureNotice_PreservesPriorSuccessfulReview(t *testing.T) { | ||
| fc := forge.NewFakeClient() | ||
| fc.AuthenticatedUser = "bot" | ||
| // Seed an existing sticky comment with a successful review. | ||
| fc.IssueComments = map[string][]forge.IssueComment{ | ||
| "o/r/1": {{ | ||
| ID: 100, | ||
| HTMLURL: "https://github.com/o/r/issues/1#issuecomment-100", | ||
| Body: "<!-- test -->\n## Review\n\n**Verdict:** Approved\n\nLooks good, no issues found.", | ||
| Author: "bot", | ||
| }}, | ||
| } | ||
| printer := ui.New(io.Discard) | ||
|
|
||
| cfg := sticky.Config{Marker: "<!-- test -->"} | ||
| parsed := ReviewResult{Action: "failure", Reason: "tool-failure"} | ||
| err := postFailureNotice(context.Background(), fc, "o", "r", 1, parsed, cfg, printer) | ||
| require.NoError(t, err) | ||
|
|
||
| require.Len(t, fc.UpdatedComments, 1) | ||
| updated := fc.UpdatedComments[0].Body | ||
| // The prior review findings should be visible (not just in collapsed history). | ||
| assert.Contains(t, updated, "Looks good, no issues found.") | ||
| // A warning banner should be present. | ||
| assert.Contains(t, updated, "Latest review run failed") | ||
| assert.Contains(t, updated, "tool-failure") | ||
| // Should NOT contain the blanket "NOT reviewed" message at top level. | ||
| // The top-level content should show the preserved review. | ||
| topLevel := sticky.ExtractCurrentContent(updated, cfg) | ||
| assert.NotContains(t, topLevel, "NOT reviewed") | ||
| } | ||
|
|
||
| func TestPostFailureNotice_NoPriorReview(t *testing.T) { | ||
| fc := forge.NewFakeClient() | ||
| fc.AuthenticatedUser = "bot" | ||
| printer := ui.New(io.Discard) | ||
|
|
||
| cfg := sticky.Config{Marker: "<!-- test -->"} | ||
| parsed := ReviewResult{Action: "failure", Reason: "tool-failure"} | ||
| err := postFailureNotice(context.Background(), fc, "o", "r", 1, parsed, cfg, printer) | ||
| require.NoError(t, err) | ||
|
|
||
| comments := fc.IssueComments["o/r/1"] | ||
| require.Len(t, comments, 1) | ||
| assert.Contains(t, comments[0].Body, "NOT reviewed") | ||
| assert.Contains(t, comments[0].Body, "tool-failure") | ||
| } | ||
|
|
||
| func TestPostFailureNotice_PriorFailureNotPreserved(t *testing.T) { | ||
| fc := forge.NewFakeClient() | ||
| fc.AuthenticatedUser = "bot" | ||
| // Seed an existing sticky comment that is itself a failure notice. | ||
| fc.IssueComments = map[string][]forge.IssueComment{ | ||
| "o/r/1": {{ | ||
| ID: 100, | ||
| Body: "<!-- test -->\n## Review\n\n**Reason:** token-limit\n\nThis PR was NOT reviewed. Do not count this as an approval.", | ||
| Author: "bot", | ||
| }}, | ||
| } | ||
| printer := ui.New(io.Discard) | ||
|
|
||
| cfg := sticky.Config{Marker: "<!-- test -->"} | ||
| parsed := ReviewResult{Action: "failure", Reason: "tool-failure"} | ||
| err := postFailureNotice(context.Background(), fc, "o", "r", 1, parsed, cfg, printer) | ||
| require.NoError(t, err) | ||
|
|
||
| require.Len(t, fc.UpdatedComments, 1) | ||
| updated := fc.UpdatedComments[0].Body | ||
| topLevel := sticky.ExtractCurrentContent(updated, cfg) | ||
| // When the prior content was also a failure, the new failure should | ||
| // replace it (current behavior, not preserved). | ||
| assert.Contains(t, topLevel, "NOT reviewed") | ||
| assert.Contains(t, topLevel, "tool-failure") | ||
| } | ||
|
|
||
| func TestPostFailureNotice_PriorStaleHeadNotPreserved(t *testing.T) { | ||
| fc := forge.NewFakeClient() | ||
| fc.AuthenticatedUser = "bot" | ||
| fc.IssueComments = map[string][]forge.IssueComment{ | ||
| "o/r/1": {{ | ||
| ID: 100, | ||
| Body: "<!-- test -->\n## Review\n\n**Reason:** stale-head\n\nThe review agent reviewed commit `abc` but the PR HEAD is now `def`.", | ||
| Author: "bot", | ||
| }}, | ||
| } | ||
| printer := ui.New(io.Discard) | ||
|
|
||
| cfg := sticky.Config{Marker: "<!-- test -->"} | ||
| parsed := ReviewResult{Action: "failure", Reason: "tool-failure"} | ||
| err := postFailureNotice(context.Background(), fc, "o", "r", 1, parsed, cfg, printer) | ||
| require.NoError(t, err) | ||
|
|
||
| require.Len(t, fc.UpdatedComments, 1) | ||
| topLevel := sticky.ExtractCurrentContent(fc.UpdatedComments[0].Body, cfg) | ||
| assert.Contains(t, topLevel, "NOT reviewed") | ||
| } | ||
|
|
||
| func TestIsFailureOrStaleContent(t *testing.T) { | ||
| tests := []struct { | ||
| name string | ||
| content string | ||
| want bool | ||
| }{ | ||
| {"successful review", "## Review\n\n**Verdict:** Approved", false}, | ||
| {"failure notice", "## Review\n\nThis PR was NOT reviewed.", true}, | ||
| {"stale-head", "## Review\n\n**Reason:** stale-head\n\nDiscarded.", true}, | ||
| {"prior preserved failure", "## Review\n\n> ⚠️ **Latest review run failed** (x).", true}, | ||
| {"empty", "", false}, | ||
| } | ||
| for _, tt := range tests { | ||
| t.Run(tt.name, func(t *testing.T) { | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [medium] correctness The "prior preserved failure" test case uses input with the emoji outside bold markers, which matches the search pattern but not the actual format string from postFailureNotice. The test passes but validates the wrong input format. Suggested fix: Change the test input to match the actual format string. The test should then fail, confirming the bug, and both should be fixed together. |
||
| assert.Equal(t, tt.want, isFailureOrStaleContent(tt.content)) | ||
| }) | ||
| } | ||
| } | ||
|
|
||
| func TestCheckStaleHead_CaseInsensitive(t *testing.T) { | ||
| fc := forge.NewFakeClient() | ||
| fc.PullRequestHeadSHA = "abc1234567890abcdef1234567890abcdef123456" | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
[low] style
GetAuthenticatedUser error silently discarded without logging, inconsistent with sticky.Post which logs a warning for the same failure.
Suggested fix: Log a warning via printer.StepInfo when the error is non-nil.