From deacf0055a00f302b88ac64141edcd27eb470d01 Mon Sep 17 00:00:00 2001 From: fullsend-code <278716306+fullsend-ai-coder[bot]@users.noreply.github.com> Date: Wed, 1 Jul 2026 18:06:03 +0000 Subject: [PATCH 1/2] fix(#2786): surface agent API errors when Claude Code exits 0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude Code exits with code 0 on API/infrastructure failures (e.g., invalid_grant, quota exhaustion) while setting is_error:true in the transcript result event. This made infrastructure errors invisible — green workflows with no PR, no comment, and no GHA error annotation. Two changes in run.go: 1. After exit code 0, parse the tee'd output.jsonl for is_error:true result events. If found, override lastExitCode to 1 and emit StepWarn so downstream gating treats the run as failed. 2. Remove the lastExitCode != 0 gate around ParseTranscriptErrors so ::error:: annotations are always emitted, regardless of exit code. Added ParseTranscriptFile to the TranscriptHandler interface to support checking a single JSONL file (output.jsonl) without scanning an entire directory. Note: make lint could not run (shellcheck-py fails to download in sandbox). go vet and go test both pass. Closes #2786 --- internal/cli/run.go | 39 ++++++++---- internal/runtime/claude.go | 4 ++ internal/runtime/claude_transcript_test.go | 74 ++++++++++++++++++++++ internal/runtime/transcript.go | 4 ++ 4 files changed, 109 insertions(+), 12 deletions(-) diff --git a/internal/cli/run.go b/internal/cli/run.go index 8316761095..ab37a777a4 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -1008,12 +1008,28 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep } lastExitCode = exitCode + // Check the tee'd output.jsonl for is_error:true result events. + // Claude Code may exit 0 on API/infrastructure failures (e.g., + // invalid_grant, quota exhaustion) while setting is_error:true in + // the transcript. Treat these as failures so downstream gating + // (transcript surfacing, post-script behavior) can act. See #2786. + if exitCode == 0 { + outputJSONL := filepath.Join(iterDir, "output.jsonl") + if te, ok := tx.ParseTranscriptFile(outputJSONL); ok && te.IsError { + printer.StepWarn(fmt.Sprintf("Agent exited with code 0 but transcript contains error: %s", te.ErrorMessage)) + lastExitCode = 1 + } + } + printer.Blank() // Non-zero exit is a warning, not a failure — the validation loop is the success gate. - if exitCode == 0 { + if lastExitCode == 0 { printer.StepDone(fmt.Sprintf("Agent exited with code %d (%.1fs)", exitCode, time.Since(agentStart).Seconds())) - } else { + } else if exitCode != 0 { printer.StepWarn(fmt.Sprintf("Agent exited with code %d", exitCode)) + } else { + // exitCode was 0 but lastExitCode was overridden due to transcript error. + printer.StepWarn(fmt.Sprintf("Agent exited with code %d (transcript error detected)", exitCode)) } // 9b. Extract output files. @@ -1103,16 +1119,15 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep rec.SetModel(aggMetrics.Model) // 9e-bis. Surface transcript errors in workflow logs (GitHub Actions). - // When the agent exits non-zero, parse transcript JSONL files and emit - // ::error:: annotations so operators can diagnose failures without - // downloading artifacts. See #704. - if lastExitCode != 0 { - lastIterDir := filepath.Join(runDir, fmt.Sprintf("iteration-%d", runCount)) - lastTranscriptDir := filepath.Join(lastIterDir, "transcripts") - if errorSummaries := tx.ParseTranscriptErrors(lastTranscriptDir); len(errorSummaries) > 0 { - printer.StepWarn(fmt.Sprintf("Found %d transcript error(s) — emitting to workflow log", len(errorSummaries))) - tx.EmitTranscriptErrors(os.Stderr, errorSummaries) - } + // Parse transcript JSONL files and emit ::error:: annotations so operators + // can diagnose failures without downloading artifacts. This runs + // regardless of exit code because Claude Code may exit 0 with + // is_error:true on API/infrastructure failures. See #704, #2786. + lastIterDir := filepath.Join(runDir, fmt.Sprintf("iteration-%d", runCount)) + lastTranscriptDir := filepath.Join(lastIterDir, "transcripts") + if errorSummaries := tx.ParseTranscriptErrors(lastTranscriptDir); len(errorSummaries) > 0 { + printer.StepWarn(fmt.Sprintf("Found %d transcript error(s) — emitting to workflow log", len(errorSummaries))) + tx.EmitTranscriptErrors(os.Stderr, errorSummaries) } // 9f. Post-agent output scan — redact secrets from extracted output. diff --git a/internal/runtime/claude.go b/internal/runtime/claude.go index 0c83d51f17..156c46edfa 100644 --- a/internal/runtime/claude.go +++ b/internal/runtime/claude.go @@ -194,6 +194,10 @@ func (ClaudeRuntime) ParseTranscriptErrors(transcriptDir string) []TranscriptErr return parseTranscriptErrors(transcriptDir) } +func (ClaudeRuntime) ParseTranscriptFile(path string) (TranscriptError, bool) { + return parseTranscriptFile(path) +} + func (ClaudeRuntime) EmitTranscriptErrors(w io.Writer, summaries []TranscriptError) { emitTranscriptErrors(w, summaries) } diff --git a/internal/runtime/claude_transcript_test.go b/internal/runtime/claude_transcript_test.go index 2be4bdd3f5..6c3d772f33 100644 --- a/internal/runtime/claude_transcript_test.go +++ b/internal/runtime/claude_transcript_test.go @@ -246,6 +246,80 @@ func TestEmitTranscriptErrors_NoSummaries(t *testing.T) { } } +// TestParseTranscriptFile_APIErrorExitZero covers the scenario from #2786: +// Claude Code exits 0 with is_error:true and subtype "success" on API errors +// (e.g., invalid_grant from a stale OIDC token). +func TestParseTranscriptFile_APIErrorExitZero(t *testing.T) { + dir := t.TempDir() + // Real-world transcript shape: subtype is "success" but is_error is true. + content := `{"type":"system","subtype":"init","session_id":"abc123"} +{"type":"result","subtype":"success","is_error":true,"result":"API Error: Error code invalid_grant: ID Token issued at 1782810237 is stale to sign-in.","session_id":"abc123"} +` + path := filepath.Join(dir, "output.jsonl") + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + + summary, ok := parseTranscriptFile(path) + if !ok { + t.Fatal("expected result event to be found") + } + if !summary.IsError { + t.Error("expected IsError to be true for API error with exit 0") + } + if summary.Subtype != "success" { + t.Errorf("expected subtype 'success', got %q", summary.Subtype) + } + if !strings.Contains(summary.ErrorMessage, "invalid_grant") { + t.Errorf("expected error message to contain 'invalid_grant', got %q", summary.ErrorMessage) + } +} + +// TestParseTranscriptErrors_SurfacesErrorRegardlessOfExitCode verifies that +// parseTranscriptErrors returns errors from transcripts where is_error:true, +// which is the key fix from #2786 — errors must be surfaced even when the +// process exit code was 0. +func TestParseTranscriptErrors_SurfacesErrorRegardlessOfExitCode(t *testing.T) { + dir := t.TempDir() + + // Transcript with is_error:true but subtype "success" (API error scenario). + content := `{"type":"result","subtype":"success","is_error":true,"result":"API Error: quota exhausted"}` + if err := os.WriteFile(filepath.Join(dir, "agent.jsonl"), []byte(content), 0o644); err != nil { + t.Fatal(err) + } + + summaries := parseTranscriptErrors(dir) + if len(summaries) != 1 { + t.Fatalf("expected 1 error summary, got %d", len(summaries)) + } + if !summaries[0].IsError { + t.Error("expected IsError to be true") + } + if !strings.Contains(summaries[0].ErrorMessage, "quota exhausted") { + t.Errorf("unexpected error message: %q", summaries[0].ErrorMessage) + } +} + +// TestClaudeRuntime_ParseTranscriptFile verifies the exported method on +// ClaudeRuntime satisfies the TranscriptHandler interface. +func TestClaudeRuntime_ParseTranscriptFile(t *testing.T) { + dir := t.TempDir() + content := `{"type":"result","subtype":"success","is_error":true,"result":"infrastructure error"}` + path := filepath.Join(dir, "output.jsonl") + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + + var handler TranscriptHandler = ClaudeRuntime{} + summary, ok := handler.ParseTranscriptFile(path) + if !ok { + t.Fatal("expected result event to be found") + } + if !summary.IsError { + t.Error("expected IsError to be true") + } +} + func TestIsResultLine(t *testing.T) { tests := []struct { line string diff --git a/internal/runtime/transcript.go b/internal/runtime/transcript.go index ef4bcdec6d..cd8e4ce0e4 100644 --- a/internal/runtime/transcript.go +++ b/internal/runtime/transcript.go @@ -9,5 +9,9 @@ type TranscriptHandler interface { ExtractTranscripts(sandboxName, agentLabel, outputDir string) error ExtractDebugLog(sandboxName, localPath, debug string) error ParseTranscriptErrors(transcriptDir string) []TranscriptError + // ParseTranscriptFile parses a single JSONL transcript or output file + // and returns the last result event, if any. Use this to check a tee'd + // output.jsonl for is_error:true without scanning an entire directory. + ParseTranscriptFile(path string) (TranscriptError, bool) EmitTranscriptErrors(w io.Writer, summaries []TranscriptError) } From 5f1462de4478a49fabbc65361ffac12c39e6cd37 Mon Sep 17 00:00:00 2001 From: Wayne Sun Date: Wed, 1 Jul 2026 16:34:24 -0400 Subject: [PATCH 2/2] fix(#2786): skip post-script on transcript errors, clean up StepWarn - Add transcriptErrorOverride flag to skip the post-script when Claude Code exits 0 with is_error:true in the transcript. The flag is scoped to transcript-error overrides only (not all non-zero exits) and reset each iteration. - Simplify the three-way StepWarn branch to two-way, printing lastExitCode (the effective value) instead of exitCode. Assisted-by: Claude (fix), Gemini (review), Codex (review) Signed-off-by: Wayne Sun --- internal/cli/run.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/internal/cli/run.go b/internal/cli/run.go index ab37a777a4..8df2b401e4 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -619,6 +619,7 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep // registered before the post-script and cleanup defers so that — by LIFO // order — it runs last and the summary captures the whole run. var lastExitCode int + var transcriptErrorOverride bool rec := telemetry.New(runDir, wTraceID, rootSpanID, agentName, workItemID, runStart) defer func() { rec.Finalize(telemetryExitCode(lastExitCode, runErr)) }() @@ -660,6 +661,10 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep printer.StepWarn("Skipping post-script: agent run failed") return } + if transcriptErrorOverride { + printer.StepWarn("Skipping post-script: agent reported error via transcript") + return + } postStart := time.Now() printer.StepStart("Running post-script: " + h.PostScript) postCmd := exec.Command(h.PostScript) @@ -939,6 +944,7 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep for iteration := 1; iteration <= maxIterations; iteration++ { runCount = iteration + transcriptErrorOverride = false // Each iteration gets its own subdirectory for output and transcripts. iterDir := filepath.Join(runDir, fmt.Sprintf("iteration-%d", iteration)) @@ -1012,12 +1018,13 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep // Claude Code may exit 0 on API/infrastructure failures (e.g., // invalid_grant, quota exhaustion) while setting is_error:true in // the transcript. Treat these as failures so downstream gating - // (transcript surfacing, post-script behavior) can act. See #2786. + // (transcript surfacing, post-script skip) can act. See #2786. if exitCode == 0 { outputJSONL := filepath.Join(iterDir, "output.jsonl") if te, ok := tx.ParseTranscriptFile(outputJSONL); ok && te.IsError { printer.StepWarn(fmt.Sprintf("Agent exited with code 0 but transcript contains error: %s", te.ErrorMessage)) lastExitCode = 1 + transcriptErrorOverride = true } } @@ -1025,11 +1032,8 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep // Non-zero exit is a warning, not a failure — the validation loop is the success gate. if lastExitCode == 0 { printer.StepDone(fmt.Sprintf("Agent exited with code %d (%.1fs)", exitCode, time.Since(agentStart).Seconds())) - } else if exitCode != 0 { - printer.StepWarn(fmt.Sprintf("Agent exited with code %d", exitCode)) } else { - // exitCode was 0 but lastExitCode was overridden due to transcript error. - printer.StepWarn(fmt.Sprintf("Agent exited with code %d (transcript error detected)", exitCode)) + printer.StepWarn(fmt.Sprintf("Agent exited with code %d", lastExitCode)) } // 9b. Extract output files.