From 1d53a1459e6e0b7b1049d87b29e527903ae52b56 Mon Sep 17 00:00:00 2001 From: Yuhan Lei Date: Tue, 26 May 2026 10:34:08 +0800 Subject: [PATCH 1/4] fix: allow safe retry before provider progress --- .../src/session/run-incident/policy.ts | 90 ++++++++++++--- .../test/session/run-observability.test.ts | 106 +++++++++++++++++- 2 files changed, 180 insertions(+), 16 deletions(-) diff --git a/packages/opencode/src/session/run-incident/policy.ts b/packages/opencode/src/session/run-incident/policy.ts index 941f46b1b..92b87e5d8 100644 --- a/packages/opencode/src/session/run-incident/policy.ts +++ b/packages/opencode/src/session/run-incident/policy.ts @@ -9,15 +9,41 @@ export function recoveryFor(input: { const base = { safety_scope: "visible_output_and_tool_side_effects" as const } const terminalFacts = input.terminalFacts ?? input.facts const noToolActivity = - !terminalFacts.tool_input_started && - !terminalFacts.tool_call_materialized && - !terminalFacts.tool_execution_started + !terminalFacts.tool_input_started && !terminalFacts.tool_call_materialized && !terminalFacts.tool_execution_started const retryableTransport = input.retryable === true && (input.cause.category === "provider_transport_disconnect" || input.cause.category === "watchdog_timeout") + if (input.cause.category === "user_cancel") { + return { ...base, recommendation: "do_not_retry", confidence: "high", reason: "user_cancel" } + } + if (input.cause.category === "local_lifecycle_close") { + return { + ...base, + recommendation: "do_not_retry", + confidence: input.cause.confidence, + reason: "local_lifecycle_close", + } + } + if ( + canAutoRetryBeforeFirstProviderProgress({ + cause: input.cause, + facts: input.facts, + terminalFacts, + retryableTransport, + }) + ) { + return { + ...base, + recommendation: "auto_retry_once", + confidence: "high", + reason: "no_visible_output_or_tool_execution", + auto_retry: { max_attempts: 1, backoff_ms: 1_000 }, + } + } if ( retryableTransport && noToolActivity && + !isBeforeFirstProviderProgressCause(input.cause) && terminalFacts.reasoning_output_started && !terminalFacts.text_output_started && !terminalFacts.unsafe_side_effect_started && @@ -31,17 +57,6 @@ export function recoveryFor(input: { auto_retry: { max_attempts: 1, backoff_ms: 1_000 }, } } - if (input.cause.category === "user_cancel") { - return { ...base, recommendation: "do_not_retry", confidence: "high", reason: "user_cancel" } - } - if (input.cause.category === "local_lifecycle_close") { - return { - ...base, - recommendation: "do_not_retry", - confidence: input.cause.confidence, - reason: "local_lifecycle_close", - } - } if (!terminalFacts.side_effect_facts_complete) { return { ...base, @@ -132,6 +147,53 @@ export function recoveryFor(input: { return { ...base, recommendation: "unknown", confidence: "low", reason: "unknown" } } +function canAutoRetryBeforeFirstProviderProgress(input: { + cause: TerminalCause + facts: IncidentFacts + terminalFacts: IncidentFacts + retryableTransport: boolean +}) { + if (!input.retryableTransport) return false + if (input.facts.user_cancel_seen || input.facts.lifecycle_close_seen) return false + if (!isBeforeFirstProviderProgressCause(input.cause)) return false + if (input.terminalFacts.provider_progress_seen) return false + if (!attemptHasNoOutputOrToolActivity(input.terminalFacts)) return false + return boundaryAllowsBeforeProgressRetry(input.terminalFacts) +} + +function isBeforeFirstProviderProgressCause(cause: TerminalCause) { + if (cause.category === "provider_transport_disconnect") return cause.subcategory === "before_first_provider_progress" + if (cause.category === "watchdog_timeout") return cause.subcategory === "connect" + return false +} + +function attemptHasNoOutputOrToolActivity(facts: IncidentFacts) { + return ( + !facts.visible_output_seen && + !facts.text_output_started && + !facts.reasoning_output_started && + !facts.tool_input_started && + !facts.tool_input_completed && + !facts.tool_call_materialized && + !facts.tool_execution_started && + !facts.tool_execution_completed && + !facts.read_only_tool_started && + !facts.unsafe_side_effect_started && + (facts.pending_tool_parts_interrupted ?? 0) === 0 + ) +} + +function boundaryAllowsBeforeProgressRetry(facts: IncidentFacts) { + const snapshot = facts.side_effect_boundary_snapshot + if (!snapshot) return false + if (snapshot.provider_executed_capability_present !== false) return false + if (snapshot.external_boundary_present !== false) return false + if (snapshot.proof_reason === "provider_executed_capability" || snapshot.proof_reason === "external_boundary") { + return false + } + return true +} + function boundaryAllowsReasoningRetry(facts: IncidentFacts) { const snapshot = facts.side_effect_boundary_snapshot if (!snapshot) return false diff --git a/packages/opencode/test/session/run-observability.test.ts b/packages/opencode/test/session/run-observability.test.ts index 46e52f5c7..c709c8670 100644 --- a/packages/opencode/test/session/run-observability.test.ts +++ b/packages/opencode/test/session/run-observability.test.ts @@ -3,6 +3,48 @@ import { MessageID, SessionID } from "../../src/session/schema" import { RunIncident } from "../../src/session/run-incident" import { RunObservability } from "../../src/session/run-observability" +const beforeProgressCause = { + category: "provider_transport_disconnect", + subcategory: "before_first_provider_progress", + confidence: "medium", +} satisfies RunIncident.TerminalCause + +function beforeProgressFacts(overrides: Partial = {}): RunIncident.Facts { + return { + provider_progress_seen: false, + visible_output_seen: false, + text_output_started: false, + reasoning_output_started: false, + tool_input_started: false, + tool_input_completed: false, + tool_call_materialized: false, + tool_execution_started: false, + tool_execution_completed: false, + read_only_tool_started: false, + unsafe_side_effect_started: false, + unsafe_side_effect_kinds: [], + side_effect_facts_complete: false, + side_effect_boundary_snapshot: { + exposed_tool_count: 16, + unknown_tool_count: 10, + unclassified_effect_count: 10, + provider_executed_capability_present: false, + external_boundary_present: false, + proof_result: "incomplete", + proof_reason: "unknown_tool_boundary", + }, + lifecycle_close_seen: false, + user_cancel_seen: false, + watchdog_fired: false, + ...overrides, + } +} + +function recoveryForBeforeProgress(overrides: Partial = {}, retryable = true) { + const facts = beforeProgressFacts(overrides) + return RunIncident.recoveryFor({ cause: beforeProgressCause, facts, retryable }) +} + describe("RunObservability", () => { test("does not treat stream lifecycle events as provider progress", () => { expect(RunObservability.isProviderProgressEvent({ type: "start" })).toBe(false) @@ -951,7 +993,7 @@ describe("RunObservability", () => { expect(summary.incident?.evidence?.map((event) => event.event_type)).toContain("provider_executed_tool_boundary") }) - test("unknown request side-effect boundary prevents auto retry before local tool events", () => { + test("unknown exposed tools still allow retry before first provider progress when no tool activity occurred", () => { const recorder = RunObservability.createRecorder({ runID: RunObservability.RunID.make("run_unknown_request_boundary"), traceID: MessageID.make("msg_unknown_request_boundary"), @@ -988,6 +1030,7 @@ describe("RunObservability", () => { cause: { name: "SocketError", message: "other side closed", code: "UND_ERR_SOCKET" }, }, evidence: ["iterator_error"], + retryable: true, }) const summary = recorder.finalize({ completedAt: 14, monotonicMs: 140 }) @@ -1002,11 +1045,70 @@ describe("RunObservability", () => { proof_result: "incomplete", proof_reason: "unknown_tool_boundary", }) + expect(decision).toMatchObject({ + recommendation: "auto_retry_once", + reason: "no_visible_output_or_tool_execution", + }) + expect(summary.incident?.recovery).toMatchObject({ + recommendation: "auto_retry_once", + reason: "no_visible_output_or_tool_execution", + }) + }) + + test("before-progress retry stays conservative when the boundary snapshot is missing", () => { + const decision = recoveryForBeforeProgress({ side_effect_boundary_snapshot: undefined }) + expect(decision).toMatchObject({ recommendation: "ask_user_before_retry", reason: "side_effect_facts_incomplete", }) - expect(summary.incident?.recovery).toMatchObject({ + }) + + test("before-progress retry fails closed for output, tool activity, or external boundary evidence", () => { + const cases: Array<[string, Partial]> = [ + ["visible output", { visible_output_seen: true }], + ["text output", { text_output_started: true }], + ["reasoning output", { reasoning_output_started: true }], + ["tool input started", { tool_input_started: true }], + ["tool input completed", { tool_input_completed: true }], + ["tool call materialized", { tool_call_materialized: true }], + ["tool execution started", { tool_execution_started: true }], + ["tool execution completed", { tool_execution_completed: true }], + ["read-only tool started", { read_only_tool_started: true }], + ["unsafe side effect", { unsafe_side_effect_started: true }], + ["pending tool parts", { pending_tool_parts_interrupted: 1 }], + [ + "provider-executed capability", + { + side_effect_boundary_snapshot: { + ...beforeProgressFacts().side_effect_boundary_snapshot!, + provider_executed_capability_present: true, + proof_reason: "provider_executed_capability", + }, + }, + ], + [ + "external boundary", + { + side_effect_boundary_snapshot: { + ...beforeProgressFacts().side_effect_boundary_snapshot!, + external_boundary_present: true, + proof_reason: "external_boundary", + }, + }, + ], + ] + + for (const [name, overrides] of cases) { + const decision = recoveryForBeforeProgress(overrides) + expect(decision, name).not.toMatchObject({ recommendation: "auto_retry_once" }) + } + }) + + test("before-progress retry is denied when the transport error is not retryable", () => { + const decision = recoveryForBeforeProgress({}, false) + + expect(decision).toMatchObject({ recommendation: "ask_user_before_retry", reason: "side_effect_facts_incomplete", }) From adfd11da55eb3734a731b3b4a1bf4d5f0c32edf0 Mon Sep 17 00:00:00 2001 From: Yuhan Lei Date: Tue, 26 May 2026 10:52:03 +0800 Subject: [PATCH 2/4] test: cover before-progress boundary retry denial --- .../src/session/run-incident/policy.ts | 16 ++++++++++++++ .../test/session/run-observability.test.ts | 22 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/packages/opencode/src/session/run-incident/policy.ts b/packages/opencode/src/session/run-incident/policy.ts index 92b87e5d8..05a411cb8 100644 --- a/packages/opencode/src/session/run-incident/policy.ts +++ b/packages/opencode/src/session/run-incident/policy.ts @@ -40,6 +40,14 @@ export function recoveryFor(input: { auto_retry: { max_attempts: 1, backoff_ms: 1_000 }, } } + if (isBeforeFirstProviderProgressCause(input.cause) && beforeProgressBoundaryEvidenceBlocksRetry(terminalFacts)) { + return { + ...base, + recommendation: "ask_user_before_retry", + confidence: "high", + reason: "side_effect_facts_incomplete", + } + } if ( retryableTransport && noToolActivity && @@ -194,6 +202,14 @@ function boundaryAllowsBeforeProgressRetry(facts: IncidentFacts) { return true } +function beforeProgressBoundaryEvidenceBlocksRetry(facts: IncidentFacts) { + const snapshot = facts.side_effect_boundary_snapshot + if (!snapshot) return false + if (snapshot.provider_executed_capability_present !== false) return true + if (snapshot.external_boundary_present !== false) return true + return snapshot.proof_reason === "provider_executed_capability" || snapshot.proof_reason === "external_boundary" +} + function boundaryAllowsReasoningRetry(facts: IncidentFacts) { const snapshot = facts.side_effect_boundary_snapshot if (!snapshot) return false diff --git a/packages/opencode/test/session/run-observability.test.ts b/packages/opencode/test/session/run-observability.test.ts index c709c8670..73566d9fe 100644 --- a/packages/opencode/test/session/run-observability.test.ts +++ b/packages/opencode/test/session/run-observability.test.ts @@ -1087,6 +1087,17 @@ describe("RunObservability", () => { }, }, ], + [ + "provider-executed capability with complete facts", + { + side_effect_facts_complete: true, + side_effect_boundary_snapshot: { + ...beforeProgressFacts().side_effect_boundary_snapshot!, + provider_executed_capability_present: true, + proof_reason: "provider_executed_capability", + }, + }, + ], [ "external boundary", { @@ -1097,6 +1108,17 @@ describe("RunObservability", () => { }, }, ], + [ + "external boundary with complete facts", + { + side_effect_facts_complete: true, + side_effect_boundary_snapshot: { + ...beforeProgressFacts().side_effect_boundary_snapshot!, + external_boundary_present: true, + proof_reason: "external_boundary", + }, + }, + ], ] for (const [name, overrides] of cases) { From b2d9920dd6a50a395ec7b173349c51c29ba9ab13 Mon Sep 17 00:00:00 2001 From: Yuhan Lei Date: Tue, 26 May 2026 11:19:56 +0800 Subject: [PATCH 3/4] fix: reject unknown before-progress boundary proof --- packages/opencode/src/session/run-incident/policy.ts | 12 ++++++++++-- .../opencode/test/session/run-observability.test.ts | 10 ++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/packages/opencode/src/session/run-incident/policy.ts b/packages/opencode/src/session/run-incident/policy.ts index 05a411cb8..c60a665c3 100644 --- a/packages/opencode/src/session/run-incident/policy.ts +++ b/packages/opencode/src/session/run-incident/policy.ts @@ -196,7 +196,11 @@ function boundaryAllowsBeforeProgressRetry(facts: IncidentFacts) { if (!snapshot) return false if (snapshot.provider_executed_capability_present !== false) return false if (snapshot.external_boundary_present !== false) return false - if (snapshot.proof_reason === "provider_executed_capability" || snapshot.proof_reason === "external_boundary") { + if ( + snapshot.proof_reason === "provider_executed_capability" || + snapshot.proof_reason === "external_boundary" || + snapshot.proof_reason === "unknown" + ) { return false } return true @@ -207,7 +211,11 @@ function beforeProgressBoundaryEvidenceBlocksRetry(facts: IncidentFacts) { if (!snapshot) return false if (snapshot.provider_executed_capability_present !== false) return true if (snapshot.external_boundary_present !== false) return true - return snapshot.proof_reason === "provider_executed_capability" || snapshot.proof_reason === "external_boundary" + return ( + snapshot.proof_reason === "provider_executed_capability" || + snapshot.proof_reason === "external_boundary" || + snapshot.proof_reason === "unknown" + ) } function boundaryAllowsReasoningRetry(facts: IncidentFacts) { diff --git a/packages/opencode/test/session/run-observability.test.ts b/packages/opencode/test/session/run-observability.test.ts index 73566d9fe..cc6630728 100644 --- a/packages/opencode/test/session/run-observability.test.ts +++ b/packages/opencode/test/session/run-observability.test.ts @@ -1119,6 +1119,16 @@ describe("RunObservability", () => { }, }, ], + [ + "unknown boundary proof with complete facts", + { + side_effect_facts_complete: true, + side_effect_boundary_snapshot: { + ...beforeProgressFacts().side_effect_boundary_snapshot!, + proof_reason: "unknown", + }, + }, + ], ] for (const [name, overrides] of cases) { From 5e43a7d5e4200490e148c04ce6b3b9bf40a16d30 Mon Sep 17 00:00:00 2001 From: Yuhan Lei Date: Tue, 26 May 2026 11:54:46 +0800 Subject: [PATCH 4/4] fix: require boundary snapshot before retry fallback --- .../src/session/run-incident/policy.ts | 2 +- .../test/session/run-observability.test.ts | 48 +++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/packages/opencode/src/session/run-incident/policy.ts b/packages/opencode/src/session/run-incident/policy.ts index c60a665c3..9c387f00a 100644 --- a/packages/opencode/src/session/run-incident/policy.ts +++ b/packages/opencode/src/session/run-incident/policy.ts @@ -208,7 +208,7 @@ function boundaryAllowsBeforeProgressRetry(facts: IncidentFacts) { function beforeProgressBoundaryEvidenceBlocksRetry(facts: IncidentFacts) { const snapshot = facts.side_effect_boundary_snapshot - if (!snapshot) return false + if (!snapshot) return true if (snapshot.provider_executed_capability_present !== false) return true if (snapshot.external_boundary_present !== false) return true return ( diff --git a/packages/opencode/test/session/run-observability.test.ts b/packages/opencode/test/session/run-observability.test.ts index cc6630728..aa55864e5 100644 --- a/packages/opencode/test/session/run-observability.test.ts +++ b/packages/opencode/test/session/run-observability.test.ts @@ -45,6 +45,18 @@ function recoveryForBeforeProgress(overrides: Partial = {}, r return RunIncident.recoveryFor({ cause: beforeProgressCause, facts, retryable }) } +function noToolBoundarySnapshot(): RunObservability.SideEffectBoundarySnapshot { + return { + exposed_tool_count: 0, + unknown_tool_count: 0, + unclassified_effect_count: 0, + provider_executed_capability_present: false, + external_boundary_present: false, + proof_result: "complete", + proof_reason: "all_boundaries_classified", + } +} + describe("RunObservability", () => { test("does not treat stream lifecycle events as provider progress", () => { expect(RunObservability.isProviderProgressEvent({ type: "start" })).toBe(false) @@ -100,6 +112,12 @@ describe("RunObservability", () => { }) const attempt = recorder.beginAttempt({ attemptIndex: 1, at: 11, monotonicMs: 110 }) + recorder.recordSideEffectBoundarySnapshot({ + attemptID: attempt.attemptID, + at: 12, + monotonicMs: 120, + snapshot: noToolBoundarySnapshot(), + }) const decision = recorder.recordAttemptFailureAndDeriveRecovery({ attemptID: attempt.attemptID, at: 130, @@ -899,6 +917,12 @@ describe("RunObservability", () => { monotonicStartMs: 200, }) const currentAttempt = current.beginAttempt({ attemptIndex: 1, at: 21, monotonicMs: 210 }) + current.recordSideEffectBoundarySnapshot({ + attemptID: currentAttempt.attemptID, + at: 21, + monotonicMs: 211, + snapshot: noToolBoundarySnapshot(), + }) const decision = current.recordAttemptFailureAndDeriveRecovery({ attemptID: currentAttempt.attemptID, at: 22, @@ -1064,6 +1088,18 @@ describe("RunObservability", () => { }) }) + test("before-progress retry stays conservative when complete facts lack a boundary snapshot", () => { + const decision = recoveryForBeforeProgress({ + side_effect_facts_complete: true, + side_effect_boundary_snapshot: undefined, + }) + + expect(decision).toMatchObject({ + recommendation: "ask_user_before_retry", + reason: "side_effect_facts_incomplete", + }) + }) + test("before-progress retry fails closed for output, tool activity, or external boundary evidence", () => { const cases: Array<[string, Partial]> = [ ["visible output", { visible_output_seen: true }], @@ -1987,6 +2023,12 @@ describe("RunObservability", () => { effect: RunObservability.toolEffect("bash"), }) const second = recorder.beginAttempt({ attemptIndex: 2, at: 20, monotonicMs: 200 }) + recorder.recordSideEffectBoundarySnapshot({ + attemptID: second.attemptID, + at: 20, + monotonicMs: 201, + snapshot: noToolBoundarySnapshot(), + }) recorder.recordTransportFailure({ attemptID: second.attemptID, at: 21, @@ -2026,6 +2068,12 @@ describe("RunObservability", () => { effect: RunObservability.toolEffect("read"), }) const second = recorder.beginAttempt({ attemptIndex: 2, at: 20, monotonicMs: 200 }) + recorder.recordSideEffectBoundarySnapshot({ + attemptID: second.attemptID, + at: 20, + monotonicMs: 201, + snapshot: noToolBoundarySnapshot(), + }) recorder.recordTransportFailure({ attemptID: second.attemptID, at: 21,