diff --git a/packages/opencode/src/session/run-incident/policy.ts b/packages/opencode/src/session/run-incident/policy.ts index 941f46b1b..9c387f00a 100644 --- a/packages/opencode/src/session/run-incident/policy.ts +++ b/packages/opencode/src/session/run-incident/policy.ts @@ -9,15 +9,49 @@ export function recoveryFor(input: { const base = { safety_scope: "visible_output_and_tool_side_effects" as const } const terminalFacts = input.terminalFacts ?? input.facts const noToolActivity = - !terminalFacts.tool_input_started && - !terminalFacts.tool_call_materialized && - !terminalFacts.tool_execution_started + !terminalFacts.tool_input_started && !terminalFacts.tool_call_materialized && !terminalFacts.tool_execution_started const retryableTransport = input.retryable === true && (input.cause.category === "provider_transport_disconnect" || input.cause.category === "watchdog_timeout") + if (input.cause.category === "user_cancel") { + return { ...base, recommendation: "do_not_retry", confidence: "high", reason: "user_cancel" } + } + if (input.cause.category === "local_lifecycle_close") { + return { + ...base, + recommendation: "do_not_retry", + confidence: input.cause.confidence, + reason: "local_lifecycle_close", + } + } + if ( + canAutoRetryBeforeFirstProviderProgress({ + cause: input.cause, + facts: input.facts, + terminalFacts, + retryableTransport, + }) + ) { + return { + ...base, + recommendation: "auto_retry_once", + confidence: "high", + reason: "no_visible_output_or_tool_execution", + auto_retry: { max_attempts: 1, backoff_ms: 1_000 }, + } + } + if (isBeforeFirstProviderProgressCause(input.cause) && beforeProgressBoundaryEvidenceBlocksRetry(terminalFacts)) { + return { + ...base, + recommendation: "ask_user_before_retry", + confidence: "high", + reason: "side_effect_facts_incomplete", + } + } if ( retryableTransport && noToolActivity && + !isBeforeFirstProviderProgressCause(input.cause) && terminalFacts.reasoning_output_started && !terminalFacts.text_output_started && !terminalFacts.unsafe_side_effect_started && @@ -31,17 +65,6 @@ export function recoveryFor(input: { auto_retry: { max_attempts: 1, backoff_ms: 1_000 }, } } - if (input.cause.category === "user_cancel") { - return { ...base, recommendation: "do_not_retry", confidence: "high", reason: "user_cancel" } - } - if (input.cause.category === "local_lifecycle_close") { - return { - ...base, - recommendation: "do_not_retry", - confidence: input.cause.confidence, - reason: "local_lifecycle_close", - } - } if (!terminalFacts.side_effect_facts_complete) { return { ...base, @@ -132,6 +155,69 @@ export function recoveryFor(input: { return { ...base, recommendation: "unknown", confidence: "low", reason: "unknown" } } +function canAutoRetryBeforeFirstProviderProgress(input: { + cause: TerminalCause + facts: IncidentFacts + terminalFacts: IncidentFacts + retryableTransport: boolean +}) { + if (!input.retryableTransport) return false + if (input.facts.user_cancel_seen || input.facts.lifecycle_close_seen) return false + if (!isBeforeFirstProviderProgressCause(input.cause)) return false + if (input.terminalFacts.provider_progress_seen) return false + if (!attemptHasNoOutputOrToolActivity(input.terminalFacts)) return false + return boundaryAllowsBeforeProgressRetry(input.terminalFacts) +} + +function isBeforeFirstProviderProgressCause(cause: TerminalCause) { + if (cause.category === "provider_transport_disconnect") return cause.subcategory === "before_first_provider_progress" + if (cause.category === "watchdog_timeout") return cause.subcategory === "connect" + return false +} + +function attemptHasNoOutputOrToolActivity(facts: IncidentFacts) { + return ( + !facts.visible_output_seen && + !facts.text_output_started && + !facts.reasoning_output_started && + !facts.tool_input_started && + !facts.tool_input_completed && + !facts.tool_call_materialized && + !facts.tool_execution_started && + !facts.tool_execution_completed && + !facts.read_only_tool_started && + !facts.unsafe_side_effect_started && + (facts.pending_tool_parts_interrupted ?? 0) === 0 + ) +} + +function boundaryAllowsBeforeProgressRetry(facts: IncidentFacts) { + const snapshot = facts.side_effect_boundary_snapshot + if (!snapshot) return false + if (snapshot.provider_executed_capability_present !== false) return false + if (snapshot.external_boundary_present !== false) return false + if ( + snapshot.proof_reason === "provider_executed_capability" || + snapshot.proof_reason === "external_boundary" || + snapshot.proof_reason === "unknown" + ) { + return false + } + return true +} + +function beforeProgressBoundaryEvidenceBlocksRetry(facts: IncidentFacts) { + const snapshot = facts.side_effect_boundary_snapshot + if (!snapshot) return true + if (snapshot.provider_executed_capability_present !== false) return true + if (snapshot.external_boundary_present !== false) return true + return ( + snapshot.proof_reason === "provider_executed_capability" || + snapshot.proof_reason === "external_boundary" || + snapshot.proof_reason === "unknown" + ) +} + function boundaryAllowsReasoningRetry(facts: IncidentFacts) { const snapshot = facts.side_effect_boundary_snapshot if (!snapshot) return false diff --git a/packages/opencode/test/session/run-observability.test.ts b/packages/opencode/test/session/run-observability.test.ts index 46e52f5c7..aa55864e5 100644 --- a/packages/opencode/test/session/run-observability.test.ts +++ b/packages/opencode/test/session/run-observability.test.ts @@ -3,6 +3,60 @@ import { MessageID, SessionID } from "../../src/session/schema" import { RunIncident } from "../../src/session/run-incident" import { RunObservability } from "../../src/session/run-observability" +const beforeProgressCause = { + category: "provider_transport_disconnect", + subcategory: "before_first_provider_progress", + confidence: "medium", +} satisfies RunIncident.TerminalCause + +function beforeProgressFacts(overrides: Partial = {}): RunIncident.Facts { + return { + provider_progress_seen: false, + visible_output_seen: false, + text_output_started: false, + reasoning_output_started: false, + tool_input_started: false, + tool_input_completed: false, + tool_call_materialized: false, + tool_execution_started: false, + tool_execution_completed: false, + read_only_tool_started: false, + unsafe_side_effect_started: false, + unsafe_side_effect_kinds: [], + side_effect_facts_complete: false, + side_effect_boundary_snapshot: { + exposed_tool_count: 16, + unknown_tool_count: 10, + unclassified_effect_count: 10, + provider_executed_capability_present: false, + external_boundary_present: false, + proof_result: "incomplete", + proof_reason: "unknown_tool_boundary", + }, + lifecycle_close_seen: false, + user_cancel_seen: false, + watchdog_fired: false, + ...overrides, + } +} + +function recoveryForBeforeProgress(overrides: Partial = {}, retryable = true) { + const facts = beforeProgressFacts(overrides) + return RunIncident.recoveryFor({ cause: beforeProgressCause, facts, retryable }) +} + +function noToolBoundarySnapshot(): RunObservability.SideEffectBoundarySnapshot { + return { + exposed_tool_count: 0, + unknown_tool_count: 0, + unclassified_effect_count: 0, + provider_executed_capability_present: false, + external_boundary_present: false, + proof_result: "complete", + proof_reason: "all_boundaries_classified", + } +} + describe("RunObservability", () => { test("does not treat stream lifecycle events as provider progress", () => { expect(RunObservability.isProviderProgressEvent({ type: "start" })).toBe(false) @@ -58,6 +112,12 @@ describe("RunObservability", () => { }) const attempt = recorder.beginAttempt({ attemptIndex: 1, at: 11, monotonicMs: 110 }) + recorder.recordSideEffectBoundarySnapshot({ + attemptID: attempt.attemptID, + at: 12, + monotonicMs: 120, + snapshot: noToolBoundarySnapshot(), + }) const decision = recorder.recordAttemptFailureAndDeriveRecovery({ attemptID: attempt.attemptID, at: 130, @@ -857,6 +917,12 @@ describe("RunObservability", () => { monotonicStartMs: 200, }) const currentAttempt = current.beginAttempt({ attemptIndex: 1, at: 21, monotonicMs: 210 }) + current.recordSideEffectBoundarySnapshot({ + attemptID: currentAttempt.attemptID, + at: 21, + monotonicMs: 211, + snapshot: noToolBoundarySnapshot(), + }) const decision = current.recordAttemptFailureAndDeriveRecovery({ attemptID: currentAttempt.attemptID, at: 22, @@ -951,7 +1017,7 @@ describe("RunObservability", () => { expect(summary.incident?.evidence?.map((event) => event.event_type)).toContain("provider_executed_tool_boundary") }) - test("unknown request side-effect boundary prevents auto retry before local tool events", () => { + test("unknown exposed tools still allow retry before first provider progress when no tool activity occurred", () => { const recorder = RunObservability.createRecorder({ runID: RunObservability.RunID.make("run_unknown_request_boundary"), traceID: MessageID.make("msg_unknown_request_boundary"), @@ -988,6 +1054,7 @@ describe("RunObservability", () => { cause: { name: "SocketError", message: "other side closed", code: "UND_ERR_SOCKET" }, }, evidence: ["iterator_error"], + retryable: true, }) const summary = recorder.finalize({ completedAt: 14, monotonicMs: 140 }) @@ -1002,11 +1069,114 @@ describe("RunObservability", () => { proof_result: "incomplete", proof_reason: "unknown_tool_boundary", }) + expect(decision).toMatchObject({ + recommendation: "auto_retry_once", + reason: "no_visible_output_or_tool_execution", + }) + expect(summary.incident?.recovery).toMatchObject({ + recommendation: "auto_retry_once", + reason: "no_visible_output_or_tool_execution", + }) + }) + + test("before-progress retry stays conservative when the boundary snapshot is missing", () => { + const decision = recoveryForBeforeProgress({ side_effect_boundary_snapshot: undefined }) + expect(decision).toMatchObject({ recommendation: "ask_user_before_retry", reason: "side_effect_facts_incomplete", }) - expect(summary.incident?.recovery).toMatchObject({ + }) + + test("before-progress retry stays conservative when complete facts lack a boundary snapshot", () => { + const decision = recoveryForBeforeProgress({ + side_effect_facts_complete: true, + side_effect_boundary_snapshot: undefined, + }) + + expect(decision).toMatchObject({ + recommendation: "ask_user_before_retry", + reason: "side_effect_facts_incomplete", + }) + }) + + test("before-progress retry fails closed for output, tool activity, or external boundary evidence", () => { + const cases: Array<[string, Partial]> = [ + ["visible output", { visible_output_seen: true }], + ["text output", { text_output_started: true }], + ["reasoning output", { reasoning_output_started: true }], + ["tool input started", { tool_input_started: true }], + ["tool input completed", { tool_input_completed: true }], + ["tool call materialized", { tool_call_materialized: true }], + ["tool execution started", { tool_execution_started: true }], + ["tool execution completed", { tool_execution_completed: true }], + ["read-only tool started", { read_only_tool_started: true }], + ["unsafe side effect", { unsafe_side_effect_started: true }], + ["pending tool parts", { pending_tool_parts_interrupted: 1 }], + [ + "provider-executed capability", + { + side_effect_boundary_snapshot: { + ...beforeProgressFacts().side_effect_boundary_snapshot!, + provider_executed_capability_present: true, + proof_reason: "provider_executed_capability", + }, + }, + ], + [ + "provider-executed capability with complete facts", + { + side_effect_facts_complete: true, + side_effect_boundary_snapshot: { + ...beforeProgressFacts().side_effect_boundary_snapshot!, + provider_executed_capability_present: true, + proof_reason: "provider_executed_capability", + }, + }, + ], + [ + "external boundary", + { + side_effect_boundary_snapshot: { + ...beforeProgressFacts().side_effect_boundary_snapshot!, + external_boundary_present: true, + proof_reason: "external_boundary", + }, + }, + ], + [ + "external boundary with complete facts", + { + side_effect_facts_complete: true, + side_effect_boundary_snapshot: { + ...beforeProgressFacts().side_effect_boundary_snapshot!, + external_boundary_present: true, + proof_reason: "external_boundary", + }, + }, + ], + [ + "unknown boundary proof with complete facts", + { + side_effect_facts_complete: true, + side_effect_boundary_snapshot: { + ...beforeProgressFacts().side_effect_boundary_snapshot!, + proof_reason: "unknown", + }, + }, + ], + ] + + for (const [name, overrides] of cases) { + const decision = recoveryForBeforeProgress(overrides) + expect(decision, name).not.toMatchObject({ recommendation: "auto_retry_once" }) + } + }) + + test("before-progress retry is denied when the transport error is not retryable", () => { + const decision = recoveryForBeforeProgress({}, false) + + expect(decision).toMatchObject({ recommendation: "ask_user_before_retry", reason: "side_effect_facts_incomplete", }) @@ -1853,6 +2023,12 @@ describe("RunObservability", () => { effect: RunObservability.toolEffect("bash"), }) const second = recorder.beginAttempt({ attemptIndex: 2, at: 20, monotonicMs: 200 }) + recorder.recordSideEffectBoundarySnapshot({ + attemptID: second.attemptID, + at: 20, + monotonicMs: 201, + snapshot: noToolBoundarySnapshot(), + }) recorder.recordTransportFailure({ attemptID: second.attemptID, at: 21, @@ -1892,6 +2068,12 @@ describe("RunObservability", () => { effect: RunObservability.toolEffect("read"), }) const second = recorder.beginAttempt({ attemptIndex: 2, at: 20, monotonicMs: 200 }) + recorder.recordSideEffectBoundarySnapshot({ + attemptID: second.attemptID, + at: 20, + monotonicMs: 201, + snapshot: noToolBoundarySnapshot(), + }) recorder.recordTransportFailure({ attemptID: second.attemptID, at: 21,