Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 40 additions & 18 deletions packages/opencode/src/session/processor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import { isOverflow } from "./overflow"
import { PartID } from "./schema"
import type { SessionID } from "./schema"
import { SessionRetry } from "./retry"
import { buildModelRetryDecision, selectRetryTimeoutPolicy, type RetryTimeoutPolicy } from "./retry-decision"
import { SessionStatus } from "./status"
import { SessionSummary } from "./summary"
import { SessionDiagnostics } from "./diagnostics"
Expand Down Expand Up @@ -151,6 +152,20 @@ function attemptStreamTimeouts(
}
}

function retryTimeoutPolicyFor(
model: Provider.Model,
automaticStreamRetriesUsed: number,
streamInput: Pick<LLM.StreamInput, "connectTimeoutMs">,
boundary: RunObservability.SideEffectBoundarySnapshot,
): RetryTimeoutPolicy {
return selectRetryTimeoutPolicy({
modelSupportsReasoning: model.capabilities.reasoning,
explicitConnectTimeout: streamInput.connectTimeoutMs !== undefined,
beforeProgressAutoRetryAllowed: RunObservability.boundaryAllowsBeforeProgressRetry(boundary),
safeRecoveryAttempt: automaticStreamRetriesUsed,
})
}

function sideEffectBoundarySnapshot(tools: LLM.StreamInput["tools"]): RunObservability.SideEffectBoundarySnapshot {
const entries = Object.entries(tools ?? {})
const names = entries.map(([name]) => name)
Expand Down Expand Up @@ -1397,20 +1412,25 @@ export const layer: Layer.Layer<
watchdog: retrySignal.watchdog,
retryable: retrySignal.retryable,
})
const reasoningOnlySafeRetry =
decision.recommendation === "auto_retry_once" &&
decision.reason === "reasoning_only_without_final_text_or_tool_activity"
const beforeProgressSafeRetry =
decision.recommendation === "auto_retry_once" &&
decision.reason === "no_visible_output_or_tool_execution"
const safeRecoveryRetry = reasoningOnlySafeRetry || beforeProgressSafeRetry
const retryDecision = buildModelRetryDecision({
technicalRetryability: retrySignal.retryable
? { retryable: true, message: retrySignal.message }
: {
retryable: false,
reason: ctx.terminalClassification ? "terminal_classification" : "not_retryable",
},
safetyGateDecision: decision,
modelStreamAttempt: ctx.attemptCount,
safeRecoveryAttempt: automaticStreamRetriesUsed,
timeoutPolicy: retryTimeoutPolicyFor(
streamInput.model,
automaticStreamRetriesUsed,
streamInput,
sideEffectBoundarySnapshot(LLM.resolveTools(streamInput)),
),
})

if (
attemptID &&
retrySignal.retryable &&
decision.recommendation === "auto_retry_once" &&
automaticStreamRetriesUsed === 0
) {
if (attemptID && retryDecision.canRetry && retryDecision.recoveryMode === "replay") {
const beforeRetry = yield* retryStillAllowed("before_backoff")
if (beforeRetry.allowed) {
automaticStreamRetriesUsed += 1
Expand All @@ -1419,9 +1439,12 @@ export const layer: Layer.Layer<
yield* status.set(ctx.sessionID, {
type: "retry",
attempt: ctx.attemptCount,
message: safeRecoveryRetry ? "" : (retrySignal.message ?? "Retrying interrupted stream"),
message:
retryDecision.presentation === "safe_recovery"
? ""
: (retrySignal.message ?? "Retrying interrupted stream"),
next,
...(safeRecoveryRetry
...(retryDecision.presentation === "safe_recovery"
? { presentation: "safe_recovery" as const, reason: "network_connection_dropped" as const }
: {}),
})
Expand Down Expand Up @@ -1452,9 +1475,8 @@ export const layer: Layer.Layer<

if (
attemptID &&
retrySignal.retryable &&
safeRecoveryRetry &&
automaticStreamRetriesUsed > 0
retryDecision.recoveryMode === "auto_replay_blocked" &&
retryDecision.presentation === "safe_recovery_failed"
) {
yield* writeSafeRetryFailedNotice(attemptID)
break
Expand Down
127 changes: 127 additions & 0 deletions packages/opencode/src/session/retry-decision.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
import type { RunIncident } from "./run-incident"
import type { RetryClassification } from "./retry-classification"

export type TechnicalRetryability =
| {
retryable: true
classification?: RetryClassification
message?: string
}
| {
retryable: false
classification?: RetryClassification
reason: "not_retryable" | "terminal_classification"
}

export type RetryAttemptKind = "provider_retry" | "safe_recovery_replay"
export type RecoveryMode = "replay" | "auto_replay_blocked" | "ask_user" | "offer_continue" | "stop"
export type RetryTimeoutPolicy =
| "default"
| "reasoning_global_protected"
| "reasoning_first_attempt"
| "reasoning_safe_recovery"
export type RetryPresentation = "default" | "safe_recovery" | "safe_recovery_failed"
export type RetryBlockedReason =
| "technical_not_retryable"
| "terminal_classification"
| "safe_recovery_budget_exhausted"
| RunIncident.Recovery["reason"]

export type ModelRetryDecision = {
technicalRetryability: TechnicalRetryability
safetyGateDecision: RunIncident.Recovery
canRetry: boolean
recoveryMode: RecoveryMode
blockedReason?: RetryBlockedReason
attemptKind?: RetryAttemptKind
modelStreamAttempt: number
safeRecoveryAttempt: number
timeoutPolicy: RetryTimeoutPolicy
presentation: RetryPresentation
}

export function selectRetryTimeoutPolicy(input: {
modelSupportsReasoning: boolean
explicitConnectTimeout: boolean
beforeProgressAutoRetryAllowed: boolean
safeRecoveryAttempt: number
}): RetryTimeoutPolicy {
if (input.explicitConnectTimeout) return "default"
if (!input.modelSupportsReasoning) return "default"
if (input.safeRecoveryAttempt > 0) return "reasoning_safe_recovery"
return input.beforeProgressAutoRetryAllowed ? "reasoning_first_attempt" : "reasoning_global_protected"
}

export function buildModelRetryDecision(input: {
technicalRetryability: TechnicalRetryability
safetyGateDecision: RunIncident.Recovery
modelStreamAttempt: number
safeRecoveryAttempt: number
timeoutPolicy: RetryTimeoutPolicy
}): ModelRetryDecision {
if (!input.technicalRetryability.retryable) {
return {
...input,
canRetry: false,
recoveryMode: "stop",
blockedReason:
input.technicalRetryability.reason === "terminal_classification"
? "terminal_classification"
: "technical_not_retryable",
presentation: "default",
}
}

const safety = input.safetyGateDecision
if (safety.recommendation === "auto_retry_once") {
const maxAttempts = safety.auto_retry?.max_attempts ?? 1
if (input.safeRecoveryAttempt < maxAttempts) {
return {
...input,
canRetry: true,
recoveryMode: "replay",
attemptKind: "safe_recovery_replay",
presentation: "safe_recovery",
}
}
return {
...input,
canRetry: false,
recoveryMode: "auto_replay_blocked",
blockedReason: "safe_recovery_budget_exhausted",
attemptKind: "safe_recovery_replay",
presentation: "safe_recovery_failed",
}
}

if (safety.recommendation === "offer_continue") {
return {
...input,
canRetry: false,
recoveryMode: "offer_continue",
blockedReason: safety.reason,
presentation: "default",
}
}

if (
safety.recommendation === "ask_user_before_retry" ||
safety.recommendation === "offer_resume_with_confirmation"
) {
return {
...input,
canRetry: false,
recoveryMode: "ask_user",
blockedReason: safety.reason,
presentation: "default",
}
}

return {
...input,
canRetry: false,
recoveryMode: "stop",
blockedReason: safety.reason,
presentation: "default",
}
}
137 changes: 137 additions & 0 deletions packages/opencode/test/session/retry-decision.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
import { describe, expect, test } from "bun:test"
import { buildModelRetryDecision, selectRetryTimeoutPolicy } from "../../src/session/retry-decision"
import type { RunIncident } from "../../src/session/run-incident"

const safeReplayGate: RunIncident.Recovery = {
recommendation: "auto_retry_once",
confidence: "high",
reason: "reasoning_only_without_final_text_or_tool_activity",
auto_retry: { max_attempts: 1, backoff_ms: 1_000 },
safety_scope: "visible_output_and_tool_side_effects",
}

const visibleOutputGate: RunIncident.Recovery = {
recommendation: "offer_continue",
confidence: "high",
reason: "visible_output_without_tool_execution",
safety_scope: "visible_output_and_tool_side_effects",
}

const ambiguousToolGate: RunIncident.Recovery = {
recommendation: "ask_user_before_retry",
confidence: "high",
reason: "side_effect_facts_incomplete",
safety_scope: "visible_output_and_tool_side_effects",
}

describe("session.retry-decision", () => {
test("keeps technical retryability separate from safe recovery replay metadata", () => {
const decision = buildModelRetryDecision({
technicalRetryability: { retryable: true, message: "Connection timed out" },
safetyGateDecision: safeReplayGate,
modelStreamAttempt: 3,
safeRecoveryAttempt: 0,
timeoutPolicy: "reasoning_first_attempt",
})

expect(decision).toMatchObject({
canRetry: true,
recoveryMode: "replay",
attemptKind: "safe_recovery_replay",
modelStreamAttempt: 3,
safeRecoveryAttempt: 0,
timeoutPolicy: "reasoning_first_attempt",
presentation: "safe_recovery",
})
expect(decision.blockedReason).toBeUndefined()
expect(decision.technicalRetryability.retryable).toBe(true)
expect(decision.safetyGateDecision.reason).toBe("reasoning_only_without_final_text_or_tool_activity")
})

test("blocks automatic replay when the safe recovery budget is exhausted", () => {
const decision = buildModelRetryDecision({
technicalRetryability: { retryable: true, message: "Connection timed out" },
safetyGateDecision: safeReplayGate,
modelStreamAttempt: 3,
safeRecoveryAttempt: 1,
timeoutPolicy: "reasoning_safe_recovery",
})

expect(decision).toMatchObject({
canRetry: false,
recoveryMode: "auto_replay_blocked",
attemptKind: "safe_recovery_replay",
modelStreamAttempt: 3,
safeRecoveryAttempt: 1,
timeoutPolicy: "reasoning_safe_recovery",
presentation: "safe_recovery_failed",
blockedReason: "safe_recovery_budget_exhausted",
})
})

test("does not ask the safety gate to own terminal technical classification", () => {
const decision = buildModelRetryDecision({
technicalRetryability: { retryable: false, reason: "terminal_classification" },
safetyGateDecision: safeReplayGate,
modelStreamAttempt: 1,
safeRecoveryAttempt: 0,
timeoutPolicy: "default",
})

expect(decision).toMatchObject({
canRetry: false,
recoveryMode: "stop",
blockedReason: "terminal_classification",
presentation: "default",
})
expect(decision.safetyGateDecision).toBe(safeReplayGate)
})
Comment thread
Astro-Han marked this conversation as resolved.

test("represents continuation offers without treating them as replay", () => {
const decision = buildModelRetryDecision({
technicalRetryability: { retryable: true, message: "socket closed" },
safetyGateDecision: visibleOutputGate,
modelStreamAttempt: 2,
safeRecoveryAttempt: 0,
timeoutPolicy: "default",
})

expect(decision).toMatchObject({
canRetry: false,
recoveryMode: "offer_continue",
blockedReason: "visible_output_without_tool_execution",
presentation: "default",
})
expect(decision.attemptKind).toBeUndefined()
})

test("represents safety-confirmation gates without consuming the replay budget", () => {
const decision = buildModelRetryDecision({
technicalRetryability: { retryable: true, message: "socket closed" },
safetyGateDecision: ambiguousToolGate,
modelStreamAttempt: 2,
safeRecoveryAttempt: 0,
timeoutPolicy: "default",
})

expect(decision).toMatchObject({
canRetry: false,
recoveryMode: "ask_user",
blockedReason: "side_effect_facts_incomplete",
safeRecoveryAttempt: 0,
presentation: "default",
})
expect(decision.attemptKind).toBeUndefined()
})

test("marks blocked-boundary reasoning first attempts as global protected timeout", () => {
const timeoutPolicy = selectRetryTimeoutPolicy({
modelSupportsReasoning: true,
explicitConnectTimeout: false,
beforeProgressAutoRetryAllowed: false,
safeRecoveryAttempt: 0,
})

expect(timeoutPolicy).toBe("reasoning_global_protected")
})
})