From 3f4c7719d813037e51ad5f87001a8986074da9d5 Mon Sep 17 00:00:00 2001 From: Bil0000 Date: Sat, 3 Oct 2026 17:39:19 +0300 Subject: [PATCH 01/10] fix(v2): compact full sessions before provider handoffs --- .../ContextHandoffDelivery.ts | 51 +- apps/server/src/orchestration-v2/EventSink.ts | 9 +- .../FoundationPersistence.test.ts | 12 + .../src/orchestration-v2/Orchestrator.ts | 10 +- .../ProviderTurnStartService.ts | 614 +++++++++++------- .../RestartContinuation.test.ts | 1 + .../orchestration-v2/RestartContinuation.ts | 10 +- .../orchestration-v2/RunExecutionService.ts | 64 ++ .../ProviderSwitch.integration.test.ts | 400 ++++++++++-- docs/user/portable-handoffs.md | 20 +- packages/contracts/src/orchestrationV2.ts | 1 + 11 files changed, 870 insertions(+), 322 deletions(-) diff --git a/apps/server/src/orchestration-v2/ContextHandoffDelivery.ts b/apps/server/src/orchestration-v2/ContextHandoffDelivery.ts index 2ef3a5a5cd89..7945e4afcd78 100644 --- a/apps/server/src/orchestration-v2/ContextHandoffDelivery.ts +++ b/apps/server/src/orchestration-v2/ContextHandoffDelivery.ts @@ -7,19 +7,16 @@ import * as Effect from "effect/Effect"; import * as Schema from "effect/Schema"; import { historyCost, renderHistory, selectHistory } from "./ContextHandoffBudget.ts"; -/** Persist before/after injection: an ambiguous pending delivery requires a fresh native thread. */ -export const deliverContextHandoffs = Effect.fn("orchestrationV2.deliverContextHandoffs")( - function* (input: { - readonly handoffs: ReadonlyArray; - readonly providerThread: OrchestrationV2ProviderThread; - readonly budget: number | Effect.Effect; - readonly deferInline?: boolean; - readonly alreadyDeliveredItemIds: ReadonlySet; - readonly inject?: ( - history: ProviderAdapterV2HistoricalContext, - ) => Effect.Effect; - readonly persist: (handoff: OrchestrationV2ContextHandoff) => Effect.Effect; - }) { +type ContextHandoffInput = { + readonly handoffs: ReadonlyArray; + readonly providerThread: OrchestrationV2ProviderThread; + readonly budget: number | Effect.Effect; + readonly deferInline?: boolean; + readonly alreadyDeliveredItemIds: ReadonlySet; +}; + +export const prepareContextHandoffs = Effect.fn("orchestrationV2.prepareContextHandoffs")( + function* (input: ContextHandoffInput) { const nativeThreadId = input.providerThread.nativeThreadRef?.nativeId ?? undefined; const pending = input.handoffs.filter( (handoff) => @@ -27,8 +24,7 @@ export const deliverContextHandoffs = Effect.fn("orchestrationV2.deliverContextH handoff.delivery?.nativeThreadId !== nativeThreadId || handoff.delivery.status === "pending", ); - if (pending.length === 0 || (input.deferInline && input.inject === undefined)) - return { context: "", delivered: Effect.void }; + if (pending.length === 0) return undefined; const budget = typeof input.budget === "number" ? input.budget : yield* input.budget; let coverage = pending .map( @@ -71,9 +67,30 @@ export const deliverContextHandoffs = Effect.fn("orchestrationV2.deliverContextH budget, }); if (historyCost(selected.messages, selected.context) > budget) { - if (input.deferInline) return { context: "", delivered: Effect.void }; + if (input.deferInline) return undefined; return yield* new ContextHandoffBudgetError(); } + return { nativeThreadId, pending, selected }; + }, +); + +/** Persist before/after injection: an ambiguous pending delivery requires a fresh native thread. */ +export const deliverContextHandoffs = Effect.fn("orchestrationV2.deliverContextHandoffs")( + function* ( + input: ContextHandoffInput & { + readonly inject?: ( + history: ProviderAdapterV2HistoricalContext, + ) => Effect.Effect; + readonly persist: ( + handoff: OrchestrationV2ContextHandoff, + ) => Effect.Effect; + }, + ) { + if (input.deferInline && input.inject === undefined) + return { context: "", delivered: Effect.void }; + const prepared = yield* prepareContextHandoffs(input); + if (prepared === undefined) return { context: "", delivered: Effect.void }; + const { nativeThreadId, pending, selected } = prepared; const omittedItemIds = new Set(selected.omittedItemIds); const persist = (status: "pending" | "injected" | "inline") => Effect.forEach( @@ -146,7 +163,7 @@ export class ContextHandoffBudgetError extends Schema.TaggedError()( diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index 71b6f0ea994f..ca20e8f7217d 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -90,6 +90,7 @@ export interface EventSinkV2Shape { readonly activeAttemptId: RunAttemptId; readonly expectedStatus: OrchestrationV2Run["status"]; readonly events: ReadonlyArray; + readonly effects?: ReadonlyArray; }) => Effect.Effect< { readonly committed: boolean; @@ -437,9 +438,15 @@ const baseLayer: Layer.Layer< events: normalized, }); yield* applyStoredEvents(storedEvents); + yield* effectOutbox.enqueue(input.effects ?? []); return { committed: true as const, storedEvents }; }), - (result) => (result.committed ? publishStoredEvents(result.storedEvents) : Effect.void), + (result) => + Effect.gen(function* () { + if (!result.committed) return; + if (input.effects?.length) yield* effectOutbox.notifyAvailable(input.effects.length); + yield* publishStoredEvents(result.storedEvents); + }), ); }, ); diff --git a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts index dc7eacc59ba5..6e5d740e39ad 100644 --- a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts +++ b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts @@ -1513,6 +1513,8 @@ it.layer(TestLayer)("orchestration V2 foundation persistence", (it) => { Effect.gen(function* () { const eventSink = yield* EventSink.EventSinkV2; const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const outbox = yield* EffectOutbox.EffectOutboxV2; + const commandId = CommandId.make("guarded-continuation"); const now = yield* DateTime.now; const threadId = ThreadId.make("thread:foundation-stale-provider-start"); const runId = RunId.make("run:foundation-stale-provider-start"); @@ -1566,6 +1568,14 @@ it.layer(TestLayer)("orchestration V2 foundation persistence", (it) => { runId, activeAttemptId: attemptId, expectedStatus: "starting", + effects: [ + { + id: "guarded-continuation", + commandId, + threadId, + request: { type: "provider-turn.start", runId }, + }, + ], events: [ { id: EventId.make("event:foundation-stale-provider-start:running"), @@ -1607,6 +1617,7 @@ it.layer(TestLayer)("orchestration V2 foundation persistence", (it) => { const staleResult = yield* Fiber.join(staleStartFiber); assert.isFalse(staleResult.committed); + assert.deepEqual(yield* outbox.listByCommandId(commandId), []); assert.deepEqual(staleResult.storedEvents, []); assert.equal(yield* Ref.get(providerStartCount), 0); const projection = yield* projectionStore.getThreadProjection(threadId); @@ -2630,6 +2641,7 @@ it.layer(TestLayer)("orchestration V2 foundation persistence", (it) => { activeAttemptId: attemptId, }, ], + attempts: [], providerThreads: [ { id: providerThreadId, diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 2707fe5b8344..f515bb4225da 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -3609,12 +3609,18 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio const targetMessage = input.projection.messages.find( (message) => message.id === targetRun.userMessageId, ); - if (targetMessage !== undefined && isNativeMaintenanceCommand(targetMessage)) { + const compacting = input.projection.attempts.some( + (attempt) => attempt.id === targetRun.activeAttemptId && attempt.contextCompaction, + ); + if ( + compacting || + (targetMessage !== undefined && isNativeMaintenanceCommand(targetMessage)) + ) { return yield* new OrchestratorDispatchError({ commandId: input.command.commandId, commandType: input.command.type, cause: - targetMessage.text.trim().toLowerCase() === "/compact" + compacting || targetMessage?.text.trim().toLowerCase() === "/compact" ? "Wait for context compaction to finish before steering the thread." : "Wait for sign-out to finish before steering the thread.", }); diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index ce2088d911d4..eebbb6882baa 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -34,7 +34,11 @@ import { historicalMessage, latestNativeContextUsage, } from "./ContextHandoffBudget.ts"; -import { deliverContextHandoffs } from "./ContextHandoffDelivery.ts"; +import { + ContextHandoffBudgetError, + deliverContextHandoffs, + prepareContextHandoffs, +} from "./ContextHandoffDelivery.ts"; import { ProviderAdapterTurnStartError, type ProviderAdapterV2Error, @@ -601,7 +605,305 @@ export const layer: Layer.Layer< }); return undefined; }); + // Delivered once: this run's provider turn marks the work as told. A + // restart continuation is prompted by its own text or resumes natively. + const noteContinuation = isRestartNoteContinuation( + run, + projection.runs, + projection.providerTurns, + ); + const restartCancelledWork = pendingRestartCancelledBackgroundWork({ + runs: projection.runs, + providerTurns: projection.providerTurns, + compactionMessageIds: new Set( + projection.messages + .filter( + (candidate) => + candidate.attachments.length === 0 && + candidate.text.trim().toLowerCase() === "/compact", + ) + .map((candidate) => candidate.id), + ), + run, + runAttemptIds: projection.attempts + .filter((candidate) => candidate.runId === run.id) + .map((candidate) => candidate.id), + }); + const restartNote = + restartCancelledWork.length === 0 + ? "" + : restartCancelledBackgroundWorkNote(restartCancelledWork); + const userText = projectComposerContextForProvider({ + text: message.text, + records: message.context?.records ?? [], + }); + const tokenCap = yield* handoffTokenCapConfig.pipe( + Effect.orElseSucceed(() => DEFAULT_HANDOFF_TOKEN_CAP), + ); let effectiveHandoffs = handoffs; + const getHandoffContext = Effect.fn("orchestrationV2.providerTurnStart.handoffContext")( + function* (loadedProviderThread: OrchestrationV2ProviderThread) { + // Only started runs reached the provider-thread update below. Queued runs and + // failures during session setup cannot establish a new telemetry selection. + const compactionAttempts = new Map( + projection.attempts + .filter((candidate) => candidate.contextCompaction) + .map((candidate) => [candidate.id, candidate]), + ); + const compactedTurn = + compactionAttempts.size === 0 + ? undefined + : projection.providerTurns.findLast((turn) => { + if (turn.providerThreadId !== providerThread.id || turn.runAttemptId === null) + return false; + const candidate = compactionAttempts.get(turn.runAttemptId); + return ( + candidate?.status === "completed" && + candidate.nativeThreadId === loadedProviderThread.nativeThreadRef?.nativeId + ); + }); + const priorContext = latestNativeContextUsage(projection, providerThread); + const measuredContext = + compactedTurn === undefined + ? priorContext + : latestNativeContextUsage( + { + ...projection, + providerTurns: projection.providerTurns.filter( + (turn) => turn.ordinal >= compactedTurn.ordinal, + ), + }, + providerThread, + ); + const previousSelection = + measuredContext?.modelSelection ?? + projection.runs.findLast( + (source) => + source.ordinal < run.ordinal && + source.startedAt !== null && + source.providerThreadId === providerThread.id, + )?.modelSelection; + const sameSelection = + previousSelection === undefined || + modelSelectionsEqual(previousSelection, run.modelSelection); + const sameNativeThread = + loadedProviderThread.nativeThreadRef?.nativeId === + providerThread.nativeThreadRef?.nativeId; + const threadUsage = + compactedTurn === undefined + ? (loadedProviderThread.contextUsage ?? providerThread.contextUsage) + : null; + const previousUsage = measuredContext + ? { ...threadUsage, ...measuredContext.usage } + : threadUsage; + const settledHandoffs = projection.contextHandoffs.filter( + (handoff) => + handoff.toProviderThreadId === providerThread.id && + handoff.delivery?.nativeThreadId === loadedProviderThread.nativeThreadRef?.nativeId && + handoff.delivery?.status !== "pending", + ); + const deliveredItemIds = new Set( + settledHandoffs.flatMap((handoff) => handoff.delivery?.itemIds ?? []), + ); + const coveredItemIds = new Set([ + ...deliveredItemIds, + ...settledHandoffs.flatMap((handoff) => handoff.delivery?.omittedItemIds ?? []), + ]); + const deliveredAttemptIds = new Set( + projection.providerTurns + .filter( + (turn) => turn.runAttemptId !== null && !compactionAttempts.has(turn.runAttemptId), + ) + .map((turn) => turn.runAttemptId), + ); + const acceptedAttempts = projection.attempts.filter( + (source) => + source.providerThreadId === providerThread.id && deliveredAttemptIds.has(source.id), + ); + const nativeInputRunIds = new Set( + acceptedAttempts + .filter( + (source) => + source.nativeThreadId !== undefined && + source.nativeThreadId === loadedProviderThread.nativeThreadRef?.nativeId, + ) + .map((source) => source.runId), + ); + const replacedRunIds = new Set( + acceptedAttempts + .filter( + (source) => + source.nativeThreadId !== undefined && !nativeInputRunIds.has(source.runId), + ) + .map((source) => source.runId), + ); + const legacyInputRunIds = new Set( + acceptedAttempts + .filter((source) => source.nativeThreadId === undefined) + .map((source) => source.runId), + ); + const legacyRecoveredRunIds = new Set( + projection.runs + .filter( + (source) => + source.providerThreadId === providerThread.id && + settledHandoffs.some( + (handoff) => + handoff.strategy === "full_thread_summary" && + handoff.fromProviderThreadIds.includes(providerThread.id) && + source.ordinal >= handoff.coveredRunOrdinals.from && + source.ordinal <= handoff.coveredRunOrdinals.to, + ), + ) + .map((source) => source.id), + ); + // Use saved text and actual native attachments when telemetry is absent. + // Legacy attempts lack native identity; exclude their explicitly recovered + // history, whose attachments were not replayed into the replacement thread. + const nativeContextEstimate = Effect.gen(function* () { + return sameNativeThread + ? (yield* projectionStore.getTurnStartHistory(input.threadId)).reduce((sum, item) => { + if ( + item.runId === run.id || + (compactedTurn?.completedAt != null && + item.startedAt !== null && + DateTime.toEpochMillis(item.startedAt) <= + DateTime.toEpochMillis(compactedTurn.completedAt)) || + (item.runId !== null && + (missedRunIds.has(item.runId) || replacedRunIds.has(item.runId)) && + !deliveredItemIds.has(item.id)) || + (item.providerThreadId !== providerThread.id && !deliveredItemIds.has(item.id)) + ) + return sum; + const historical = historicalMessage(item); + const nativeAttachments = + item.type === "user_message" && + item.providerThreadId === providerThread.id && + item.runId !== null && + (nativeInputRunIds.has(item.runId) || + (legacyInputRunIds.has(item.runId) && + !coveredItemIds.has(item.id) && + !legacyRecoveredRunIds.has(item.runId))) + ? attachmentTokenAllowance(item.attachments) + : 0; + return ( + sum + + (historical === null ? 0 : Buffer.byteLength(historical.text)) + + nativeAttachments + ); + }, 0) + : 0; + }); + const reuseTelemetry = + sameSelection || + (previousSelection !== undefined && + session.canReuseContextUsage?.(previousSelection, run.modelSelection) === true); + const selectedModelContextWindow = session.getModelContextWindow?.( + run.modelSelection, + resolvedRuntimePolicy.cwd, + ); + const handoffUsage = contextUsageForHandoff({ + sameNativeThread, + sameSelection, + reuseTelemetry, + previousUsage, + knownModelWindow: selectedModelContextWindow, + }); + const modelContextWindow = + selectedModelContextWindow ?? + (handoffUsage !== null || reuseTelemetry ? previousUsage?.maxTokens : undefined) ?? + (priorContext !== undefined && + modelSelectionsEqual(priorContext.modelSelection, run.modelSelection) + ? priorContext.usage.maxTokens + : undefined); + const previousModelContextWindow = + previousSelection === undefined + ? undefined + : (session.getModelContextWindow?.(previousSelection, resolvedRuntimePolicy.cwd) ?? + previousUsage?.maxTokens); + const checkModelChange = + !sameSelection && + selectedModelContextWindow !== undefined && + selectedModelContextWindow < (previousModelContextWindow ?? Infinity); + const budgetProviderThread = { + ...loadedProviderThread, + contextUsage: handoffUsage, + }; + const missedRuns = projection.runs.filter( + (source) => + source.ordinal < run.ordinal && + source.providerThreadId === providerThread.id && + (source.status === "failed" || source.status === "interrupted") && + !deliveredAttemptIds.has(source.activeAttemptId), + ); + const missedRunIds = new Set(missedRuns.map((source) => source.id)); + const missedItems = + missedRunIds.size === 0 + ? [] + : (yield* projectionStore.getTurnStartHistory(input.threadId, [ + ...missedRunIds, + ])).filter( + (item) => + item.runId !== null && + missedRunIds.has(item.runId) && + !coveredItemIds.has(item.id) && + historicalMessage(item) !== null, + ); + // A failed turn/start can leave the requested turn absent from + // native history even when its preceding handoff was injected. + const retryHandoff = + missedItems.length === 0 + ? [] + : [ + yield* contextHandoffService.prepareProviderHandoff({ + threadId: projection.thread.id, + targetRunId: run.id, + transferId: null, + fromProviderThreadIds: [providerThread.id], + toProviderThreadId: providerThread.id, + fromProviderInstanceId: run.providerInstanceId, + toProviderInstanceId: run.providerInstanceId, + coveredRunOrdinals: { + from: missedRuns[0]!.ordinal, + to: missedRuns.at(-1)!.ordinal, + }, + strategy: "delta_since_target_last_seen", + items: missedItems, + runs: projection.runs, + createdAt: yield* DateTime.now, + }), + ]; + const budgetInput = { + tokenCap, + modelContextWindow, + userText: restartNote === "" ? userText : `${restartNote}\n\n${userText}`, + attachments: message.attachments, + providerThread: budgetProviderThread, + nativeContextEstimate: + budgetProviderThread.contextUsage?.usedTokens === undefined && + (effectiveHandoffs.length > 0 || missedItems.length > 0 || checkModelChange) + ? yield* nativeContextEstimate + : 0, + }; + return { + checkModelChange, + modelContextWindow, + contextUsage: budgetProviderThread.contextUsage, + budget: handoffBudget(budgetInput), + preflightBudget: handoffBudget({ + ...budgetInput, + nativeContextEstimate: Math.max( + budgetInput.nativeContextEstimate, + sameNativeThread ? (previousUsage?.usedTokens ?? 0) : 0, + ), + }), + deliveredItemIds, + retryHandoff, + }; + }, + ); + let resumedContext: Effect.Success> | undefined; + let contextCompaction = false; const loadedProviderThread = yield* Effect.gen(function* () { if (nativeForkTransfer !== undefined) { const sourceProjection = yield* projectionStore.getThreadRecords( @@ -681,9 +983,50 @@ export const layer: Layer.Layer< }), ); if (resumed._tag === "Success") { + if (message.text.trim().toLowerCase() === "/compact" && message.attachments.length === 0) + return resumed.success; + const context = yield* getHandoffContext(resumed.success); + const prepared = yield* Effect.result( + prepareContextHandoffs({ + handoffs: [...effectiveHandoffs, ...context.retryHandoff], + providerThread: resumed.success, + budget: context.preflightBudget, + alreadyDeliveredItemIds: context.deliveredItemIds, + }), + ); + const contextExhausted = + prepared._tag === "Failure" || + (context.checkModelChange && context.preflightBudget === 0); + if (contextExhausted) { + if ( + session.compactThread === undefined || + projection.attempts.some( + (candidate) => candidate.runId === run.id && candidate.contextCompaction, + ) + ) + return yield* new ContextHandoffBudgetError(); + const emptyBudget = handoffBudget({ + tokenCap, + modelContextWindow: context.modelContextWindow, + userText: restartNote === "" ? userText : `${restartNote}\n\n${userText}`, + attachments: message.attachments, + providerThread: { ...resumed.success, contextUsage: null }, + nativeContextEstimate: 0, + }); + if (emptyBudget === 0) return yield* new ContextHandoffBudgetError(); + yield* prepareContextHandoffs({ + handoffs: [...effectiveHandoffs, ...context.retryHandoff], + providerThread: resumed.success, + budget: emptyBudget, + alreadyDeliveredItemIds: context.deliveredItemIds, + }); + contextCompaction = true; + } + resumedContext = context; return resumed.success; } + if (!(yield* isCurrentAttemptInStatus("starting"))) return providerThread; yield* Effect.logWarning("Provider resume failed; attempting a fresh native session", { driver: session.driver, providerThreadId: providerThread.id, @@ -771,54 +1114,41 @@ export const layer: Layer.Layer< ], }); return replacement; - }); - // The last attempt already failed the run. - if (loadedProviderThread === undefined) return; - if (!(yield* isCurrentAttemptInStatus("starting"))) { + }).pipe( + Effect.catchTags({ + ContextHandoffBudgetError: (cause) => + Effect.gen(function* () { + yield* settleRunBeforeStart({ + signal: "provider-thread-setup-failure", + status: "failed", + now: yield* DateTime.now, + providerInstanceId: run.providerInstanceId, + itemProviderThreadId: providerThread.id, + item: { + type: "error", + title: "Provider error", + failure: makeProviderFailure({ + class: "validation_error", + cause, + message: cause.message, + }), + }, + }); + return null; + }), + }), + ); + if (loadedProviderThread == null || !(yield* isCurrentAttemptInStatus("starting"))) { return; } const now = yield* DateTime.now; - // Only started runs reached the provider-thread update below. Queued runs and - // failures during session setup cannot establish a new telemetry selection. - const measuredContext = latestNativeContextUsage(projection, providerThread); - const previousSelection = - measuredContext?.modelSelection ?? - projection.runs.findLast( - (source) => - source.ordinal < run.ordinal && - source.startedAt !== null && - source.providerThreadId === providerThread.id, - )?.modelSelection; - const sameSelection = - previousSelection === undefined || - modelSelectionsEqual(previousSelection, run.modelSelection); - const sameNativeThread = - loadedProviderThread.nativeThreadRef?.nativeId === providerThread.nativeThreadRef?.nativeId; - const threadUsage = loadedProviderThread.contextUsage ?? providerThread.contextUsage; - const previousUsage = measuredContext - ? { ...threadUsage, ...measuredContext.usage } - : threadUsage; - const reuseTelemetry = - sameSelection || - (previousSelection !== undefined && - session.canReuseContextUsage?.(previousSelection, run.modelSelection) === true); - const knownModelWindow = session.getModelContextWindow?.( - run.modelSelection, - resolvedRuntimePolicy.cwd, - ); - // Persist before delivery. Keep this native transcript's measured - // occupancy. A different model drops compaction telemetry and uses the - // new window when that window is known. - const handoffUsage = contextUsageForHandoff({ - sameNativeThread, - sameSelection, - reuseTelemetry, - previousUsage, - knownModelWindow, - }); + const { contextUsage, budget, deliveredItemIds, retryHandoff } = + resumedContext ?? (yield* getHandoffContext(loadedProviderThread)); const runningProviderThread: OrchestrationV2ProviderThread = { ...loadedProviderThread, - contextUsage: handoffUsage, + // Persist invalidation before delivery: a failed start must not let the next + // attempt mistake old-model telemetry for usage of the new selection. + contextUsage, id: providerThread.id, driver: session.driver, providerInstanceId: run.providerInstanceId, @@ -840,6 +1170,7 @@ export const layer: Layer.Layer< }; const runningAttempt: OrchestrationV2RunAttempt = { ...attempt, + ...(contextCompaction ? { contextCompaction: true } : {}), ...(runningProviderThread.nativeThreadRef?.nativeId == null ? {} : { nativeThreadId: runningProviderThread.nativeThreadRef.nativeId }), @@ -943,196 +1274,16 @@ export const layer: Layer.Layer< const routableSubagents = projection.subagents.filter((subagent) => RunExecutionService.canRouteRelatedSubagent(subagent.status), ); - const userText = projectComposerContextForProvider({ - text: message.text, - records: message.context?.records ?? [], - }); - // Delivered once: this run's provider turn marks the work as told. A - // restart continuation is prompted by its own text or resumes natively. - const noteContinuation = isRestartNoteContinuation( - run, - projection.runs, - projection.providerTurns, - ); - const restartCancelledWork = pendingRestartCancelledBackgroundWork({ - runs: projection.runs, - providerTurns: projection.providerTurns, - compactionMessageIds: new Set( - projection.messages - .filter( - (candidate) => - candidate.attachments.length === 0 && - candidate.text.trim().toLowerCase() === "/compact", - ) - .map((candidate) => candidate.id), - ), - run, - runAttemptIds: projection.attempts - .filter((candidate) => candidate.runId === run.id) - .map((candidate) => candidate.id), - }); - const restartNote = - restartCancelledWork.length === 0 - ? "" - : restartCancelledBackgroundWorkNote(restartCancelledWork); - const tokenCap = yield* handoffTokenCapConfig.pipe( - Effect.orElseSucceed(() => DEFAULT_HANDOFF_TOKEN_CAP), - ); - const settledHandoffs = projection.contextHandoffs.filter( - (handoff) => - handoff.toProviderThreadId === providerThread.id && - handoff.delivery?.nativeThreadId === runningProviderThread.nativeThreadRef?.nativeId && - handoff.delivery?.status !== "pending", - ); - const deliveredItemIds = new Set( - settledHandoffs.flatMap((handoff) => handoff.delivery?.itemIds ?? []), - ); - const coveredItemIds = new Set([ - ...deliveredItemIds, - ...settledHandoffs.flatMap((handoff) => handoff.delivery?.omittedItemIds ?? []), - ]); - const deliveredAttemptIds = new Set( - projection.providerTurns.map((turn) => turn.runAttemptId), - ); - const acceptedAttempts = projection.attempts.filter( - (source) => - source.providerThreadId === providerThread.id && deliveredAttemptIds.has(source.id), - ); - const nativeInputRunIds = new Set( - acceptedAttempts - .filter( - (source) => - source.nativeThreadId !== undefined && - source.nativeThreadId === runningProviderThread.nativeThreadRef?.nativeId, - ) - .map((source) => source.runId), - ); - const legacyInputRunIds = new Set( - acceptedAttempts - .filter((source) => source.nativeThreadId === undefined) - .map((source) => source.runId), - ); - const legacyRecoveredRunIds = new Set( - projection.runs - .filter( - (source) => - source.providerThreadId === providerThread.id && - settledHandoffs.some( - (handoff) => - handoff.strategy === "full_thread_summary" && - handoff.fromProviderThreadIds.includes(providerThread.id) && - source.ordinal >= handoff.coveredRunOrdinals.from && - source.ordinal <= handoff.coveredRunOrdinals.to, - ), - ) - .map((source) => source.id), - ); - // Use saved text and actual native attachments when telemetry is absent. - // Legacy attempts lack native identity; exclude their explicitly recovered - // history, whose attachments were not replayed into the replacement thread. - const nativeContextEstimate = Effect.gen(function* () { - return sameNativeThread - ? (yield* projectionStore.getTurnStartHistory(input.threadId)).reduce((sum, item) => { - if ( - item.runId === run.id || - (item.runId !== null && - missedRunIds.has(item.runId) && - !deliveredItemIds.has(item.id)) || - (item.providerThreadId !== providerThread.id && !deliveredItemIds.has(item.id)) - ) - return sum; - const historical = historicalMessage(item); - const nativeAttachments = - item.type === "user_message" && - item.providerThreadId === providerThread.id && - item.runId !== null && - (nativeInputRunIds.has(item.runId) || - (legacyInputRunIds.has(item.runId) && - !coveredItemIds.has(item.id) && - !legacyRecoveredRunIds.has(item.runId))) - ? attachmentTokenAllowance(item.attachments) - : 0; - return ( - sum + - (historical === null ? 0 : Buffer.byteLength(historical.text)) + - nativeAttachments - ); - }, 0) - : 0; - }); - const modelContextWindow = - knownModelWindow ?? - (handoffUsage !== null || reuseTelemetry ? previousUsage?.maxTokens : undefined); - // Replacing a native thread clears its usage, not the selected model's capacity. - const budgetProviderThread = { - ...runningProviderThread, - contextUsage: handoffUsage, - }; - const missedRuns = projection.runs.filter( - (source) => - source.ordinal < run.ordinal && - source.providerThreadId === providerThread.id && - (source.status === "failed" || source.status === "interrupted") && - !deliveredAttemptIds.has(source.activeAttemptId), - ); - const missedRunIds = new Set(missedRuns.map((source) => source.id)); - const missedItems = - missedRunIds.size === 0 - ? [] - : (yield* projectionStore.getTurnStartHistory(input.threadId, [...missedRunIds])).filter( - (item) => - item.runId !== null && - missedRunIds.has(item.runId) && - !coveredItemIds.has(item.id) && - historicalMessage(item) !== null, - ); const startWithHandoffs = ( turnInput: Parameters[0], compact = false, ) => Effect.gen(function* () { - // A failed turn/start can leave the requested turn absent from - // native history even when its preceding handoff was injected. - const retryHandoff = - missedItems.length === 0 - ? [] - : [ - yield* contextHandoffService.prepareProviderHandoff({ - threadId: projection.thread.id, - targetRunId: run.id, - transferId: null, - fromProviderThreadIds: [providerThread.id], - toProviderThreadId: providerThread.id, - fromProviderInstanceId: run.providerInstanceId, - toProviderInstanceId: run.providerInstanceId, - coveredRunOrdinals: { - from: missedRuns[0]!.ordinal, - to: missedRuns.at(-1)!.ordinal, - }, - strategy: "delta_since_target_last_seen", - items: missedItems, - runs: projection.runs, - createdAt: yield* DateTime.now, - }), - ]; const delivery = yield* deliverContextHandoffs({ handoffs: [...effectiveHandoffs, ...retryHandoff], deferInline: compact, providerThread: runningProviderThread, - budget: Effect.gen(function* () { - return handoffBudget({ - tokenCap, - modelContextWindow, - // The note is sent with the user text, so it spends the same allowance. - userText: restartNote === "" ? userText : `${restartNote}\n\n${userText}`, - attachments: message.attachments, - providerThread: budgetProviderThread, - nativeContextEstimate: - budgetProviderThread.contextUsage?.usedTokens === undefined - ? yield* nativeContextEstimate - : 0, - }); - }), + budget, alreadyDeliveredItemIds: deliveredItemIds, ...(session.injectHistory === undefined ? {} @@ -1201,10 +1352,11 @@ export const layer: Layer.Layer< ), ); const deliverySession = - effectiveHandoffs.length === 0 && - missedItems.length === 0 && - restartNote === "" && - !noteContinuation + contextCompaction || + (effectiveHandoffs.length === 0 && + retryHandoff.length === 0 && + restartNote === "" && + !noteContinuation) ? session : makeDeliverySession(session, startWithHandoffs); yield* runExecution.startRootRun({ @@ -1237,8 +1389,8 @@ export const layer: Layer.Layer< hasUnpairedRunInterruptRequest: runControls.hasUnpairedRunInterruptRequest, message: { messageId: message.id, - text: userText, - attachments: message.attachments, + text: contextCompaction ? "/compact" : userText, + attachments: contextCompaction ? [] : message.attachments, createdBy: message.createdBy, creationSource: message.creationSource, ...(message.scheduledTaskId === undefined diff --git a/apps/server/src/orchestration-v2/RestartContinuation.test.ts b/apps/server/src/orchestration-v2/RestartContinuation.test.ts index c282fa7a26d6..22f6d2fa3c4d 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.test.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.test.ts @@ -107,6 +107,7 @@ it("requires matching saved native state for an unfinished root run", () => { ], }, { ...projection, providerTurns: [] }, + { ...projection, attempts: [{ id: attemptId, contextCompaction: true }] }, ...[ "queued", "preparing", diff --git a/apps/server/src/orchestration-v2/RestartContinuation.ts b/apps/server/src/orchestration-v2/RestartContinuation.ts index 216b830de41e..fdcd60e39616 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.ts @@ -25,7 +25,7 @@ import { export function restartContinuationRun( projection: Pick< ProjectionRuntimeRecoveryState, - "thread" | "runs" | "providerThreads" | "providerSessions" | "providerTurns" + "thread" | "runs" | "attempts" | "providerThreads" | "providerSessions" | "providerTurns" >, cancelledWorkProviderThreadIds: ReadonlySet = new Set(), ): OrchestrationV2Run | undefined { @@ -34,7 +34,13 @@ export function restartContinuationRun( (latest, candidate) => (!latest || candidate.ordinal > latest.ordinal ? candidate : latest), undefined, ); - if (!run) return; + if ( + !run || + projection.attempts.some( + (attempt) => attempt.id === run.activeAttemptId && attempt.contextCompaction, + ) + ) + return; const preparedContinuation = run.status === "starting" && run.restartContinuationOfRunId !== undefined; // Background work outlived this settled turn; the provider has no live turn. diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 6aff5da24f1e..22787f161bd3 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -615,6 +615,70 @@ export const layer: Layer.Layer< } return; } + if ( + input.attempt.contextCompaction && + input.terminal.status === "completed" && + // A provider may finish compacting after Stop; Stop still ends the run. + !(yield* input.hasUnpairedRunInterruptRequest?.() ?? Effect.succeed(false)) + ) { + const attemptOrdinal = input.attempt.attemptOrdinal + 1; + const nextAttempt: OrchestrationV2RunAttempt = { + ...input.attempt, + id: idAllocator.derive.runAttempt({ runId: input.run.id, attemptOrdinal }), + attemptOrdinal, + contextCompaction: false, + providerTurnId: null, + reason: "provider_recovery", + status: "pending", + startedAt: null, + completedAt: null, + }; + const { delegatedCompletion: _completion, ...run } = input.run; + const events = yield* Effect.forEach( + [ + { type: "run-attempt.updated", payload: finalizedAttempt }, + { type: "run-attempt.created", payload: nextAttempt }, + { + type: "run.updated", + payload: { + ...run, + activeAttemptId: nextAttempt.id, + status: "starting", + completedAt: null, + }, + }, + ] as const, + (event) => + Effect.gen(function* () { + return { + ...event, + id: yield* idAllocator.allocate.event({ threadId: input.run.threadId }), + threadId: input.run.threadId, + runId: input.run.id, + nodeId: input.rootNode.id, + providerInstanceId: input.run.providerInstanceId, + occurredAt: completedAt, + } satisfies OrchestrationV2DomainEvent; + }), + ); + const commandId = CommandId.make(`command:context-compaction:${input.attempt.id}`); + yield* eventSink.writeIfRunCurrent({ + threadId: input.run.threadId, + runId: input.run.id, + activeAttemptId: input.attempt.id, + expectedStatus: "running", + events, + effects: [ + { + id: `effect:${commandId}:provider-turn.start`, + commandId, + threadId: input.run.threadId, + request: { type: "provider-turn.start", runId: input.run.id }, + }, + ], + }); + return; + } const allocateEventId = () => idAllocator.allocate.event({ threadId: input.run.threadId }); const open = input.openRunOwnedSubagents ?? emptyOpenRunOwnedSubagentProjection(); const hasOpenSubagentProjection = diff --git a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts index 9d01f22fe6a0..a9e04bcdb026 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts @@ -53,6 +53,7 @@ import { type ProviderAdapterV2Event, type ProviderAdapterV2HistoricalContext, ProviderAdapterProtocolError, + ProviderAdapterEnsureThreadError, type ProviderAdapterV2Shape, type ProviderAdapterV2SessionRuntime, } from "../ProviderAdapter.ts"; @@ -101,6 +102,14 @@ function makeTestAdapter(input: { readonly capturedTurns: Ref.Ref>; readonly injectedHistory?: Ref.Ref>; readonly failStartOnce?: Ref.Ref; + readonly failEnsureOnce?: Ref.Ref; + readonly compaction?: + | "success" + | "failure" + | "interrupted" + | "no-shrink" + | "unsupported" + | "no-usage"; readonly failInjectionOnce?: Ref.Ref; readonly nativeThreadGeneration?: Ref.Ref; readonly failResume?: boolean; @@ -116,6 +125,7 @@ function makeTestAdapter(input: { readonly holdRunOrdinal?: number; readonly holdFirstTurn?: Deferred.Deferred; readonly releaseFirstTurn?: Deferred.Deferred; + readonly runningWhileHeld?: boolean; }): ProviderAdapterV2Shape { return { instanceId: input.instanceId, @@ -153,6 +163,15 @@ function makeTestAdapter(input: { : { canReuseContextUsage: input.canReuseContextUsage }), ensureThread: (threadInput) => Effect.gen(function* () { + if ( + input.failEnsureOnce !== undefined && + (yield* Ref.getAndSet(input.failEnsureOnce, false)) + ) + return yield* new ProviderAdapterEnsureThreadError({ + driver: input.driver, + threadId: threadInput.threadId, + cause: "session creation failed", + }); const createdAt = yield* DateTime.now; const generation = input.nativeThreadGeneration === undefined @@ -215,7 +234,12 @@ function makeTestAdapter(input: { ), ), }), - compactThread: (turnInput) => runtime.startTurn(turnInput), + ...(input.compaction === "unsupported" + ? {} + : { + compactThread: (turnInput: Parameters[0]) => + runtime.startTurn(turnInput), + }), startTurn: (turnInput) => Effect.gen(function* () { if ( @@ -234,23 +258,49 @@ function makeTestAdapter(input: { attachments: turnInput.message.attachments, }, ]); + const eventTime = yield* DateTime.now; + const providerTurnId = ProviderTurnId.make( + `provider-turn:${input.driver}:${turnInput.threadId}:${turnInput.attemptId}`, + ); + const providerTurn = { + id: providerTurnId, + providerThreadId: turnInput.providerThread.id, + nodeId: turnInput.rootNodeId, + runAttemptId: turnInput.attemptId, + nativeTurnRef: { + driver: input.driver, + nativeId: `native-turn:${turnInput.threadId}:${turnInput.attemptId}`, + strength: "strong" as const, + }, + ordinal: turnInput.providerTurnOrdinal, + startedAt: eventTime, + }; if ( turnInput.runOrdinal === (input.holdRunOrdinal ?? 1) && input.holdFirstTurn !== undefined ) { + if (input.runningWhileHeld === true) + yield* PubSub.publish(events, { + type: "provider_turn.updated", + driver: input.driver, + providerTurn: { ...providerTurn, status: "running", completedAt: null }, + }); yield* Deferred.succeed(input.holdFirstTurn, undefined); if (input.releaseFirstTurn === undefined) return; yield* Deferred.await(input.releaseFirstTurn); } - const eventTime = yield* DateTime.now; - const providerTurnId = ProviderTurnId.make( - `provider-turn:${input.driver}:${turnInput.threadId}:${turnInput.runOrdinal}`, - ); - const terminalStatus = input.failedRunOrdinals?.has(turnInput.runOrdinal) - ? "failed" - : input.interruptedRunOrdinals?.has(turnInput.runOrdinal) - ? "interrupted" - : "completed"; + const compacting = + input.compaction !== undefined && turnInput.message.text === "/compact"; + const terminalStatus = + compacting && input.compaction === "failure" + ? "failed" + : compacting && input.compaction === "interrupted" + ? "interrupted" + : input.failedRunOrdinals?.has(turnInput.runOrdinal) + ? "failed" + : input.interruptedRunOrdinals?.has(turnInput.runOrdinal) + ? "interrupted" + : "completed"; const response = input.responseByThreadId?.[turnInput.threadId]?.[turnInput.runOrdinal] ?? input.responseByRunOrdinal[turnInput.runOrdinal] ?? @@ -259,28 +309,14 @@ function makeTestAdapter(input: { { type: "provider_turn.updated", driver: input.driver, - providerTurn: { - id: providerTurnId, - providerThreadId: turnInput.providerThread.id, - nodeId: turnInput.rootNodeId, - runAttemptId: turnInput.attemptId, - nativeTurnRef: { - driver: input.driver, - nativeId: `native-turn:${turnInput.threadId}:${turnInput.runOrdinal}`, - strength: "strong", - }, - ordinal: turnInput.runOrdinal, - status: terminalStatus, - startedAt: eventTime, - completedAt: eventTime, - }, + providerTurn: { ...providerTurn, status: terminalStatus, completedAt: eventTime }, }, { type: "turn_item.updated", driver: input.driver, turnItem: { id: TurnItemId.make( - `turn-item:${input.driver}:${turnInput.threadId}:${turnInput.runOrdinal}:assistant`, + `turn-item:${input.driver}:${turnInput.threadId}:${turnInput.attemptId}:assistant`, ), threadId: turnInput.threadId, runId: turnInput.runId, @@ -297,7 +333,7 @@ function makeTestAdapter(input: { updatedAt: eventTime, type: "assistant_message", messageId: MessageId.make( - `message:${input.driver}:${turnInput.threadId}:${turnInput.runOrdinal}:assistant`, + `message:${input.driver}:${turnInput.threadId}:${turnInput.attemptId}:assistant`, ), text: response, streaming: false, @@ -323,7 +359,15 @@ function makeTestAdapter(input: { threadDisposition: "reusable", }, ]; - const reportedUsage = input.tokenUsageByRunOrdinal?.[turnInput.runOrdinal]; + const reportedUsage = + compacting && input.compaction === "no-usage" + ? undefined + : compacting + ? { + usedTokens: input.compaction === "no-shrink" ? 207_362 : 1_000, + maxTokens: 32_000, + } + : input.tokenUsageByRunOrdinal?.[turnInput.runOrdinal]; const turnEvent = providerEvents[0]; if (reportedUsage && turnEvent?.type === "provider_turn.updated") { yield* PubSub.publish(events, { @@ -376,9 +420,27 @@ const waitForIdle = Effect.fn("ProviderSwitchTest.waitForIdle")(function* ( describe("orchestration v2 provider switching", () => { it.live.each([ "compact-native", + "first-bind-retry-native", "compact-fallback", "compact-legacy", "large-current-input", + "exhausted-native", + "exhausted-effort-change-native", + "exhausted-manual-uppercase-native", + "exhausted-missed-input-native", + "exhausted-fallback", + "exhausted-downgrade-native", + "exhausted-visible-downgrade-native", + "exhausted-oversized-native", + "exhausted-compaction-failure-native", + "exhausted-compaction-interrupted-native", + "exhausted-no-shrink-native", + "exhausted-no-usage-native", + "exhausted-stop-native", + "exhausted-stop-running-native", + "exhausted-steer-native", + "exhausted-unsupported-native", + "exhausted-unknown-capacity-native", "screenshot-native", "screenshot-fallback", "screenshot-pair-native", @@ -420,22 +482,34 @@ describe("orchestration v2 provider switching", () => { scenario.includes("model") || scenario.includes("option-change") || reasoningScenario; const failStartOnce = yield* Ref.make(false); const failResumeOnce = yield* Ref.make(false); + const failEnsureOnce = yield* Ref.make(false); const generation = yield* Ref.make(0); + const compactionStarted = yield* Deferred.make(); + const finishCompaction = yield* Deferred.make(); const priorImages = scenario.includes("prior-images"); + const exhausted = scenario.startsWith("exhausted"); const replaceNative = scenario.includes("replacement"); const capacityScenario = modelScenario || scenario.includes("reported-capacity"); const returning = - scenario === "large-current-input" || scenario.includes("-change-") || priorImages; - const targetSelection: ModelSelection = !modelScenario - ? CLAUDE_MODEL_SELECTION - : { - ...CLAUDE_MODEL_SELECTION, - ...(reasoningScenario - ? { options: [{ id: "reasoningEffort", value: "low" }] } - : scenario.includes("option-change") - ? { options: [{ id: "contextWindow", value: "1m" }] } - : { model: `${CLAUDE_MODEL_SELECTION.model}-large` }), - }; + scenario === "large-current-input" || + scenario.includes("-change-") || + priorImages || + exhausted; + const targetSelection: ModelSelection = + scenario.includes("downgrade") || scenario === "exhausted-unknown-capacity-native" + ? { ...CLAUDE_MODEL_SELECTION, model: "small-model" } + : scenario === "exhausted-effort-change-native" + ? { ...CLAUDE_MODEL_SELECTION, options: [{ id: "reasoningEffort", value: "low" }] } + : !modelScenario + ? CLAUDE_MODEL_SELECTION + : { + ...CLAUDE_MODEL_SELECTION, + ...(reasoningScenario + ? { options: [{ id: "reasoningEffort", value: "low" }] } + : scenario.includes("option-change") + ? { options: [{ id: "contextWindow", value: "1m" }] } + : { model: `${CLAUDE_MODEL_SELECTION.model}-large` }), + }; const registry = ProviderAdapterRegistry.makeLayer([ makeTestAdapter({ instanceId: CODEX_MODEL_SELECTION.instanceId, @@ -452,8 +526,49 @@ describe("orchestration v2 provider switching", () => { modelSelection: CLAUDE_MODEL_SELECTION, responseByRunOrdinal: {}, capturedTurns, - ...(scenario.includes("retry") || scenario.includes("unsent") ? { failStartOnce } : {}), - ...(replaceNative ? { failResumeOnce, nativeThreadGeneration: generation } : {}), + failEnsureOnce, + ...(scenario === "exhausted-stop-native" || + scenario === "exhausted-stop-running-native" || + scenario === "exhausted-steer-native" + ? { + holdRunOrdinal: 4, + holdFirstTurn: compactionStarted, + releaseFirstTurn: finishCompaction, + runningWhileHeld: scenario === "exhausted-stop-running-native", + } + : {}), + ...(scenario.includes("retry") || + scenario.includes("unsent") || + scenario === "exhausted-missed-input-native" + ? { failStartOnce } + : {}), + ...(replaceNative || exhausted + ? { failResumeOnce, nativeThreadGeneration: generation } + : {}), + ...(exhausted + ? { + ...(scenario === "exhausted-unknown-capacity-native" || + scenario === "exhausted-no-usage-native" + ? {} + : { + getModelContextWindow: (selection: ModelSelection) => + selection.model === "small-model" ? 32_000 : 258_400, + }), + tokenUsageByRunOrdinal: { 2: { usedTokens: 207_362, maxTokens: 258_400 } }, + compaction: + scenario === "exhausted-compaction-failure-native" + ? ("failure" as const) + : scenario === "exhausted-compaction-interrupted-native" + ? ("interrupted" as const) + : scenario === "exhausted-no-shrink-native" + ? ("no-shrink" as const) + : scenario === "exhausted-unsupported-native" + ? ("unsupported" as const) + : scenario === "exhausted-no-usage-native" + ? ("no-usage" as const) + : ("success" as const), + } + : {}), ...(scenario.includes("reported-capacity") ? { initialContextUsage: { usedTokens: 999_999, maxTokens: 1_000_000 } } : {}), @@ -522,18 +637,22 @@ describe("orchestration v2 provider switching", () => { creationSource: "web", text, attachments: - turnUsageScenario && ordinal === 2 - ? Array.from({ length: 8 }, (_, index) => ({ - ...screenshot, - id: `prior-${index}`, - })) - : turnUsageScenario && ordinal >= targetOrdinal - ? [screenshot, { ...screenshot, id: "current-2" }] - : scenario.startsWith("screenshot") && ordinal >= targetOrdinal - ? screenshots - : priorImages && ordinal === (scenario.startsWith("imported") ? 1 : 2) - ? [screenshot] - : [], + scenario === "exhausted-manual-uppercase-native" && ordinal === targetOrdinal + ? [] + : exhausted && ordinal === targetOrdinal + ? [screenshot] + : turnUsageScenario && ordinal === 2 + ? Array.from({ length: 8 }, (_, index) => ({ + ...screenshot, + id: `prior-${index}`, + })) + : turnUsageScenario && ordinal >= targetOrdinal + ? [screenshot, { ...screenshot, id: "current-2" }] + : scenario.startsWith("screenshot") && ordinal >= targetOrdinal + ? screenshots + : priorImages && ordinal === (scenario.startsWith("imported") ? 1 : 2) + ? [screenshot] + : [], modelSelection: selection, dispatchMode: { type: "start_immediately" }, }); @@ -543,7 +662,9 @@ describe("orchestration v2 provider switching", () => { ({ event }) => event.type === "run.updated" && event.payload.ordinal === ordinal && - (event.payload.status === "completed" || event.payload.status === "failed"), + (event.payload.status === "completed" || + event.payload.status === "failed" || + event.payload.status === "interrupted"), ), Stream.runHead, Effect.andThen(worker.drain()), @@ -583,20 +704,33 @@ describe("orchestration v2 provider switching", () => { ) : undefined; yield* Effect.addFinalizer(() => Effect.sync(() => spy?.mockRestore())); - const current = scenario.startsWith("compact") - ? "/compact" - : capacityScenario && !turnUsageScenario - ? "x".repeat(70_000) - : scenario === "large-current-input" - ? "x".repeat(9_000) - : "Continue work"; + const current = + scenario === "exhausted-manual-uppercase-native" + ? "/COMPACT" + : scenario === "exhausted-oversized-native" + ? "x".repeat(260_000) + : scenario === "exhausted-no-usage-native" + ? "x".repeat(110_000) + : scenario.startsWith("compact") + ? "/compact" + : capacityScenario && !turnUsageScenario + ? "x".repeat(70_000) + : scenario === "large-current-input" + ? "x".repeat(9_000) + : "Continue work"; // First establish the returning native thread: the current request must // not be charged as existing context on the subsequent handoff. if (returning) { if (scenario.includes("unsent")) yield* Ref.set(failStartOnce, true); yield* dispatch( 2, - turnUsageScenario ? "x".repeat(27_460) : "Establish target", + exhausted + ? scenario === "exhausted-downgrade-native" + ? "Establish target" + : "x".repeat(70_000) + : turnUsageScenario + ? "x".repeat(27_460) + : "Establish target", CLAUDE_MODEL_SELECTION, ); yield* wait(2); @@ -657,12 +791,18 @@ describe("orchestration v2 provider switching", () => { ], }); } + if (scenario === "exhausted-missed-input-native") yield* Ref.set(failStartOnce, true); yield* dispatch( 3, priorImages && !replaceNative ? "New source constraint " + "q".repeat(9_000) : "New source constraint", - CODEX_MODEL_SELECTION, + scenario.includes("downgrade") || + scenario === "exhausted-effort-change-native" || + scenario === "exhausted-unknown-capacity-native" || + scenario === "exhausted-missed-input-native" + ? CLAUDE_MODEL_SELECTION + : CODEX_MODEL_SELECTION, ); yield* wait(3); } @@ -680,7 +820,52 @@ describe("orchestration v2 provider switching", () => { yield* Ref.set(failResumeOnce, true); } if (scenario.includes("retry")) yield* Ref.set(failStartOnce, true); + if (scenario === "first-bind-retry-native") yield* Ref.set(failEnsureOnce, true); yield* dispatch(targetOrdinal, current, targetSelection); + if ( + scenario === "exhausted-stop-native" || + scenario === "exhausted-stop-running-native" || + scenario === "exhausted-steer-native" + ) { + yield* Deferred.await(compactionStarted); + const active = (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)!; + // Stop while the provider is compacting, not before its turn starts. + if (scenario === "exhausted-stop-running-native") + yield* orchestrator.streamStoredEvents.pipe( + Stream.filter( + ({ event }) => + event.type === "provider-turn.updated" && + event.payload.runAttemptId === active.activeAttemptId && + event.payload.status === "running", + ), + Stream.runHead, + ); + if (scenario !== "exhausted-steer-native") { + yield* orchestrator.dispatch({ + type: "run.interrupt", + commandId: CommandId.make("stop-compaction"), + threadId, + runId: active.id, + }); + } else { + const steered = yield* Effect.result( + orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("steer-compaction"), + threadId, + messageId: MessageId.make("steer-compaction"), + createdBy: "user", + creationSource: "web", + text: "Change the plan", + attachments: [], + modelSelection: targetSelection, + dispatchMode: { type: "steer_active", targetRunId: active.id }, + }), + ); + assert.equal(steered._tag, "Failure"); + } + yield* Deferred.succeed(finishCompaction, undefined); + } yield* wait(targetOrdinal); if (scenario.includes("retry")) { const failed = yield* orchestrator.getThreadProjection(threadId); @@ -688,7 +873,9 @@ describe("orchestration v2 provider switching", () => { const failedUsage = failed.providerThreads.find( (thread) => thread.providerInstanceId === CLAUDE_MODEL_SELECTION.instanceId, )!.contextUsage; - if (reasoningScenario || scenario.includes("turn-usage")) { + if (scenario === "first-bind-retry-native") { + assert.isNull(failedUsage); + } else if (reasoningScenario || scenario.includes("turn-usage")) { assert.deepEqual(failedUsage, { usedTokens: 37_321, maxTokens: 258_400 }); } else { assert.deepEqual(failedUsage, { usedTokens: 30_000, maxTokens: 1_000_000 }); @@ -765,6 +952,91 @@ describe("orchestration v2 provider switching", () => { yield* wait(targetOrdinal + 3); } const projection = yield* orchestrator.getThreadProjection(threadId); + if (scenario === "exhausted-manual-uppercase-native") { + assert.equal(projection.runs.at(-1)?.status, "completed"); + assert.equal( + (yield* Ref.get(capturedTurns)).filter((turn) => turn.text === "/compact").length, + 0, + ); + assert.equal((yield* Ref.get(capturedTurns)).at(-1)?.text, "/COMPACT"); + return; + } + if (exhausted) { + const failed = + scenario === "exhausted-oversized-native" || + scenario === "exhausted-compaction-failure-native" || + scenario === "exhausted-no-shrink-native" || + scenario === "exhausted-unsupported-native"; + const interrupted = + scenario === "exhausted-compaction-interrupted-native" || + scenario === "exhausted-stop-native"; + // The provider finished compacting after Stop, so the run completes unsent. + const stoppedAfterCompaction = scenario === "exhausted-stop-running-native"; + assert.equal( + projection.runs.at(-1)?.status, + interrupted ? "interrupted" : failed ? "failed" : "completed", + ); + assert.equal(yield* Ref.get(generation), 1); + const compactTurns = (yield* Ref.get(capturedTurns)).filter( + (turn) => turn.text === "/compact", + ); + assert.equal( + compactTurns.length, + scenario === "exhausted-oversized-native" || + scenario === "exhausted-effort-change-native" || + scenario === "exhausted-unknown-capacity-native" || + scenario === "exhausted-unsupported-native" + ? 0 + : 1, + ); + assert.isTrue(compactTurns.every((turn) => turn.attachments.length === 0)); + assert.isTrue( + projection.turnItems.some( + (item) => + item.type === "user_message" && + item.text === + (scenario === "exhausted-downgrade-native" + ? "Establish target" + : "x".repeat(70_000)), + ), + ); + if (failed || interrupted || stoppedAfterCompaction) { + assert.equal( + (yield* Ref.get(capturedTurns)).filter((turn) => turn.text.endsWith(current)) + .length, + 0, + ); + return; + } + if ( + scenario === "exhausted-unknown-capacity-native" || + scenario === "exhausted-effort-change-native" + ) + return; + const latestAttempt = projection.attempts.find( + (attempt) => attempt.id === projection.runs.at(-1)!.activeAttemptId, + )!; + assert.include(latestAttempt.nativeThreadId!, ":0"); + assert.deepEqual((yield* Ref.get(capturedTurns)).at(-1)!.attachments, [screenshot]); + assert.equal( + (yield* Ref.get(capturedTurns)).filter((turn) => turn.text.endsWith(current)).length, + 1, + ); + const history = scenario.endsWith("-native") + ? yield* encodeJson(yield* Ref.get(injectedHistory)) + : (yield* Ref.get(capturedTurns)).at(-1)!.text; + if (!scenario.includes("downgrade")) assert.include(history, "New source constraint"); + yield* dispatch(5, "Further source work", CODEX_MODEL_SELECTION); + yield* wait(5); + yield* dispatch(6, "Return to the recovered session", targetSelection); + yield* wait(6); + assert.equal( + (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)?.status, + "completed", + ); + assert.equal(yield* Ref.get(generation), 1); + return; + } assert.equal(projection.runs.at(-1)?.status, "completed"); if (priorImages) { const handoff = projection.contextHandoffs.at(-1)!; diff --git a/docs/user/portable-handoffs.md b/docs/user/portable-handoffs.md index e2d84458ed93..5077655ca896 100644 --- a/docs/user/portable-handoffs.md +++ b/docs/user/portable-handoffs.md @@ -20,9 +20,14 @@ agent-written summary. ## Context limits A handoff must leave room for existing provider context, your request and attachments, instructions, -tools, and subsequent work. If even its retrieval references cannot fit, T3 Code reports an error -instead of shortening your request. Compact the target conversation or select a larger-context -model before trying again. +tools, and subsequent work. If the target conversation leaves too little room, T3 Code runs native +compaction in that same conversation, then retries your request once. Your saved chat and original +request stay intact. This also applies to an overfull model switch within one provider when the +new model capacity is known. No encrypted state is copied between providers. + +If compaction is unsupported, fails, is stopped, or still leaves too little room, T3 Code reports +the failure without silently replacing the session or shortening your request. Reduce the request +or select a larger-context model before trying again. Server operators can set `T3CODE_CONTEXT_HANDOFF_TOKEN_CAP` to change the initial history allowance (default 16,000; clamped to 1,024–64,000). This is an upper bound, not a provider context-window @@ -40,5 +45,10 @@ least 16,000 tokens, or a quarter of the window when larger, for instructions, t work. Images reserve an estimated 8,192 tokens each, independent of their file size; other attachments reserve 4,096 each for their references. Existing context is estimated from saved activity when usage telemetry is unavailable. Known smaller windows still constrain the handoff. -These are fallback estimates, not exact token counts. Image resolution, custom models, and hidden -native context can differ, so the provider may still reject an input. +These are fallback estimates, not exact token counts. Switching models within one provider does not +trigger automatic compaction solely because the new model's capacity is unknown. Image resolution, +custom models, and hidden native context can differ, so the provider may still reject an input. + +After automatic compaction, T3 Code uses fresh usage reports when available. Otherwise it estimates +new activity and keeps the history allowance bounded; the provider still decides whether the retry +fits. Old saved messages are not counted as unchanged native context after compaction. diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index 6004ab119917..e76b3500bcb4 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -564,6 +564,7 @@ export function orchestrationV2RunWorkStartedAt( export const OrchestrationV2RunAttempt = Schema.Struct({ id: RunAttemptId, + contextCompaction: Schema.optional(Schema.Boolean), // Provider-thread rows can be reused after recovery; retain the native input destination. nativeThreadId: Schema.optional(Schema.String), runId: RunId, From 5cdf03cc2fa11acff1e40fd25d924bf972a4af71 Mon Sep 17 00:00:00 2001 From: Bil0000 <62337003+Bil0000@users.noreply.github.com> Date: Sun, 4 Oct 2026 15:01:41 +0200 Subject: [PATCH 02/10] fix(v2): guard context recovery across model switches --- apps/server/src/orchestration-v2/EventSink.ts | 12 + .../FoundationPersistence.test.ts | 232 ++++++++++-------- .../orchestration-v2/ProviderFailure.test.ts | 40 +++ .../src/orchestration-v2/ProviderFailure.ts | 36 ++- .../ProviderTurnStartService.ts | 23 +- .../orchestration-v2/RunExecutionService.ts | 161 ++++++++---- .../testkit/ProviderReplayHarness.ts | 11 +- .../ProviderSwitch.integration.test.ts | 199 +++++++++++++-- docs/user/portable-handoffs.md | 16 +- 9 files changed, 557 insertions(+), 173 deletions(-) diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index ca20e8f7217d..342446ab3011 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -10,6 +10,7 @@ import { NodeId, type ProjectId, ThreadId, + TurnItemId, } from "@t3tools/contracts"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; @@ -91,6 +92,7 @@ export interface EventSinkV2Shape { readonly expectedStatus: OrchestrationV2Run["status"]; readonly events: ReadonlyArray; readonly effects?: ReadonlyArray; + readonly rejectInterruptRequestId?: TurnItemId; }) => Effect.Effect< { readonly committed: boolean; @@ -417,7 +419,17 @@ const baseLayer: Layer.Layer< LIMIT 1 `; const current = rows[0]; + const interrupted = + input.rejectInterruptRequestId === undefined + ? false + : (yield* sql` + SELECT 1 FROM orchestration_v2_projection_turn_items + WHERE thread_id = ${input.threadId} + AND turn_item_id = ${input.rejectInterruptRequestId} + LIMIT 1 + `).length > 0; if ( + interrupted || current === undefined || current.status !== input.expectedStatus || current.active_attempt_id !== input.activeAttemptId diff --git a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts index 6e5d740e39ad..110a7b08b2e8 100644 --- a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts +++ b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts @@ -1509,120 +1509,154 @@ it.layer(TestLayer)("orchestration V2 foundation persistence", (it) => { }).pipe(Effect.provide(Layer.fresh(TestLayer))), ); - it.effect("does not publish a stale provider start after an interrupt wins", () => - Effect.gen(function* () { - const eventSink = yield* EventSink.EventSinkV2; - const projectionStore = yield* ProjectionStore.ProjectionStoreV2; - const outbox = yield* EffectOutbox.EffectOutboxV2; - const commandId = CommandId.make("guarded-continuation"); - const now = yield* DateTime.now; - const threadId = ThreadId.make("thread:foundation-stale-provider-start"); - const runId = RunId.make("run:foundation-stale-provider-start"); - const attemptId = RunAttemptId.make("run-attempt:foundation-stale-provider-start"); - const thread = makeThread(threadId, now); - const startingRun: OrchestrationV2Run = { - id: runId, - threadId, - ordinal: 1, - providerInstanceId, - modelSelection, - providerThreadId: null, - userMessageId: MessageId.make("message:foundation-stale-provider-start"), - rootNodeId: null, - activeAttemptId: attemptId, - status: "starting", - queuePosition: null, - requestedAt: now, - startedAt: null, - completedAt: null, - checkpointId: null, - contextHandoffId: null, - }; - yield* eventSink.write({ - events: [ - threadCreatedEvent({ - id: "event:foundation-stale-provider-start:thread", - thread, - now, - }), - { - id: EventId.make("event:foundation-stale-provider-start:run"), - type: "run.created", - threadId, - runId, - providerInstanceId, - occurredAt: now, - payload: startingRun, - }, - ], - }); - - const reachedPrecommitGap = yield* Deferred.make(); - const releaseStaleStart = yield* Deferred.make(); - const providerStartCount = yield* Ref.make(0); - const staleStartFiber = yield* Effect.gen(function* () { - yield* Deferred.succeed(reachedPrecommitGap, undefined); - yield* Deferred.await(releaseStaleStart); - const result = yield* eventSink.writeIfRunCurrent({ + it.effect.each(["cancelled", "stop-request"] as const)( + "does not publish a stale provider start after %s", + (mode) => + Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const outbox = yield* EffectOutbox.EffectOutboxV2; + const commandId = CommandId.make(`guarded-continuation:${mode}`); + const now = yield* DateTime.now; + const threadId = ThreadId.make(`thread:foundation-stale-provider-start:${mode}`); + const runId = RunId.make(`run:foundation-stale-provider-start:${mode}`); + const attemptId = RunAttemptId.make(`run-attempt:foundation-stale-provider-start:${mode}`); + const interruptRequestId = TurnItemId.make(`turn-item:foundation-stop-request:${mode}`); + const thread = makeThread(threadId, now); + const startingRun: OrchestrationV2Run = { + id: runId, threadId, - runId, + ordinal: 1, + providerInstanceId, + modelSelection, + providerThreadId: null, + userMessageId: MessageId.make(`message:foundation-stale-provider-start:${mode}`), + rootNodeId: null, activeAttemptId: attemptId, - expectedStatus: "starting", - effects: [ - { - id: "guarded-continuation", - commandId, - threadId, - request: { type: "provider-turn.start", runId }, - }, - ], + status: "starting", + queuePosition: null, + requestedAt: now, + startedAt: null, + completedAt: null, + checkpointId: null, + contextHandoffId: null, + }; + yield* eventSink.write({ events: [ + threadCreatedEvent({ + id: `event:foundation-stale-provider-start:thread:${mode}`, + thread, + now, + }), { - id: EventId.make("event:foundation-stale-provider-start:running"), - type: "run.updated", + id: EventId.make(`event:foundation-stale-provider-start:run:${mode}`), + type: "run.created", threadId, runId, providerInstanceId, occurredAt: now, - payload: { ...startingRun, status: "running", startedAt: now }, + payload: startingRun, }, ], }); - if (result.committed) { - yield* Ref.update(providerStartCount, (count) => count + 1); - } - return result; - }).pipe(Effect.forkChild); - yield* Deferred.await(reachedPrecommitGap); - const interruptedAt = yield* DateTime.now; - yield* eventSink.write({ - events: [ - { - id: EventId.make("event:foundation-stale-provider-start:cancelled"), - type: "run.updated", + const reachedPrecommitGap = yield* Deferred.make(); + const releaseStaleStart = yield* Deferred.make(); + const providerStartCount = yield* Ref.make(0); + const staleStartFiber = yield* Effect.gen(function* () { + yield* Deferred.succeed(reachedPrecommitGap, undefined); + yield* Deferred.await(releaseStaleStart); + const result = yield* eventSink.writeIfRunCurrent({ threadId, runId, - providerInstanceId, - occurredAt: interruptedAt, - payload: { - ...startingRun, - status: "cancelled", - completedAt: interruptedAt, - }, - }, - ], - }); - yield* Deferred.succeed(releaseStaleStart, undefined); + activeAttemptId: attemptId, + expectedStatus: "starting", + rejectInterruptRequestId: interruptRequestId, + effects: [ + { + id: `guarded-continuation:${mode}`, + commandId, + threadId, + request: { type: "provider-turn.start", runId }, + }, + ], + events: [ + { + id: EventId.make(`event:foundation-stale-provider-start:running:${mode}`), + type: "run.updated", + threadId, + runId, + providerInstanceId, + occurredAt: now, + payload: { ...startingRun, status: "running", startedAt: now }, + }, + ], + }); + if (result.committed) { + yield* Ref.update(providerStartCount, (count) => count + 1); + } + return result; + }).pipe(Effect.forkChild); + + yield* Deferred.await(reachedPrecommitGap); + const interruptedAt = yield* DateTime.now; + yield* eventSink.write({ + events: + mode === "cancelled" + ? [ + { + id: EventId.make(`event:foundation-stale-provider-start:cancelled:${mode}`), + type: "run.updated", + threadId, + runId, + providerInstanceId, + occurredAt: interruptedAt, + payload: { + ...startingRun, + status: "cancelled", + completedAt: interruptedAt, + }, + }, + ] + : [ + { + id: EventId.make(`event:foundation-stale-provider-start:stop-request:${mode}`), + type: "turn-item.updated", + threadId, + runId, + providerInstanceId, + occurredAt: interruptedAt, + payload: { + id: interruptRequestId, + threadId, + runId, + nodeId: null, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 1, + type: "run_interrupt_request", + status: "completed", + title: "Interrupt requested", + message: "Interrupt requested", + startedAt: interruptedAt, + completedAt: interruptedAt, + updatedAt: interruptedAt, + }, + }, + ], + }); + yield* Deferred.succeed(releaseStaleStart, undefined); - const staleResult = yield* Fiber.join(staleStartFiber); - assert.isFalse(staleResult.committed); - assert.deepEqual(yield* outbox.listByCommandId(commandId), []); - assert.deepEqual(staleResult.storedEvents, []); - assert.equal(yield* Ref.get(providerStartCount), 0); - const projection = yield* projectionStore.getThreadProjection(threadId); - assert.equal(projection.runs[0]?.status, "cancelled"); - }), + const staleResult = yield* Fiber.join(staleStartFiber); + assert.isFalse(staleResult.committed); + assert.deepEqual(yield* outbox.listByCommandId(commandId), []); + assert.deepEqual(staleResult.storedEvents, []); + assert.equal(yield* Ref.get(providerStartCount), 0); + const projection = yield* projectionStore.getThreadProjection(threadId); + assert.equal(projection.runs[0]?.status, mode === "cancelled" ? "cancelled" : "starting"); + }).pipe(Effect.provide(Layer.fresh(TestLayer))), ); it.effect("guards post-terminal provider-thread writes by attempt and run ordinal", () => diff --git a/apps/server/src/orchestration-v2/ProviderFailure.test.ts b/apps/server/src/orchestration-v2/ProviderFailure.test.ts index f05d3e91cd8a..de8b0cfcf647 100644 --- a/apps/server/src/orchestration-v2/ProviderFailure.test.ts +++ b/apps/server/src/orchestration-v2/ProviderFailure.test.ts @@ -13,6 +13,7 @@ import * as Effect from "effect/Effect"; import { makeProviderFailure, + isContextWindowFailure, makeProviderFailureTurnItem, MAX_PROVIDER_FAILURE_CODE_LENGTH, MAX_PROVIDER_FAILURE_MESSAGE_LENGTH, @@ -21,6 +22,45 @@ import * as IdAllocator from "./IdAllocator.ts"; import { ContextHandoffBudgetError } from "./ContextHandoffDelivery.ts"; import { ProviderAdapterTurnStartError } from "./ProviderAdapter.ts"; +it.each([ + ["contextWindowExceeded", "Input rejected", true], + ["context_length_exceeded", "Input rejected", true], + ["ContextOverflowError", "Input rejected", true], + ["prompt_too_long", "Input rejected", true], + [null, "Prompt is too long", true], + [null, "Model context window exceeded", true], + [null, "This model's maximum context length is 32000 tokens", true], + [null, "Input token count 40000 exceeds the maximum of 32000", true], + [null, "The request exceeds the model's context window.", true], + ["rateLimitExceeded", "Too many requests", false], + [null, "Failed to load context", false], + [null, "Attachment is too large", false], +] as const)("classifies context rejection %s / %s", (code, message, expected) => { + const failure = makeProviderFailure({ code, message, class: "provider_error" }); + assert.equal(isContextWindowFailure(failure), expected); + assert.isFalse(isContextWindowFailure({ ...failure, class: "permission_error" })); + assert.isFalse(isContextWindowFailure({ ...failure, class: "usage_limit" })); +}); + +it("recognizes nested native context rejections without exposing their cause text", () => { + for (const cause of [ + "Prompt is too long api_key=secret", + { errorMessage: "Prompt is too long api_key=secret" }, + new Error("Prompt is too long api_key=secret"), + new ProviderAdapterTurnStartError({ + driver: ProviderDriverKind.make("codex"), + threadId: ThreadId.make("context-rejection"), + providerThreadId: ProviderThreadId.make("context-rejection"), + runId: RunId.make("context-rejection"), + cause: { code: "contextWindowExceeded" }, + }), + ]) { + const failure = makeProviderFailure({ cause, class: "provider_error" }); + assert.isTrue(isContextWindowFailure(failure)); + assert.notInclude(failure.message, "secret"); + } +}); + it("redacts credentials and URL secrets from provider failures", () => { const failure = makeProviderFailure({ message: diff --git a/apps/server/src/orchestration-v2/ProviderFailure.ts b/apps/server/src/orchestration-v2/ProviderFailure.ts index 6cb9275017d9..73519fd052f7 100644 --- a/apps/server/src/orchestration-v2/ProviderFailure.ts +++ b/apps/server/src/orchestration-v2/ProviderFailure.ts @@ -21,6 +21,24 @@ export const MAX_PROVIDER_FAILURE_CODE_LENGTH = 128; const DEFAULT_PROVIDER_FAILURE_MESSAGE = "Provider turn failed."; +export function isContextWindowFailure( + failure: Pick, +): boolean { + return ( + (failure.class === "provider_error" || failure.class === "validation_error") && + ([ + "contextWindowExceeded", + "context_window_exceeded", + "context_length_exceeded", + "ContextOverflowError", + "prompt_too_long", + ].includes(failure.code ?? "") || + /\b(?:prompt is too long|context (?:window|length) (?:has been |is )?exceeded|exceeds? (?:(?:the|this) )?(?:model's )?(?:maximum )?context (?:window|length)|maximum context length is \d+|input token count.{0,80}exceeds.{0,80}maximum)\b/iu.test( + failure.message, + )) + ); +} + /** Translate known categories without exposing arbitrary provider defect text. */ function causeMessage(cause: unknown): string | undefined { const seen = new Set(); @@ -32,7 +50,20 @@ function causeMessage(cause: unknown): string | undefined { cause = Cause.squash(cause); continue; } + if ( + typeof cause === "string" && + isContextWindowFailure({ class: "provider_error", code: null, message: cause }) + ) + return "The request exceeds the model's context window."; if (typeof cause !== "object") break; + if ( + isContextWindowFailure({ + class: "provider_error", + code: stringField(cause, "code") ?? stringField(cause, "name") ?? null, + message: stringField(cause, "message") ?? stringField(cause, "errorMessage") ?? "", + }) + ) + return "The request exceeds the model's context window."; switch ((cause as Record)._tag) { case "ContextHandoffBudgetError": return new ContextHandoffBudgetError().message; @@ -65,7 +96,10 @@ function causeMessage(cause: unknown): string | undefined { return message; } -function stringField(value: unknown, key: "message" | "code"): string | undefined { +function stringField( + value: unknown, + key: "message" | "code" | "name" | "errorMessage", +): string | undefined { if (typeof value !== "object" || value === null) return undefined; try { const candidate = (value as Record)[key]; diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 54b54f271b3d..c49b250e375a 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -153,6 +153,17 @@ export const layer: Layer.Layer< Effect.catchCause(() => Effect.succeed(input.inheritedBackgroundTurnItems)), ), shouldStartProviderTurn: () => isCurrentAttemptInStatus("running"), + canRetryContextFailure: () => + projectionStore + .getRuntimeRecoveryProjection(input.threadId) + .pipe( + Effect.map( + (current) => + !current.attempts.some( + (candidate) => candidate.runId === input.runId && candidate.contextCompaction, + ), + ), + ), shouldFinalizeRun: () => projectionStore.getRuntimeRecoveryProjection(input.threadId).pipe( Effect.map((current) => { @@ -902,7 +913,7 @@ export const layer: Layer.Layer< }, ); let resumedContext: Effect.Success> | undefined; - let contextCompaction = false; + let contextCompaction = attempt.contextCompaction === true; const loadedProviderThread = yield* Effect.gen(function* () { if (nativeForkTransfer !== undefined) { const sourceProjection = yield* projectionStore.getThreadRecords( @@ -982,6 +993,7 @@ export const layer: Layer.Layer< }), ); if (resumed._tag === "Success") { + if (contextCompaction) return resumed.success; if (message.text.trim().toLowerCase() === "/compact" && message.attachments.length === 0) return resumed.success; const context = yield* getHandoffContext(resumed.success); @@ -1026,6 +1038,14 @@ export const layer: Layer.Layer< } if (!(yield* isCurrentAttemptInStatus("starting"))) return providerThread; + if (contextCompaction) { + yield* settleStartFailure({ + signal: "provider-thread-load-failure", + title: "Provider turn failed to start", + error: resumed.failure, + }); + return undefined; + } yield* Effect.logWarning("Provider resume failed; attempting a fresh native session", { driver: session.driver, providerThreadId: providerThread.id, @@ -1386,6 +1406,7 @@ export const layer: Layer.Layer< shouldStartProviderTurn: runControls.shouldStartProviderTurn, shouldFinalizeRun: runControls.shouldFinalizeRun, hasUnpairedRunInterruptRequest: runControls.hasUnpairedRunInterruptRequest, + canRetryContextFailure: runControls.canRetryContextFailure, message: { messageId: message.id, text: contextCompaction ? "/compact" : userText, diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 22787f161bd3..8449af635d43 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -49,7 +49,11 @@ import type { import { ProviderAdapterTurnStartError } from "./ProviderAdapter.ts"; import * as ProviderEventIngestor from "./ProviderEventIngestor.ts"; import type { ProjectionStoreV2Error } from "./ProjectionStore.ts"; -import { makeProviderFailure, makeProviderFailureTurnItem } from "./ProviderFailure.ts"; +import { + isContextWindowFailure, + makeProviderFailure, + makeProviderFailureTurnItem, +} from "./ProviderFailure.ts"; import * as RunFinalizationService from "./RunFinalizationService.ts"; export interface ProviderEventRoutingState { @@ -516,6 +520,7 @@ export interface RunExecutionServiceV2StartRootRunInput { readonly shouldStartProviderTurn?: () => Effect.Effect; readonly shouldFinalizeRun?: () => Effect.Effect; readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect; + readonly canRetryContextFailure?: () => Effect.Effect; readonly message: ProviderAdapterV2TurnMessage; readonly modelSelection: ModelSelection; readonly runtimePolicy: ProviderAdapterV2RuntimePolicy; @@ -561,6 +566,7 @@ export const layer: Layer.Layer< readonly attempt: OrchestrationV2RunAttempt; readonly shouldFinalizeRun?: () => Effect.Effect; readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect; + readonly canRetryContextFailure?: () => Effect.Effect; readonly openRunOwnedSubagents?: OpenRunOwnedSubagentProjection; readonly terminal: ProviderTerminalEvent; readonly failureItemPersisted: boolean; @@ -572,18 +578,21 @@ export const layer: Layer.Layer< }) => Effect.gen(function* () { const completedAt = yield* DateTime.now; - const finalizedAttempt: OrchestrationV2RunAttempt | null = { - ...input.attempt, - status: input.terminal.status, - completedAt, - }; + const contextRejected = + input.terminal.status === "failed" && isContextWindowFailure(input.terminal.failure); + const stopped = + (input.attempt.contextCompaction || contextRejected) && + (yield* input.hasUnpairedRunInterruptRequest?.() ?? Effect.succeed(false)); + let terminal: ProviderTerminalEvent = stopped + ? { ...input.terminal, status: "interrupted", failure: null } + : input.terminal; const shouldFinalizeRun = input.shouldFinalizeRun === undefined ? true : yield* input.shouldFinalizeRun(); if (!shouldFinalizeRun) { // Superseded attempt (steer / selection restart). Emit // run_interrupt_result only when hard Stop left an unpaired request // for this run; plain steers and already-paired stops emit nothing. - if (input.terminal.status === "interrupted") { + if (terminal.status === "interrupted") { const hasUnpairedRequest = input.hasUnpairedRunInterruptRequest === undefined ? false @@ -615,18 +624,21 @@ export const layer: Layer.Layer< } return; } + const retryContextFailure = + !stopped && + contextRejected && + !input.attempt.contextCompaction && + (yield* input.canRetryContextFailure?.() ?? Effect.succeed(false)); if ( - input.attempt.contextCompaction && - input.terminal.status === "completed" && - // A provider may finish compacting after Stop; Stop still ends the run. - !(yield* input.hasUnpairedRunInterruptRequest?.() ?? Effect.succeed(false)) + (input.attempt.contextCompaction && terminal.status === "completed") || + retryContextFailure ) { const attemptOrdinal = input.attempt.attemptOrdinal + 1; const nextAttempt: OrchestrationV2RunAttempt = { ...input.attempt, id: idAllocator.derive.runAttempt({ runId: input.run.id, attemptOrdinal }), attemptOrdinal, - contextCompaction: false, + contextCompaction: retryContextFailure, providerTurnId: null, reason: "provider_recovery", status: "pending", @@ -636,7 +648,10 @@ export const layer: Layer.Layer< const { delegatedCompletion: _completion, ...run } = input.run; const events = yield* Effect.forEach( [ - { type: "run-attempt.updated", payload: finalizedAttempt }, + { + type: "run-attempt.updated", + payload: { ...input.attempt, status: terminal.status, completedAt }, + }, { type: "run-attempt.created", payload: nextAttempt }, { type: "run.updated", @@ -662,11 +677,15 @@ export const layer: Layer.Layer< }), ); const commandId = CommandId.make(`command:context-compaction:${input.attempt.id}`); - yield* eventSink.writeIfRunCurrent({ + const continuation = yield* eventSink.writeIfRunCurrent({ threadId: input.run.threadId, runId: input.run.id, activeAttemptId: input.attempt.id, expectedStatus: "running", + rejectInterruptRequestId: idAllocator.derive.runSignalTurnItem({ + runId: input.run.id, + signal: "interrupt-request", + }), events, effects: [ { @@ -677,8 +696,15 @@ export const layer: Layer.Layer< }, ], }); - return; + if (continuation.committed) return; + if (!(yield* input.shouldFinalizeRun?.() ?? Effect.succeed(true))) return; + terminal = { ...terminal, status: "interrupted", failure: null }; } + const finalizedAttempt: OrchestrationV2RunAttempt = { + ...input.attempt, + status: terminal.status, + completedAt, + }; const allocateEventId = () => idAllocator.allocate.event({ threadId: input.run.threadId }); const open = input.openRunOwnedSubagents ?? emptyOpenRunOwnedSubagentProjection(); const hasOpenSubagentProjection = @@ -687,17 +713,16 @@ export const layer: Layer.Layer< open.childTurnItems.size > 0 || open.nodes.size > 0; const cascadedSubagentEvents = - isRunOwnedSubagentTerminalStatus(input.terminal.status) && hasOpenSubagentProjection + isRunOwnedSubagentTerminalStatus(terminal.status) && hasOpenSubagentProjection ? yield* cascadeTerminalizeRunOwnedSubagents({ run: input.run, open, - status: input.terminal.status, + status: terminal.status, completedAt, allocateEventId, }) : []; - const persistedStatus = - input.terminal.status === "completed" ? "waiting" : input.terminal.status; + const persistedStatus = terminal.status === "completed" ? "waiting" : terminal.status; // Completion cohorts are advanced by Orchestrator while a provider // turn is in flight. Do not replay the run snapshot captured at start // over a newer acknowledgement, successor, or Stop barrier. @@ -706,17 +731,17 @@ export const layer: Layer.Layer< const finalizedRun: OrchestrationV2Run = { ...runWithoutDelegatedCompletion, status: persistedStatus, - completedAt: input.terminal.status === "completed" ? null : completedAt, + completedAt: terminal.status === "completed" ? null : completedAt, }; const finalizedRootNode: OrchestrationV2ExecutionNode = { ...input.rootNode, status: persistedStatus, - completedAt: input.terminal.status === "completed" ? null : completedAt, + completedAt: terminal.status === "completed" ? null : completedAt, checkpointScopeId: input.checkpointScope.id, }; const finalizedProviderThread: OrchestrationV2ProviderThread = { ...input.providerThread, - status: finalProviderThreadStatus(input.terminal.threadDisposition), + status: finalProviderThreadStatus(terminal.threadDisposition), updatedAt: completedAt, }; const runEventId = yield* allocateEventId(); @@ -730,9 +755,9 @@ export const layer: Layer.Layer< // ahead of any later run's start on this thread's effect lane. const finalization = { effects: - input.terminal.status === "completed" || - input.terminal.status === "interrupted" || - input.terminal.status === "cancelled" + terminal.status === "completed" || + terminal.status === "interrupted" || + terminal.status === "cancelled" ? [ { id: `effect:checkpoint.capture:${input.run.id}`, @@ -750,21 +775,17 @@ export const layer: Layer.Layer< // Terminalize open run-owned subagent rows before the root run // settles so projections never keep a forever-running subagent card. ...cascadedSubagentEvents, - ...(finalizedAttempt === null - ? [] - : [ - { - id: yield* allocateEventId(), - type: "run-attempt.updated" as const, - threadId: input.run.threadId, - runId: input.run.id, - nodeId: input.rootNode.id, - providerInstanceId: input.run.providerInstanceId, - occurredAt: completedAt, - payload: finalizedAttempt, - }, - ]), - ...(input.terminal.status === "interrupted" + { + id: yield* allocateEventId(), + type: "run-attempt.updated" as const, + threadId: input.run.threadId, + runId: input.run.id, + nodeId: input.rootNode.id, + providerInstanceId: input.run.providerInstanceId, + occurredAt: completedAt, + payload: finalizedAttempt, + }, + ...(terminal.status === "interrupted" ? [ { id: yield* allocateEventId(), @@ -784,7 +805,7 @@ export const layer: Layer.Layer< }, ] : []), - ...(input.terminal.status === "failed" && !input.failureItemPersisted + ...(terminal.status === "failed" && !input.failureItemPersisted ? [ { id: yield* allocateEventId(), @@ -796,14 +817,14 @@ export const layer: Layer.Layer< occurredAt: completedAt, payload: makeProviderFailureTurnItem({ idAllocator, - driver: input.terminal.driver, + driver: terminal.driver, threadId: input.run.threadId, runId: input.run.id, nodeId: input.rootNode.id, - providerThreadId: input.terminal.providerThreadId, - providerTurnId: input.terminal.providerTurnId, - itemOrdinal: input.terminal.failureItemOrdinal, - failure: input.terminal.failure, + providerThreadId: terminal.providerThreadId, + providerTurnId: terminal.providerTurnId, + itemOrdinal: terminal.failureItemOrdinal, + failure: terminal.failure, occurredAt: completedAt, }), }, @@ -1010,6 +1031,18 @@ export const layer: Layer.Layer< : { relatedProviderThreadIds: input.relatedProviderThreadIds }), }), ); + const providerWorkObserved = yield* Ref.make(false); + const canRetryContextFailure = () => + Ref.get(providerWorkObserved).pipe( + Effect.flatMap((worked) => + worked || + input.session.compactThread === undefined || + (input.message.text.trim().toLowerCase() === "/compact" && + input.message.attachments.length === 0) + ? Effect.succeed(false) + : (input.canRetryContextFailure?.() ?? Effect.succeed(false)), + ), + ); const rootTerminalSeen = yield* Ref.make(false); const rootRunFinalized = yield* Ref.make(false); const providerThreadOwnerLost = yield* Ref.make(false); @@ -1041,6 +1074,7 @@ export const layer: Layer.Layer< hasUnpairedRunInterruptRequest: input.hasUnpairedRunInterruptRequest, }), openRunOwnedSubagents: openSubagents, + canRetryContextFailure, terminal, failureItemPersisted: terminal.status === "failed", refreshAfterTurn, @@ -1245,6 +1279,27 @@ export const layer: Layer.Layer< DateTime.toEpochMillis(yield* DateTime.now), ); if (deliveredEvent) { + if ( + event.type === "subagent.updated" || + (event.type === "node.updated" && + (event.node.kind === "tool_call" || event.node.kind === "subagent")) || + event.type === "runtime_request.updated" || + (event.type === "turn_item.updated" && + event.turnItem.type === "assistant_message" && + event.turnItem.text.trim() !== "") || + (event.type === "turn_item.updated" && + [ + "command_execution", + "dynamic_tool", + "file_change", + "file_search", + "web_search", + "subagent", + "proposed_plan", + "todo_list", + ].includes(event.turnItem.type)) + ) + yield* Ref.set(providerWorkObserved, true); // Root provider_thread.updated always uses an ownership gate: // pre-terminal writeIfRunCurrent (attempt still running), or // post-terminal writeIfProviderThreadOwner so late roster @@ -1359,6 +1414,13 @@ export const layer: Layer.Layer< expectedStatus: "running", }, openRunOwnedSubagents: openSubagents, + canRetryContextFailure, + ...(input.hasUnpairedRunInterruptRequest === undefined + ? {} + : { + hasUnpairedRunInterruptRequest: + input.hasUnpairedRunInterruptRequest, + }), terminal: makeFailedTerminalEvent( makeProviderFailure({ cause: Cause.squash(cause), @@ -1465,6 +1527,13 @@ export const layer: Layer.Layer< expectedStatus: "running", }, openRunOwnedSubagents: openSubagents, + canRetryContextFailure, + ...(input.hasUnpairedRunInterruptRequest === undefined + ? {} + : { + hasUnpairedRunInterruptRequest: + input.hasUnpairedRunInterruptRequest, + }), terminal: makeFailedTerminalEvent( makeProviderFailure({ cause: Cause.squash(cause), diff --git a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts index 053825ad6962..c49ab10ea037 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts @@ -236,7 +236,10 @@ export function makeOrchestratorV2ProviderReplayLayer< readonly replayGate?: ProviderReplayGate; } = {}, ): Layer.Layer< - Orchestrator.OrchestratorV2 | EffectWorker.OrchestrationEffectWorkerV2 | EventSink.EventSinkV2, + | Orchestrator.OrchestratorV2 + | EffectWorker.OrchestrationEffectWorkerV2 + | EventSink.EventSinkV2 + | ProviderSessionManager.ProviderSessionManagerV2, Error | MigrationError | PlatformError.PlatformError | SqlError > { const registryLayer = harness.makeProviderAdapterRegistryLayer( @@ -264,7 +267,10 @@ export function makeOrchestratorV2ReplayLayerWithRegistry( readonly continueThreadsAfterServerUpdate?: boolean; } = {}, ): Layer.Layer< - Orchestrator.OrchestratorV2 | EffectWorker.OrchestrationEffectWorkerV2 | EventSink.EventSinkV2, + | Orchestrator.OrchestratorV2 + | EffectWorker.OrchestrationEffectWorkerV2 + | EventSink.EventSinkV2 + | ProviderSessionManager.ProviderSessionManagerV2, Error | MigrationError | PlatformError.PlatformError | SqlError > { const serverConfigLayer = Layer.effect( @@ -468,6 +474,7 @@ export function makeOrchestratorV2ReplayLayerWithRegistry( orchestratorProvided, effectWorkerProvided, eventSinkProvided, + providerSessionManagerProvided, continuationWorkerProvided, ).pipe(Layer.provide(worktreeRepairDependenciesTestLayer), Layer.provide(NodeServices.layer)); diff --git a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts index a9e04bcdb026..c3325d06fdcc 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts @@ -54,10 +54,12 @@ import { type ProviderAdapterV2HistoricalContext, ProviderAdapterProtocolError, ProviderAdapterEnsureThreadError, + ProviderAdapterTurnStartError, type ProviderAdapterV2Shape, type ProviderAdapterV2SessionRuntime, } from "../ProviderAdapter.ts"; import * as ProviderAdapterRegistry from "../ProviderAdapterRegistry.ts"; +import * as ProviderSessionManager from "../ProviderSessionManager.ts"; import { makeProviderFailure } from "../ProviderFailure.ts"; import { CLAUDE_MODEL_SELECTION, @@ -115,6 +117,13 @@ function makeTestAdapter(input: { readonly failResume?: boolean; readonly failResumeOnce?: Ref.Ref; readonly initialContextUsage?: OrchestrationV2ProviderThread["contextUsage"]; + readonly rejectContextOnce?: Ref.Ref; + readonly repeatContextRejection?: boolean; + readonly contextRejectionCode?: string; + readonly workBeforeRejection?: boolean; + readonly rejectContextBeforeStart?: boolean; + readonly failCompactionResume?: boolean; + readonly responseBeforeRejection?: boolean; readonly getModelContextWindow?: (selection: ModelSelection) => number | undefined; readonly canReuseContextUsage?: ProviderAdapterV2SessionRuntime["canReuseContextUsage"]; readonly tokenUsageByRunOrdinal?: Readonly< @@ -291,8 +300,54 @@ function makeTestAdapter(input: { } const compacting = input.compaction !== undefined && turnInput.message.text === "/compact"; - const terminalStatus = - compacting && input.compaction === "failure" + const contextRejected = + !compacting && + input.rejectContextOnce !== undefined && + (input.repeatContextRejection + ? yield* Ref.get(input.rejectContextOnce) + : yield* Ref.getAndSet(input.rejectContextOnce, false)); + if (contextRejected && input.rejectContextBeforeStart) + return yield* new ProviderAdapterTurnStartError({ + driver: input.driver, + threadId: turnInput.threadId, + providerThreadId: turnInput.providerThread.id, + runId: turnInput.runId, + cause: { errorMessage: "Prompt is too long" }, + }); + if ( + contextRejected && + input.failCompactionResume && + input.failResumeOnce !== undefined + ) + yield* Ref.set(input.failResumeOnce, true); + if (contextRejected && input.workBeforeRejection) + yield* PubSub.publish(events, { + type: "turn_item.updated", + driver: input.driver, + turnItem: { + id: TurnItemId.make(`${turnInput.attemptId}:tool`), + threadId: turnInput.threadId, + runId: turnInput.runId, + nodeId: turnInput.rootNodeId, + providerThreadId: turnInput.providerThread.id, + providerTurnId, + nativeItemRef: null, + parentItemId: null, + ordinal: turnInput.runOrdinal * 100, + status: "completed", + startedAt: eventTime, + completedAt: eventTime, + updatedAt: eventTime, + type: "command_execution", + input: "echo done", + output: "done", + exitCode: 0, + title: null, + }, + }); + const terminalStatus = contextRejected + ? "failed" + : compacting && input.compaction === "failure" ? "failed" : compacting && input.compaction === "interrupted" ? "interrupted" @@ -350,8 +405,12 @@ function makeTestAdapter(input: { status: terminalStatus, failureItemOrdinal: turnInput.runOrdinal * 100 + 2, failure: makeProviderFailure({ - message: "Simulated provider failure.", - code: "simulated_failure", + message: contextRejected + ? "Prompt is too long" + : "Simulated provider failure.", + code: contextRejected + ? (input.contextRejectionCode ?? null) + : "simulated_failure", class: "provider_error", }), } @@ -381,6 +440,12 @@ function makeTestAdapter(input: { }); } for (const event of providerEvents) { + if ( + contextRejected && + !input.responseBeforeRejection && + event.type === "turn_item.updated" + ) + continue; yield* PubSub.publish(events, event); } }), @@ -425,6 +490,20 @@ describe("orchestration v2 provider switching", () => { "compact-legacy", "large-current-input", "exhausted-native", + "exhausted-rejected-codex-native", + "exhausted-rejected-claude-native", + "exhausted-rejected-cursor-native", + "exhausted-rejected-grok-native", + "exhausted-rejected-antigravity-native", + "exhausted-rejected-opencode-native", + "exhausted-rejected-opencode2-native", + "exhausted-rejected-pi-native", + "exhausted-rejected-repeat-native", + "exhausted-rejected-start-error-native", + "exhausted-rejected-resume-failure-native", + "exhausted-rejected-unsupported-native", + "exhausted-rejected-work-native", + "exhausted-rejected-response-native", "exhausted-effort-change-native", "exhausted-manual-uppercase-native", "exhausted-missed-input-native", @@ -488,6 +567,21 @@ describe("orchestration v2 provider switching", () => { const finishCompaction = yield* Deferred.make(); const priorImages = scenario.includes("prior-images"); const exhausted = scenario.startsWith("exhausted"); + const rejected = scenario.includes("rejected"); + const rejectContextOnce = yield* Ref.make(false); + const rejectionDriver = scenario.includes("rejected-codex") + ? CODEX_DRIVER + : scenario.includes("rejected-cursor") + ? CURSOR_DRIVER + : scenario.includes("rejected-grok") + ? ProviderDriverKind.make("grok") + : scenario.includes("rejected-antigravity") + ? ProviderDriverKind.make("antigravity") + : scenario.includes("rejected-opencode") + ? ProviderDriverKind.make("opencode") + : scenario.includes("rejected-pi") + ? ProviderDriverKind.make("pi") + : CLAUDE_DRIVER; const replaceNative = scenario.includes("replacement"); const capacityScenario = modelScenario || scenario.includes("reported-capacity"); const returning = @@ -496,7 +590,9 @@ describe("orchestration v2 provider switching", () => { priorImages || exhausted; const targetSelection: ModelSelection = - scenario.includes("downgrade") || scenario === "exhausted-unknown-capacity-native" + rejected || + scenario.includes("downgrade") || + scenario === "exhausted-unknown-capacity-native" ? { ...CLAUDE_MODEL_SELECTION, model: "small-model" } : scenario === "exhausted-effort-change-native" ? { ...CLAUDE_MODEL_SELECTION, options: [{ id: "reasoningEffort", value: "low" }] } @@ -521,12 +617,33 @@ describe("orchestration v2 provider switching", () => { }), makeTestAdapter({ instanceId: CLAUDE_MODEL_SELECTION.instanceId, - driver: CLAUDE_DRIVER, - capabilities: ClaudeProviderCapabilitiesV2, + driver: rejectionDriver, + capabilities: scenario.includes("rejected-codex") + ? CodexProviderCapabilitiesV2 + : scenario.includes("rejected-cursor") + ? CursorProviderCapabilitiesV2 + : scenario.includes("rejected-grok") || scenario.includes("rejected-antigravity") + ? AcpProviderCapabilitiesV2 + : ClaudeProviderCapabilitiesV2, modelSelection: CLAUDE_MODEL_SELECTION, responseByRunOrdinal: {}, capturedTurns, failEnsureOnce, + ...(rejected + ? { + rejectContextOnce, + repeatContextRejection: scenario.includes("rejected-repeat"), + workBeforeRejection: scenario.includes("rejected-work"), + rejectContextBeforeStart: scenario.includes("rejected-start-error"), + failCompactionResume: scenario.includes("rejected-resume-failure"), + responseBeforeRejection: scenario.includes("rejected-response"), + contextRejectionCode: scenario.includes("rejected-codex") + ? "contextWindowExceeded" + : scenario.includes("rejected-opencode") + ? "ContextOverflowError" + : "prompt_too_long", + } + : {}), ...(scenario === "exhausted-stop-native" || scenario === "exhausted-stop-running-native" || scenario === "exhausted-steer-native" @@ -547,7 +664,8 @@ describe("orchestration v2 provider switching", () => { : {}), ...(exhausted ? { - ...(scenario === "exhausted-unknown-capacity-native" || + ...(rejected || + scenario === "exhausted-unknown-capacity-native" || scenario === "exhausted-no-usage-native" ? {} : { @@ -562,7 +680,7 @@ describe("orchestration v2 provider switching", () => { ? ("interrupted" as const) : scenario === "exhausted-no-shrink-native" ? ("no-shrink" as const) - : scenario === "exhausted-unsupported-native" + : scenario.includes("unsupported") ? ("unsupported" as const) : scenario === "exhausted-no-usage-native" ? ("no-usage" as const) @@ -797,7 +915,8 @@ describe("orchestration v2 provider switching", () => { priorImages && !replaceNative ? "New source constraint " + "q".repeat(9_000) : "New source constraint", - scenario.includes("downgrade") || + rejected || + scenario.includes("downgrade") || scenario === "exhausted-effort-change-native" || scenario === "exhausted-unknown-capacity-native" || scenario === "exhausted-missed-input-native" @@ -821,6 +940,36 @@ describe("orchestration v2 provider switching", () => { } if (scenario.includes("retry")) yield* Ref.set(failStartOnce, true); if (scenario === "first-bind-retry-native") yield* Ref.set(failEnsureOnce, true); + const savedBeforeSwitch = (yield* orchestrator.getThreadProjection( + threadId, + )).turnItems.filter( + (item) => item.type === "user_message" || item.type === "assistant_message", + ); + if (scenario.includes("rejected-resume-failure")) { + const target = (yield* orchestrator.getThreadProjection(threadId)).providerThreads.find( + (thread) => thread.providerInstanceId === CLAUDE_MODEL_SELECTION.instanceId, + )!; + const runtime = yield* (yield* ProviderSessionManager.ProviderSessionManagerV2).get( + target.providerSessionId!, + ); + assert.equal(runtime._tag, "Some"); + if (runtime._tag === "Some") { + const resume = runtime.value.resumeThread; + const resumeSpy = vi + .spyOn(runtime.value, "resumeThread") + .mockImplementation((input) => + Ref.get(failResumeOnce).pipe( + Effect.flatMap((failed) => + failed + ? unimplemented(CLAUDE_DRIVER, "simulated native resume failure") + : resume(input), + ), + ), + ); + yield* Effect.addFinalizer(() => Effect.sync(() => resumeSpy.mockRestore())); + } + } + if (rejected) yield* Ref.set(rejectContextOnce, true); yield* dispatch(targetOrdinal, current, targetSelection); if ( scenario === "exhausted-stop-native" || @@ -952,6 +1101,11 @@ describe("orchestration v2 provider switching", () => { yield* wait(targetOrdinal + 3); } const projection = yield* orchestrator.getThreadProjection(threadId); + for (const saved of savedBeforeSwitch) + assert.deepEqual( + projection.turnItems.find((item) => item.id === saved.id), + saved, + ); if (scenario === "exhausted-manual-uppercase-native") { assert.equal(projection.runs.at(-1)?.status, "completed"); assert.equal( @@ -966,12 +1120,15 @@ describe("orchestration v2 provider switching", () => { scenario === "exhausted-oversized-native" || scenario === "exhausted-compaction-failure-native" || scenario === "exhausted-no-shrink-native" || - scenario === "exhausted-unsupported-native"; + scenario.includes("unsupported") || + scenario.includes("rejected-repeat") || + scenario.includes("rejected-resume-failure") || + scenario.includes("rejected-response") || + scenario.includes("rejected-work"); const interrupted = scenario === "exhausted-compaction-interrupted-native" || - scenario === "exhausted-stop-native"; - // The provider finished compacting after Stop, so the run completes unsent. - const stoppedAfterCompaction = scenario === "exhausted-stop-running-native"; + scenario === "exhausted-stop-native" || + scenario === "exhausted-stop-running-native"; assert.equal( projection.runs.at(-1)?.status, interrupted ? "interrupted" : failed ? "failed" : "completed", @@ -985,7 +1142,10 @@ describe("orchestration v2 provider switching", () => { scenario === "exhausted-oversized-native" || scenario === "exhausted-effort-change-native" || scenario === "exhausted-unknown-capacity-native" || - scenario === "exhausted-unsupported-native" + scenario.includes("unsupported") || + scenario.includes("rejected-resume-failure") || + scenario.includes("rejected-response") || + scenario.includes("rejected-work") ? 0 : 1, ); @@ -1000,11 +1160,11 @@ describe("orchestration v2 provider switching", () => { : "x".repeat(70_000)), ), ); - if (failed || interrupted || stoppedAfterCompaction) { + if (failed || interrupted) { assert.equal( (yield* Ref.get(capturedTurns)).filter((turn) => turn.text.endsWith(current)) .length, - 0, + rejected ? (scenario.includes("rejected-repeat") ? 2 : 1) : 0, ); return; } @@ -1020,12 +1180,13 @@ describe("orchestration v2 provider switching", () => { assert.deepEqual((yield* Ref.get(capturedTurns)).at(-1)!.attachments, [screenshot]); assert.equal( (yield* Ref.get(capturedTurns)).filter((turn) => turn.text.endsWith(current)).length, - 1, + rejected ? 2 : 1, ); const history = scenario.endsWith("-native") ? yield* encodeJson(yield* Ref.get(injectedHistory)) : (yield* Ref.get(capturedTurns)).at(-1)!.text; - if (!scenario.includes("downgrade")) assert.include(history, "New source constraint"); + if (!scenario.includes("downgrade") && !rejected) + assert.include(history, "New source constraint"); yield* dispatch(5, "Further source work", CODEX_MODEL_SELECTION); yield* wait(5); yield* dispatch(6, "Return to the recovered session", targetSelection); diff --git a/docs/user/portable-handoffs.md b/docs/user/portable-handoffs.md index 5077655ca896..d3a24e3ee395 100644 --- a/docs/user/portable-handoffs.md +++ b/docs/user/portable-handoffs.md @@ -22,11 +22,17 @@ agent-written summary. A handoff must leave room for existing provider context, your request and attachments, instructions, tools, and subsequent work. If the target conversation leaves too little room, T3 Code runs native compaction in that same conversation, then retries your request once. Your saved chat and original -request stay intact. This also applies to an overfull model switch within one provider when the -new model capacity is known. No encrypted state is copied between providers. - -If compaction is unsupported, fails, is stopped, or still leaves too little room, T3 Code reports -the failure without silently replacing the session or shortening your request. Reduce the request +request stay intact. Native compaction summarizes the provider conversation; the full saved chat +remains available through the T3 Code thread-reading tool. This also applies to an overfull model +switch within one provider when the new model capacity is known. If the provider instead reports +that the request exceeds its context limit, T3 Code can compact and retry once, even when the model +capacity is unknown. This recovery requires native compaction support and stops if the agent has +already sent an answer, started a tool, or delegated work. No encrypted state is copied between +providers. + +If compaction is stopped, the run stays stopped and the original request is not sent again. If +compaction is unsupported, fails, or still leaves too little room, T3 Code reports the failure +without silently replacing the session or shortening your request. Reduce the request or select a larger-context model before trying again. Server operators can set `T3CODE_CONTEXT_HANDOFF_TOKEN_CAP` to change the initial history allowance From 7c625fd993b4d99bf1b8f552a320b29997557cbd Mon Sep 17 00:00:00 2001 From: Bil0000 <62337003+Bil0000@users.noreply.github.com> Date: Sun, 4 Oct 2026 15:02:23 +0200 Subject: [PATCH 03/10] fix(client): show automatic context compaction --- .../src/state/use-thread-composer-state.ts | 4 ++ apps/web/src/components/ChatView.tsx | 8 +-- .../src/state/threadExecution.test.ts | 50 +++++++++++++++++++ .../src/state/threadExecution.ts | 15 ++++++ 4 files changed, 74 insertions(+), 3 deletions(-) diff --git a/apps/mobile/src/state/use-thread-composer-state.ts b/apps/mobile/src/state/use-thread-composer-state.ts index 1f9eed3d675c..50a7f54cd915 100644 --- a/apps/mobile/src/state/use-thread-composer-state.ts +++ b/apps/mobile/src/state/use-thread-composer-state.ts @@ -5,6 +5,7 @@ import { deriveProviderSubagentStatus, deriveRunlessWorkStartedAt, deriveThreadActivityRun, + deriveThreadIsCompacting, deriveThreadRuntime, threadRuntimeHasInterruptibleRun, } from "@t3tools/client-runtime/state/thread-execution"; @@ -365,6 +366,8 @@ export function useThreadComposerState() { ); const isCompacting = useMemo(() => { + if (selectedThreadProjection && deriveThreadIsCompacting(selectedThreadProjection.projection)) + return true; const queuedCompact = selectedThreadQueuedMessages.some( (message) => message.messageId === dispatchingQueuedMessageId && @@ -392,6 +395,7 @@ export function useThreadComposerState() { dispatchingQueuedMessageId, selectedThreadQueuedMessages, selectedThreadRuntime, + selectedThreadProjection, selectedThreadVisibleTurnItems, ]); diff --git a/apps/web/src/components/ChatView.tsx b/apps/web/src/components/ChatView.tsx index 78c5f113162e..4fb9f9ea6103 100644 --- a/apps/web/src/components/ChatView.tsx +++ b/apps/web/src/components/ChatView.tsx @@ -94,6 +94,7 @@ import { formatModelSelectionEffort, deriveRunlessWorkStartedAt, deriveThreadActivityRun, + deriveThreadIsCompacting, deriveLatestThreadRun, deriveThreadRuntime, presentPendingBackgroundWork, @@ -3516,9 +3517,10 @@ export default function ChatView(props: ChatViewProps) { item.status === "completed", ); const isCompacting = - (isSendBusy || phase === "connecting" || phase === "running") && - compactRequestIsActive && - !compactionSettled; + (serverProjection !== null && deriveThreadIsCompacting(serverProjection)) || + ((isSendBusy || phase === "connecting" || phase === "running") && + compactRequestIsActive && + !compactionSettled); // A rewind is not agent work: the composer shows "Rewinding conversation" // instead of the timeline growing a Thinking row. const isWorking = diff --git a/packages/client-runtime/src/state/threadExecution.test.ts b/packages/client-runtime/src/state/threadExecution.test.ts index 6ee64d2eaf66..54ff27edcf9d 100644 --- a/packages/client-runtime/src/state/threadExecution.test.ts +++ b/packages/client-runtime/src/state/threadExecution.test.ts @@ -7,6 +7,7 @@ import { ProviderSessionId, ProviderDriverKind, RunId, + RunAttemptId, ThreadId, type OrchestrationV2ExecutionNode, type OrchestrationV2RunStatus, @@ -26,6 +27,7 @@ import { deriveRunlessWorkStartedAt, deriveThreadActivityRun, deriveThreadRuntime, + deriveThreadIsCompacting, threadRuntimeHasInterruptibleRun, } from "./threadExecution.ts"; import { threadRuntimeCanArchive, type ThreadRuntimeSummary } from "./models.ts"; @@ -53,6 +55,54 @@ function run(id: string, ordinal: number, status: OrchestrationV2RunStatus) { } describe("thread execution presentation", () => { + it.each(["pending", "running", "completed", "failed", "interrupted"] as const)( + "shows native compaction only while its active attempt is %s", + (status) => { + const attemptId = RunAttemptId.make("compact-attempt"); + const activeRun = { ...run("compacting", 1, "running"), activeAttemptId: attemptId }; + const attempt = { + id: attemptId, + runId: activeRun.id, + attemptOrdinal: 1, + contextCompaction: true, + rootNodeId: NodeId.make("compact-root"), + providerInstanceId: activeRun.providerInstanceId, + providerThreadId: ProviderThreadId.make("compact-thread"), + providerTurnId: null, + reason: "provider_recovery" as const, + status, + startedAt: now, + completedAt: null, + }; + const projection = { + ...v2Projection, + runs: [activeRun, run("queued", 2, "queued")], + attempts: [attempt], + }; + expect(deriveThreadIsCompacting(projection)).toBe( + status === "pending" || status === "running", + ); + expect( + deriveThreadIsCompacting({ + ...projection, + attempts: [{ ...attempt, contextCompaction: false }], + }), + ).toBe(false); + expect( + deriveThreadIsCompacting({ + ...projection, + runs: [{ ...activeRun, status: "interrupted" }], + }), + ).toBe(false); + expect( + deriveThreadIsCompacting({ + ...projection, + runs: [{ ...activeRun, activeAttemptId: RunAttemptId.make("retry") }], + }), + ).toBe(false); + }, + ); + it("derives the current root failure without inheriting errors from children or previous runs", () => { const failed = { ...run("limited", 1, "failed"), rootNodeId: NodeId.make("root") }; const item = { diff --git a/packages/client-runtime/src/state/threadExecution.ts b/packages/client-runtime/src/state/threadExecution.ts index a7722af0391b..b6f46505d279 100644 --- a/packages/client-runtime/src/state/threadExecution.ts +++ b/packages/client-runtime/src/state/threadExecution.ts @@ -106,6 +106,21 @@ export function deriveThreadActivityRun( return run === null ? null : summarizeThreadRun(projection, run); } +export function deriveThreadIsCompacting(projection: OrchestrationV2ThreadProjection): boolean { + const run = latestMatchingRun(projection, (candidate) => + INTERRUPTIBLE_RUN_STATUSES.has(candidate.status), + ); + return ( + run !== null && + projection.attempts.some( + (attempt) => + attempt.id === run.activeAttemptId && + attempt.contextCompaction === true && + (attempt.status === "pending" || attempt.status === "running"), + ) + ); +} + /** * Provider-native subagent threads never get app runs: their work is a runless * root turn whose status follows the subagent. Returns when that work started From f7cfdb1e7be377cc9bc68d94a21ac831174ee099 Mon Sep 17 00:00:00 2001 From: Bil0000 <62337003+Bil0000@users.noreply.github.com> Date: Sun, 4 Oct 2026 15:19:45 +0200 Subject: [PATCH 04/10] fix(v2): ignore completed Stop pairs during context recovery --- apps/server/src/orchestration-v2/EventSink.ts | 18 +++++--- .../FoundationPersistence.test.ts | 44 +++++++++++++++---- .../orchestration-v2/RunExecutionService.ts | 14 ++++-- 3 files changed, 59 insertions(+), 17 deletions(-) diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index 342446ab3011..c499d29660cd 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -92,7 +92,10 @@ export interface EventSinkV2Shape { readonly expectedStatus: OrchestrationV2Run["status"]; readonly events: ReadonlyArray; readonly effects?: ReadonlyArray; - readonly rejectInterruptRequestId?: TurnItemId; + readonly rejectUnpairedInterrupt?: { + readonly requestId: TurnItemId; + readonly resultId: TurnItemId; + }; }) => Effect.Effect< { readonly committed: boolean; @@ -420,12 +423,17 @@ const baseLayer: Layer.Layer< `; const current = rows[0]; const interrupted = - input.rejectInterruptRequestId === undefined + input.rejectUnpairedInterrupt === undefined ? false : (yield* sql` - SELECT 1 FROM orchestration_v2_projection_turn_items - WHERE thread_id = ${input.threadId} - AND turn_item_id = ${input.rejectInterruptRequestId} + SELECT 1 FROM orchestration_v2_projection_turn_items AS request + WHERE request.thread_id = ${input.threadId} + AND request.turn_item_id = ${input.rejectUnpairedInterrupt.requestId} + AND NOT EXISTS ( + SELECT 1 FROM orchestration_v2_projection_turn_items AS result + WHERE result.thread_id = ${input.threadId} + AND result.turn_item_id = ${input.rejectUnpairedInterrupt.resultId} + ) LIMIT 1 `).length > 0; if ( diff --git a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts index 110a7b08b2e8..111a987682b8 100644 --- a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts +++ b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts @@ -1509,8 +1509,8 @@ it.layer(TestLayer)("orchestration V2 foundation persistence", (it) => { }).pipe(Effect.provide(Layer.fresh(TestLayer))), ); - it.effect.each(["cancelled", "stop-request"] as const)( - "does not publish a stale provider start after %s", + it.effect.each(["cancelled", "stop-request", "paired-stop-request"] as const)( + "checks %s before publishing a provider retry", (mode) => Effect.gen(function* () { const eventSink = yield* EventSink.EventSinkV2; @@ -1522,6 +1522,7 @@ it.layer(TestLayer)("orchestration V2 foundation persistence", (it) => { const runId = RunId.make(`run:foundation-stale-provider-start:${mode}`); const attemptId = RunAttemptId.make(`run-attempt:foundation-stale-provider-start:${mode}`); const interruptRequestId = TurnItemId.make(`turn-item:foundation-stop-request:${mode}`); + const interruptResultId = TurnItemId.make(`turn-item:foundation-stop-result:${mode}`); const thread = makeThread(threadId, now); const startingRun: OrchestrationV2Run = { id: runId, @@ -1571,7 +1572,7 @@ it.layer(TestLayer)("orchestration V2 foundation persistence", (it) => { runId, activeAttemptId: attemptId, expectedStatus: "starting", - rejectInterruptRequestId: interruptRequestId, + rejectUnpairedInterrupt: { requestId: interruptRequestId, resultId: interruptResultId }, effects: [ { id: `guarded-continuation:${mode}`, @@ -1647,15 +1648,42 @@ it.layer(TestLayer)("orchestration V2 foundation persistence", (it) => { }, ], }); + if (mode === "paired-stop-request") { + const request = (yield* projectionStore.getThreadProjection(threadId)).turnItems + .filter((item) => item.type === "run_interrupt_request") + .find((item) => item.id === interruptRequestId)!; + yield* eventSink.write({ + events: [ + { + id: EventId.make(`event:foundation-stale-provider-start:stop-result:${mode}`), + type: "turn-item.updated", + threadId, + runId, + providerInstanceId, + occurredAt: interruptedAt, + payload: { + ...request, + id: interruptResultId, + type: "run_interrupt_result", + message: "Stopped previous attempt", + }, + }, + ], + }); + } yield* Deferred.succeed(releaseStaleStart, undefined); const staleResult = yield* Fiber.join(staleStartFiber); - assert.isFalse(staleResult.committed); - assert.deepEqual(yield* outbox.listByCommandId(commandId), []); - assert.deepEqual(staleResult.storedEvents, []); - assert.equal(yield* Ref.get(providerStartCount), 0); + const expectedWrites = mode === "paired-stop-request" ? 1 : 0; + assert.equal(staleResult.committed, expectedWrites === 1); + assert.lengthOf(yield* outbox.listByCommandId(commandId), expectedWrites); + assert.lengthOf(staleResult.storedEvents, expectedWrites); + assert.equal(yield* Ref.get(providerStartCount), expectedWrites); const projection = yield* projectionStore.getThreadProjection(threadId); - assert.equal(projection.runs[0]?.status, mode === "cancelled" ? "cancelled" : "starting"); + assert.equal( + projection.runs[0]?.status, + mode === "cancelled" ? "cancelled" : expectedWrites ? "running" : "starting", + ); }).pipe(Effect.provide(Layer.fresh(TestLayer))), ); diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 8449af635d43..8998a95e6d22 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -682,10 +682,16 @@ export const layer: Layer.Layer< runId: input.run.id, activeAttemptId: input.attempt.id, expectedStatus: "running", - rejectInterruptRequestId: idAllocator.derive.runSignalTurnItem({ - runId: input.run.id, - signal: "interrupt-request", - }), + rejectUnpairedInterrupt: { + requestId: idAllocator.derive.runSignalTurnItem({ + runId: input.run.id, + signal: "interrupt-request", + }), + resultId: idAllocator.derive.runSignalTurnItem({ + runId: input.run.id, + signal: "interrupt-result", + }), + }, events, effects: [ { From 8df430f9fc2e6204b941c2c3e8b315816e60a74a Mon Sep 17 00:00:00 2001 From: Bil0000 <62337003+Bil0000@users.noreply.github.com> Date: Sun, 4 Oct 2026 15:21:28 +0200 Subject: [PATCH 05/10] fix(v2): clear recovered context failures from chat --- .../orchestration-v2/RunExecutionService.ts | 37 ++++++++++++++++++- .../ProviderSwitch.integration.test.ts | 6 +++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 8998a95e6d22..0de7b549f774 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -646,7 +646,7 @@ export const layer: Layer.Layer< completedAt: null, }; const { delegatedCompletion: _completion, ...run } = input.run; - const events = yield* Effect.forEach( + const events: Array = yield* Effect.forEach( [ { type: "run-attempt.updated", @@ -676,6 +676,41 @@ export const layer: Layer.Layer< } satisfies OrchestrationV2DomainEvent; }), ); + if (retryContextFailure && input.failureItemPersisted && terminal.status === "failed") { + const { + failure: _failure, + retry: _retry, + ...item + } = makeProviderFailureTurnItem({ + idAllocator, + driver: terminal.driver, + threadId: input.run.threadId, + runId: input.run.id, + nodeId: input.rootNode.id, + providerThreadId: terminal.providerThreadId, + providerTurnId: terminal.providerTurnId, + itemOrdinal: terminal.failureItemOrdinal, + failure: terminal.failure, + occurredAt: completedAt, + }); + events.push({ + id: yield* idAllocator.allocate.event({ threadId: input.run.threadId }), + type: "turn-item.updated", + threadId: input.run.threadId, + runId: input.run.id, + nodeId: input.rootNode.id, + providerInstanceId: input.run.providerInstanceId, + occurredAt: completedAt, + payload: { + ...item, + type: "system_notice", + title: "Context recovery", + status: "completed", + message: + "Compacting the conversation before retrying the request that exceeded the context limit.", + }, + }); + } const commandId = CommandId.make(`command:context-compaction:${input.attempt.id}`); const continuation = yield* eventSink.writeIfRunCurrent({ threadId: input.run.threadId, diff --git a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts index c3325d06fdcc..42119a96a1cb 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts @@ -1150,6 +1150,12 @@ describe("orchestration v2 provider switching", () => { : 1, ); assert.isTrue(compactTurns.every((turn) => turn.attachments.length === 0)); + if (rejected) { + const errors = projection.visibleTurnItems.filter( + ({ item }) => item.runId === projection.runs.at(-1)!.id && item.type === "error", + ); + assert.equal(errors.length, failed ? 1 : 0); + } assert.isTrue( projection.turnItems.some( (item) => From f497731f5a8451ce2b9eec560a17b74fb42f0668 Mon Sep 17 00:00:00 2001 From: Bil0000 <62337003+Bil0000@users.noreply.github.com> Date: Sun, 4 Oct 2026 15:29:23 +0200 Subject: [PATCH 06/10] fix(v2): block context retries after streaming output --- .../orchestration-v2/RunExecutionService.ts | 45 +++++++++--------- .../testkit/ProviderReplayHarness.ts | 9 +++- .../ProviderSwitch.integration.test.ts | 47 +++++++++++++++++-- 3 files changed, 75 insertions(+), 26 deletions(-) diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 0de7b549f774..f4d8230e86d3 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -1315,32 +1315,35 @@ export const layer: Layer.Layer< Stream.tap((event) => Effect.gen(function* () { let storedEventCount = 0; + if ( + event.type === "subagent.updated" || + (event.type === "node.updated" && + (event.node.kind === "tool_call" || event.node.kind === "subagent")) || + event.type === "runtime_request.updated" || + (event.type === "message.updated" && + event.message.role === "assistant" && + event.message.text.trim() !== "") || + (event.type === "turn_item.updated" && + event.turnItem.type === "assistant_message" && + event.turnItem.text.trim() !== "") || + (event.type === "turn_item.updated" && + [ + "command_execution", + "dynamic_tool", + "file_change", + "file_search", + "web_search", + "subagent", + "proposed_plan", + "todo_list", + ].includes(event.turnItem.type)) + ) + yield* Ref.set(providerWorkObserved, true); const deliveredEvent = filterAssistantEvent( event, DateTime.toEpochMillis(yield* DateTime.now), ); if (deliveredEvent) { - if ( - event.type === "subagent.updated" || - (event.type === "node.updated" && - (event.node.kind === "tool_call" || event.node.kind === "subagent")) || - event.type === "runtime_request.updated" || - (event.type === "turn_item.updated" && - event.turnItem.type === "assistant_message" && - event.turnItem.text.trim() !== "") || - (event.type === "turn_item.updated" && - [ - "command_execution", - "dynamic_tool", - "file_change", - "file_search", - "web_search", - "subagent", - "proposed_plan", - "todo_list", - ].includes(event.turnItem.type)) - ) - yield* Ref.set(providerWorkObserved, true); // Root provider_thread.updated always uses an ownership gate: // pre-terminal writeIfRunCurrent (attempt still running), or // post-terminal writeIfProviderThreadOwner so late roster diff --git a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts index c49ab10ea037..e7e4cb587503 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts @@ -1,7 +1,11 @@ import * as OtelEnvironment from "@t3tools/shared/otelEnvironment"; import { DEFAULT_SIGNAL_EXPORT } from "@t3tools/shared/observability"; import * as NodeServices from "@effect/platform-node/NodeServices"; -import type { ProviderDriverKind, ProviderReplayTranscript } from "@t3tools/contracts"; +import type { + ProviderDriverKind, + ProviderReplayTranscript, + ResponseStreamingMode, +} from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; @@ -253,6 +257,7 @@ export function makeOrchestratorV2ReplayLayerWithRegistry( scenario: Pick, registryLayer: Layer.Layer, options: { + readonly responseStreamingMode?: ResponseStreamingMode; readonly databaseLayer?: Layer.Layer< SqlClient.SqlClient, MigrationError | PlatformError.PlatformError | SqlError @@ -290,7 +295,7 @@ export function makeOrchestratorV2ReplayLayerWithRegistry( options.runContinuationWorker === true ? ProviderContinuationRequests.layer : Layer.empty; const providedRegistryLayer = registryLayer.pipe(Layer.provide(continuationRequestsLayer)); const serverSettingsLayer = ServerSettings.layerTest({ - responseStreamingMode: "turn", + responseStreamingMode: options.responseStreamingMode ?? "turn", ...(options.continueThreadsAfterServerUpdate === undefined ? {} : { continueThreadsAfterServerUpdate: options.continueThreadsAfterServerUpdate }), diff --git a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts index 42119a96a1cb..f43091c4e3e1 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts @@ -123,7 +123,7 @@ function makeTestAdapter(input: { readonly workBeforeRejection?: boolean; readonly rejectContextBeforeStart?: boolean; readonly failCompactionResume?: boolean; - readonly responseBeforeRejection?: boolean; + readonly responseBeforeRejection?: "completed" | "streaming" | "streaming-message"; readonly getModelContextWindow?: (selection: ModelSelection) => number | undefined; readonly canReuseContextUsage?: ProviderAdapterV2SessionRuntime["canReuseContextUsage"]; readonly tokenUsageByRunOrdinal?: Readonly< @@ -391,7 +391,9 @@ function makeTestAdapter(input: { `message:${input.driver}:${turnInput.threadId}:${turnInput.attemptId}:assistant`, ), text: response, - streaming: false, + streaming: + contextRejected && + input.responseBeforeRejection?.startsWith("streaming") === true, }, }, { @@ -446,6 +448,32 @@ function makeTestAdapter(input: { event.type === "turn_item.updated" ) continue; + if ( + contextRejected && + input.responseBeforeRejection === "streaming-message" && + event.type === "turn_item.updated" && + event.turnItem.type === "assistant_message" + ) { + yield* PubSub.publish(events, { + type: "message.updated", + driver: input.driver, + message: { + id: event.turnItem.messageId, + threadId: turnInput.threadId, + runId: turnInput.runId, + nodeId: turnInput.rootNodeId, + createdBy: "agent", + creationSource: "provider", + role: "assistant", + text: response, + attachments: [], + streaming: true, + createdAt: eventTime, + updatedAt: eventTime, + }, + }); + continue; + } yield* PubSub.publish(events, event); } }), @@ -504,6 +532,10 @@ describe("orchestration v2 provider switching", () => { "exhausted-rejected-unsupported-native", "exhausted-rejected-work-native", "exhausted-rejected-response-native", + "exhausted-rejected-response-streaming-native", + "exhausted-rejected-response-streaming-paragraph-native", + "exhausted-rejected-response-streaming-message-native", + "exhausted-rejected-response-streaming-message-paragraph-native", "exhausted-effort-change-native", "exhausted-manual-uppercase-native", "exhausted-missed-input-native", @@ -636,7 +668,15 @@ describe("orchestration v2 provider switching", () => { workBeforeRejection: scenario.includes("rejected-work"), rejectContextBeforeStart: scenario.includes("rejected-start-error"), failCompactionResume: scenario.includes("rejected-resume-failure"), - responseBeforeRejection: scenario.includes("rejected-response"), + ...(scenario.includes("rejected-response") + ? { + responseBeforeRejection: scenario.includes("response-streaming-message") + ? "streaming-message" + : scenario.includes("response-streaming") + ? "streaming" + : "completed", + } + : {}), contextRejectionCode: scenario.includes("rejected-codex") ? "contextWindowExceeded" : scenario.includes("rejected-opencode") @@ -1321,6 +1361,7 @@ describe("orchestration v2 provider switching", () => { }, }, registry, + { responseStreamingMode: scenario.includes("paragraph") ? "paragraph" : "turn" }, ), ), ); From 6b986839f254f188c878b126a826271b9e508aa1 Mon Sep 17 00:00:00 2001 From: Bil0000 Date: Sun, 4 Oct 2026 17:17:14 +0300 Subject: [PATCH 07/10] fix(v2): continue unfinished compaction runs after restart --- .../RestartContinuation.test.ts | 25 +++++++++++-------- .../orchestration-v2/RestartContinuation.ts | 10 ++------ 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/apps/server/src/orchestration-v2/RestartContinuation.test.ts b/apps/server/src/orchestration-v2/RestartContinuation.test.ts index 97022ef94e1c..52e7981c3709 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.test.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.test.ts @@ -35,7 +35,7 @@ const attemptId = RunAttemptId.make("attempt:restart"); // "No project" threads belong to the environment's Scratch project. const scratchProjectId = ProjectId.make("project:scratch"); -function makeProjection() { +function makeProjection(contextCompaction = false) { return { thread: { id: threadId, @@ -79,7 +79,7 @@ function makeProjection() { }, ], runtimeRequests: [], - attempts: [], + attempts: contextCompaction ? [{ id: attemptId, contextCompaction: true }] : [], nodes: [], subagents: [], messages: [], @@ -111,7 +111,6 @@ it("requires matching saved native state for an unfinished root run", () => { ], }, { ...projection, providerTurns: [] }, - { ...projection, attempts: [{ id: attemptId, contextCompaction: true }] }, ...[ "queued", "preparing", @@ -126,6 +125,10 @@ it("requires matching saved native state for an unfinished root run", () => { assert.isUndefined(restartContinuationRun(invalid as OrchestrationV2ThreadProjection)); }); +it("continues an unfinished root run during automatic context compaction", () => { + assert.equal(restartContinuationRun(makeProjection(true))?.id, runId); +}); + it("continues a live turn whose session the adapter never marked running", () => { const projection = makeProjection(); // Codex, Claude, Cursor and ACP sessions stay "ready" for their whole life. @@ -254,13 +257,15 @@ it.effect("does not continue a failed run that lost background work", () => ); it.effect.each([ - [false, undefined], - [true, undefined], - [false, true], - [true, false], + [false, undefined, false], + [true, undefined, false], + [false, true, false], + [true, false, false], + [false, undefined, true], + [true, undefined, true], ] as const)( - "atomically records restart intent with cancellation when opt-in is %s and project override is %s", - ([enabled, projectOverride]) => + "atomically records restart intent with cancellation when opt-in is %s, project override is %s and compaction is %s", + ([enabled, projectOverride, contextCompaction]) => Effect.gen(function* () { let committed: Parameters[0] | undefined; const recovery = yield* ProviderRuntimeRecovery.make.pipe( @@ -279,7 +284,7 @@ it.effect.each([ }), Layer.mock(ProjectionStore.ProjectionStoreV2)({ getRecoveryThreadIds: () => Effect.succeed([threadId]), - getRuntimeRecoveryProjection: () => Effect.succeed(makeProjection()), + getRuntimeRecoveryProjection: () => Effect.succeed(makeProjection(contextCompaction)), }), Layer.mock(EventSink.EventSinkV2)({ commitCommand: (input) => { diff --git a/apps/server/src/orchestration-v2/RestartContinuation.ts b/apps/server/src/orchestration-v2/RestartContinuation.ts index 8c39f80ed74b..013e7a07fbc4 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.ts @@ -28,7 +28,7 @@ const CONTINUE_PROMPT = "Continue where you left off."; export function restartContinuationRun( projection: Pick< ProjectionRuntimeRecoveryState, - "thread" | "runs" | "attempts" | "providerThreads" | "providerSessions" | "providerTurns" + "thread" | "runs" | "providerThreads" | "providerSessions" | "providerTurns" >, ): OrchestrationV2Run | undefined { if (projection.thread.archivedAt !== null || projection.thread.deletedAt !== null) return; @@ -40,13 +40,7 @@ export function restartContinuationRun( : latest, undefined, ); - if ( - !run || - projection.attempts.some( - (attempt) => attempt.id === run.activeAttemptId && attempt.contextCompaction, - ) - ) - return; + if (!run) return; const preparedContinuation = run.status === "starting" && run.restartContinuationOfRunId !== undefined; if (run.status !== "running" && !preparedContinuation) return; From cca73caec09b17da7074af9025b7b1203c194143 Mon Sep 17 00:00:00 2001 From: Bil0000 <62337003+Bil0000@users.noreply.github.com> Date: Sun, 4 Oct 2026 16:18:52 +0200 Subject: [PATCH 08/10] fix(v2): preserve pending requests across compaction restarts --- .../ProviderTurnStartService.ts | 27 ++- .../RestartContinuation.test.ts | 101 ++++++++ .../orchestration-v2/RestartContinuation.ts | 37 ++- .../ProviderSwitch.integration.test.ts | 216 +++++++++++++++++- docs/user/portable-handoffs.md | 6 +- 5 files changed, 371 insertions(+), 16 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index c49b250e375a..52f5dbc8b87f 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -51,6 +51,7 @@ import * as ProviderSessionManager from "./ProviderSessionManager.ts"; import { makeProviderFailure } from "./ProviderFailure.ts"; import * as RunExecutionService from "./RunExecutionService.ts"; import * as RuntimePolicy from "./RuntimePolicy.ts"; +import { hasPendingCompactionRequest } from "./RestartContinuation.ts"; import { isRestartNoteContinuation, pendingRestartCancelledBackgroundWork, @@ -618,12 +619,18 @@ export const layer: Layer.Layer< }); // Delivered once: this run's provider turn marks the work as told. A // restart continuation is prompted by its own text or resumes natively. - const noteContinuation = isRestartNoteContinuation( - run, - projection.runs, - projection.providerTurns, - projection.attempts, - ); + const continuationSource = + run.restartContinuationOfRunId === undefined + ? undefined + : projection.runs.find((source) => source.id === run.restartContinuationOfRunId); + const compactionContinuation = hasPendingCompactionRequest(continuationSource, projection); + const promptedContinuation = + isRestartNoteContinuation( + run, + projection.runs, + projection.providerTurns, + projection.attempts, + ) || compactionContinuation; const restartCancelledWork = pendingRestartCancelledBackgroundWork({ runs: projection.runs, providerTurns: projection.providerTurns, @@ -1038,7 +1045,7 @@ export const layer: Layer.Layer< } if (!(yield* isCurrentAttemptInStatus("starting"))) return providerThread; - if (contextCompaction) { + if (contextCompaction || compactionContinuation) { yield* settleStartFailure({ signal: "provider-thread-load-failure", title: "Provider turn failed to start", @@ -1338,10 +1345,10 @@ export const layer: Layer.Layer< const context = [delivery.context, restartNote] .filter((part) => part !== "") .join("\n\n"); - // A note continuation has no turn to resume; its text is the prompt. + // Note and compaction continuations start with their saved prompt. const { restartContinuationOfRunId: _resumedRunId, ...promptedInput } = turnInput; yield* start({ - ...(noteContinuation ? promptedInput : turnInput), + ...(promptedContinuation ? promptedInput : turnInput), message: { ...turnInput.message, text: context === "" ? userText : `${context}\n\nUser message:\n${userText}`, @@ -1375,7 +1382,7 @@ export const layer: Layer.Layer< (effectiveHandoffs.length === 0 && retryHandoff.length === 0 && restartNote === "" && - !noteContinuation) + !promptedContinuation) ? session : makeDeliverySession(session, startWithHandoffs); yield* runExecution.startRootRun({ diff --git a/apps/server/src/orchestration-v2/RestartContinuation.test.ts b/apps/server/src/orchestration-v2/RestartContinuation.test.ts index 52e7981c3709..ce899d3eb91b 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.test.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.test.ts @@ -90,6 +90,14 @@ function makeProjection(contextCompaction = false) { it("requires matching saved native state for an unfinished root run", () => { const projection = makeProjection(); assert.equal(restartContinuationRun(projection)?.id, runId); + assert.equal( + restartContinuationRun({ + ...projection, + attempts: [{ id: attemptId, contextCompaction: true }], + providerTurns: [], + } as unknown as OrchestrationV2ThreadProjection)?.id, + runId, + ); for (const invalid of [ { ...projection, thread: { ...projection.thread, archivedAt: {} } }, { ...projection, thread: { ...projection.thread, deletedAt: {} } }, @@ -649,6 +657,99 @@ const queuedFollowUp = { queueHeld: true, }; +it.effect.each([ + "resume", + "prepared-resume", + "started-continuation", + "stopped", + "disabled", + "missing-request", +] as const)("handles a restart during automatic compaction when %s", (scenario) => + Effect.gen(function* () { + const message = { + id: MessageId.make("message:user"), + text: "Use the attached diagram and keep the API unchanged.", + attachments: [ + { + type: "image", + id: "diagram", + name: "diagram.png", + mimeType: "image/png", + sizeBytes: 10, + }, + ], + context: { version: 1 as const, records: [] }, + }; + const projection = { + ...cutMidTurn(), + ...(["prepared-resume", "started-continuation"].includes(scenario) + ? { + runs: [ + { + ...cutMidTurn().runs[0]!, + id: RunId.make("run:compaction-source"), + ordinal: 0, + activeAttemptId: RunAttemptId.make("attempt:compaction-source"), + }, + { + ...cutMidTurn().runs[0]!, + restartContinuationOfRunId: RunId.make("run:compaction-source"), + }, + ], + attempts: [ + { id: RunAttemptId.make("attempt:compaction-source"), contextCompaction: true }, + ], + providerTurns: scenario === "prepared-resume" ? [] : cutMidTurn().providerTurns, + } + : { attempts: [{ id: attemptId, contextCompaction: true }] }), + messages: scenario === "missing-request" ? [] : [message], + turnItems: + scenario === "stopped" + ? [{ id: "turn-item:interrupt", runId, type: "run_interrupt_request" }] + : [], + } as unknown as OrchestrationV2ThreadProjection; + const commands: Parameters< + ThreadManagementService.ThreadManagementService["Service"]["dispatch"] + >[0][] = []; + yield* continueRestartedRun({ threadId, sourceRunId: runId }).pipe( + Effect.provide( + Layer.merge( + Layer.mock(ThreadManagementService.ThreadManagementService)({ + getThreadRecords: () => Effect.succeed(projection), + recoverDelegatedTask: () => Effect.void, + dispatch: (command) => { + commands.push(command); + return Effect.succeed({} as never); + }, + }), + ServerSettings.layerTest({ continueThreadsAfterServerUpdate: scenario !== "disabled" }), + ), + ), + ); + const resumed = ["resume", "prepared-resume", "started-continuation"].includes(scenario); + assert.lengthOf(commands, resumed ? 1 : 0); + if (resumed) { + const command = commands[0]!; + assert.equal(command.type, "message.dispatch"); + if (command.type === "message.dispatch") { + assert.equal( + command.text, + scenario === "started-continuation" ? "Continue where you left off." : message.text, + ); + assert.deepEqual( + command.attachments, + scenario === "started-continuation" ? [] : message.attachments, + ); + assert.deepEqual( + command.context, + scenario === "started-continuation" ? undefined : message.context, + ); + assert.equal(command.restartContinuationOfRunId, runId); + } + } + }), +); + it.effect("continues a cut run past queued follow-ups, which stay held", () => Effect.gen(function* () { const live = makeProjection(); diff --git a/apps/server/src/orchestration-v2/RestartContinuation.ts b/apps/server/src/orchestration-v2/RestartContinuation.ts index 013e7a07fbc4..e777a4b4d8ca 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.ts @@ -28,7 +28,7 @@ const CONTINUE_PROMPT = "Continue where you left off."; export function restartContinuationRun( projection: Pick< ProjectionRuntimeRecoveryState, - "thread" | "runs" | "providerThreads" | "providerSessions" | "providerTurns" + "thread" | "runs" | "attempts" | "providerThreads" | "providerSessions" | "providerTurns" >, ): OrchestrationV2Run | undefined { if (projection.thread.archivedAt !== null || projection.thread.deletedAt !== null) return; @@ -76,6 +76,7 @@ export function restartContinuationRun( return; if ( liveTurnRequired && + !hasPendingCompactionRequest(run, projection) && !projection.providerTurns.some( (turn) => turn.providerThreadId === providerThread.id && @@ -87,6 +88,28 @@ export function restartContinuationRun( return run; } +export function hasPendingCompactionRequest( + source: OrchestrationV2Run | undefined, + projection: Pick, +): boolean { + let current = source; + while (current !== undefined) { + const attemptId = current.activeAttemptId; + if ( + projection.attempts.some((attempt) => attempt.id === attemptId && attempt.contextCompaction) + ) + return true; + const sourceId = current.restartContinuationOfRunId; + if ( + sourceId === undefined || + projection.providerTurns.some((turn) => turn.runAttemptId === attemptId) + ) + return false; + current = projection.runs.find((run) => run.id === sourceId); + } + return false; +} + export const continueRestartedRun = Effect.fn("RestartContinuation.continueRestartedRun")( function* (input: { readonly threadId: ThreadId; readonly sourceRunId: RunId }) { const settings = yield* ServerSettings.ServerSettingsService; @@ -145,6 +168,9 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta (message) => message.id === source.userMessageId, ); if (sourceMessage !== undefined && isNativeMaintenanceCommand(sourceMessage)) return; + const compacting = hasPendingCompactionRequest(source, projection); + if (compacting && sourceMessage === undefined) return; + const continuationText = compacting ? sourceMessage!.text : CONTINUE_PROMPT; const note = restartContinuationNote( source, projection.runs, @@ -160,11 +186,14 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta messageId, text: noteText === undefined - ? CONTINUE_PROMPT + ? continuationText : note.settled ? noteText - : `${noteText}\n\n${CONTINUE_PROMPT}`, - attachments: [], + : `${noteText}\n\n${continuationText}`, + attachments: compacting ? sourceMessage!.attachments : [], + ...(compacting && sourceMessage?.context !== undefined + ? { context: sourceMessage.context } + : {}), modelSelection: source.modelSelection, dispatchMode: { type: "start_immediately" }, createdBy: "agent", diff --git a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts index f43091c4e3e1..f4b901252862 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts @@ -263,7 +263,12 @@ function makeTestAdapter(input: { driver: input.driver, threadId: turnInput.threadId, providerThreadId: turnInput.providerThread.id, - text: turnInput.message.text, + text: + input.driver === CODEX_DRIVER && + turnInput.restartContinuationOfRunId !== undefined && + turnInput.message.text !== "/compact" + ? "" + : turnInput.message.text, attachments: turnInput.message.attachments, }, ]); @@ -1368,6 +1373,215 @@ describe("orchestration v2 provider switching", () => { }), ), ); + it.live.each([ + [CODEX_MODEL_SELECTION, false, true], + [CLAUDE_MODEL_SELECTION, false, true], + [CODEX_MODEL_SELECTION, true, true], + [CLAUDE_MODEL_SELECTION, true, true], + [CODEX_MODEL_SELECTION, false, false], + ] as const)( + "keeps the waiting request after a compaction restart with %j", + ([selection, failResume, runningWhileHeld]) => + Effect.scoped( + Effect.gen(function* () { + const cwd = yield* checkpointWorkspace(`compaction-restart-${selection.instanceId}`); + const databaseLayer = Layer.succeed(SqlClient.SqlClient, yield* SqlClient.SqlClient); + const capturedTurns = yield* Ref.make>([]); + const generation = yield* Ref.make(0); + const started = yield* Deferred.make(); + const restarted = yield* Deferred.make(); + const release = yield* Deferred.make(); + const request = "Use the diagram and keep the API unchanged."; + const attachments: ReadonlyArray = [ + { + type: "image", + id: "diagram", + name: "diagram.png", + mimeType: "image/png", + sizeBytes: 10, + }, + ]; + const driver = + selection.instanceId === CODEX_MODEL_SELECTION.instanceId + ? CODEX_DRIVER + : CLAUDE_DRIVER; + const adapter = { + instanceId: selection.instanceId, + driver, + capabilities: + driver === CODEX_DRIVER ? CodexProviderCapabilitiesV2 : ClaudeProviderCapabilitiesV2, + modelSelection: selection, + responseByRunOrdinal: {}, + capturedTurns, + nativeThreadGeneration: generation, + compaction: "success" as const, + getModelContextWindow: (model: ModelSelection) => + model.model === "small-model" ? 32_000 : 258_400, + runningWhileHeld, + }; + const scenario = { name: `compaction-restart-${driver}`, runtimePolicyOverride: { cwd } }; + yield* Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const daemon = yield* EffectWorker.runDaemon.pipe(Effect.forkScoped); + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make("restart:create"), + threadId, + projectId, + createdBy: "user", + creationSource: "web", + title: "Compaction restart", + modelSelection: selection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: null, + }); + const priorCompleted = yield* orchestrator.streamStoredEvents.pipe( + Stream.filter( + ({ event }) => + event.type === "run.updated" && event.payload.status === "completed", + ), + Stream.runHead, + Effect.forkScoped, + ); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("restart:prior"), + messageId: MessageId.make("restart:prior"), + threadId, + createdBy: "user", + creationSource: "web", + text: "Prior constraints", + attachments: [], + modelSelection: selection, + dispatchMode: { type: "start_immediately" }, + }); + yield* Fiber.join(priorCompleted); + yield* worker.drain(); + const running = yield* orchestrator.streamStoredEvents.pipe( + Stream.filter( + ({ event }) => + event.type === "provider-turn.updated" && + event.payload.ordinal === 2 && + event.payload.status === "running", + ), + Stream.runHead, + Effect.forkScoped, + ); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("restart:request"), + messageId: MessageId.make("restart:request"), + threadId, + createdBy: "user", + creationSource: "web", + text: request, + attachments, + modelSelection: { ...selection, model: "small-model" }, + dispatchMode: { type: "start_immediately" }, + }); + yield* Deferred.await(started); + if (runningWhileHeld) yield* Fiber.join(running); + const before = yield* orchestrator.getThreadProjection(threadId); + yield* Fiber.interrupt(daemon); + const source = before.runs.at(-1)!; + assert.isTrue( + before.attempts.find((attempt) => attempt.id === source.activeAttemptId) + ?.contextCompaction, + ); + const saved = before.turnItems.filter( + (item) => item.type === "user_message" || item.type === "assistant_message", + ); + const after = yield* Effect.scoped( + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + if (failResume) { + yield* worker.drain(); + return yield* orchestrator.getThreadProjection(threadId); + } + yield* Deferred.await(restarted); + const completed = yield* orchestrator.streamStoredEvents.pipe( + Stream.filter( + ({ event }) => + event.type === "run.updated" && + event.payload.restartContinuationOfRunId === source.id && + event.payload.status === "completed", + ), + Stream.runHead, + Effect.forkScoped, + ); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(completed); + yield* worker.drain(); + return yield* orchestrator.getThreadProjection(threadId); + }).pipe( + Effect.provide( + Layer.fresh( + makeOrchestratorV2ReplayLayerWithRegistry( + scenario, + ProviderAdapterRegistry.makeLayer([ + makeTestAdapter({ + ...adapter, + failResume, + holdRunOrdinal: 3, + holdFirstTurn: restarted, + releaseFirstTurn: release, + }), + ]), + { + databaseLayer, + recoverOnStartup: true, + continueThreadsAfterServerUpdate: true, + }, + ), + ), + ), + ), + ); + assert.equal(after.runs.find((run) => run.id === source.id)?.status, "cancelled"); + assert.equal(after.runs.at(-1)?.restartContinuationOfRunId, source.id); + assert.equal(after.runs.at(-1)?.status, failResume ? "failed" : "completed"); + assert.equal(yield* Ref.get(generation), 1); + const delivered = (yield* Ref.get(capturedTurns)).filter( + (turn) => turn.text === request, + ); + assert.lengthOf(delivered, failResume ? 0 : 1); + if (!failResume) { + assert.deepEqual(delivered[0]!.attachments, attachments); + assert.equal(delivered[0]!.providerThreadId, source.providerThreadId); + } + for (const item of saved) + assert.deepEqual( + after.turnItems.find((candidate) => candidate.id === item.id), + item, + ); + }).pipe( + Effect.provide( + Layer.fresh( + makeOrchestratorV2ReplayLayerWithRegistry( + scenario, + ProviderAdapterRegistry.makeLayer([ + makeTestAdapter({ + ...adapter, + tokenUsageByRunOrdinal: { 1: { usedTokens: 207_362, maxTokens: 258_400 } }, + holdRunOrdinal: 2, + holdFirstTurn: started, + }), + ]), + { databaseLayer, runEffectWorker: false }, + ), + ), + ), + ), + ); + }).pipe(Effect.provide(SqlitePersistenceMemory)), + ), + ); + it.live.each(["turn-start", "injection", "large-missed-request"] as const)( "recovers %s failure without duplicating history in the same native thread", (failure) => diff --git a/docs/user/portable-handoffs.md b/docs/user/portable-handoffs.md index d3a24e3ee395..c9807dcdbfac 100644 --- a/docs/user/portable-handoffs.md +++ b/docs/user/portable-handoffs.md @@ -30,11 +30,15 @@ capacity is unknown. This recovery requires native compaction support and stops already sent an answer, started a tool, or delegated work. No encrypted state is copied between providers. -If compaction is stopped, the run stays stopped and the original request is not sent again. If +If you stop compaction, the run stays stopped and the original request is not sent again. If compaction is unsupported, fails, or still leaves too little room, T3 Code reports the failure without silently replacing the session or shortening your request. Reduce the request or select a larger-context model before trying again. +If the server restarts during automatic compaction, **Settings → General → Continue threads after +restarts** controls whether T3 Code continues with your saved request and attachments. This setting +is off by default. + Server operators can set `T3CODE_CONTEXT_HANDOFF_TOKEN_CAP` to change the initial history allowance (default 16,000; clamped to 1,024–64,000). This is an upper bound, not a provider context-window guarantee. Text accounting conservatively charges one token per UTF-8 byte, including attribution From ed7d59f126f06091cbf190add0547cdca057f198 Mon Sep 17 00:00:00 2001 From: Bil0000 <62337003+Bil0000@users.noreply.github.com> Date: Sun, 4 Oct 2026 16:25:34 +0200 Subject: [PATCH 09/10] fix(v2): keep compaction recovery in its native session --- .../ProviderTurnStartService.ts | 8 ++- .../RestartContinuation.test.ts | 59 ++++++++++++++----- .../orchestration-v2/RestartContinuation.ts | 17 +++--- .../ProviderSwitch.integration.test.ts | 16 ++++- 4 files changed, 73 insertions(+), 27 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 52f5dbc8b87f..acb98a18fd74 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -1045,7 +1045,13 @@ export const layer: Layer.Layer< } if (!(yield* isCurrentAttemptInStatus("starting"))) return providerThread; - if (contextCompaction || compactionContinuation) { + if ( + contextCompaction || + compactionContinuation || + projection.attempts.some( + (candidate) => candidate.runId === run.id && candidate.contextCompaction, + ) + ) { yield* settleStartFailure({ signal: "provider-thread-load-failure", title: "Provider turn failed to start", diff --git a/apps/server/src/orchestration-v2/RestartContinuation.test.ts b/apps/server/src/orchestration-v2/RestartContinuation.test.ts index ce899d3eb91b..5eac0fd45822 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.test.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.test.ts @@ -660,6 +660,7 @@ const queuedFollowUp = { it.effect.each([ "resume", "prepared-resume", + "request-after-compaction", "started-continuation", "stopped", "disabled", @@ -682,32 +683,53 @@ it.effect.each([ }; const projection = { ...cutMidTurn(), - ...(["prepared-resume", "started-continuation"].includes(scenario) + ...(scenario === "request-after-compaction" ? { - runs: [ - { - ...cutMidTurn().runs[0]!, - id: RunId.make("run:compaction-source"), - ordinal: 0, - activeAttemptId: RunAttemptId.make("attempt:compaction-source"), - }, + runs: cutMidTurn().runs, + attempts: [ { - ...cutMidTurn().runs[0]!, - restartContinuationOfRunId: RunId.make("run:compaction-source"), + id: RunAttemptId.make("attempt:compaction-source"), + runId, + contextCompaction: true, }, + { id: attemptId, runId, contextCompaction: false }, ], - attempts: [ - { id: RunAttemptId.make("attempt:compaction-source"), contextCompaction: true }, - ], - providerTurns: scenario === "prepared-resume" ? [] : cutMidTurn().providerTurns, + providerTurns: [], } - : { attempts: [{ id: attemptId, contextCompaction: true }] }), + : ["prepared-resume", "started-continuation"].includes(scenario) + ? { + runs: [ + { + ...cutMidTurn().runs[0]!, + id: RunId.make("run:compaction-source"), + ordinal: 0, + activeAttemptId: RunAttemptId.make("attempt:compaction-source"), + }, + { + ...cutMidTurn().runs[0]!, + restartContinuationOfRunId: RunId.make("run:compaction-source"), + }, + ], + attempts: [ + { id: RunAttemptId.make("attempt:compaction-source"), contextCompaction: true }, + ], + providerTurns: scenario === "prepared-resume" ? [] : cutMidTurn().providerTurns, + } + : { attempts: [{ id: attemptId, contextCompaction: true }] }), messages: scenario === "missing-request" ? [] : [message], turnItems: scenario === "stopped" ? [{ id: "turn-item:interrupt", runId, type: "run_interrupt_request" }] : [], } as unknown as OrchestrationV2ThreadProjection; + if (scenario === "request-after-compaction") + assert.equal( + restartContinuationRun({ + ...projection, + runs: [{ ...projection.runs[0]!, status: "starting" }], + })?.id, + runId, + ); const commands: Parameters< ThreadManagementService.ThreadManagementService["Service"]["dispatch"] >[0][] = []; @@ -726,7 +748,12 @@ it.effect.each([ ), ), ); - const resumed = ["resume", "prepared-resume", "started-continuation"].includes(scenario); + const resumed = [ + "resume", + "prepared-resume", + "request-after-compaction", + "started-continuation", + ].includes(scenario); assert.lengthOf(commands, resumed ? 1 : 0); if (resumed) { const command = commands[0]!; diff --git a/apps/server/src/orchestration-v2/RestartContinuation.ts b/apps/server/src/orchestration-v2/RestartContinuation.ts index e777a4b4d8ca..250a3d0efc26 100644 --- a/apps/server/src/orchestration-v2/RestartContinuation.ts +++ b/apps/server/src/orchestration-v2/RestartContinuation.ts @@ -41,8 +41,10 @@ export function restartContinuationRun( undefined, ); if (!run) return; + const pendingCompactionRequest = hasPendingCompactionRequest(run, projection); const preparedContinuation = - run.status === "starting" && run.restartContinuationOfRunId !== undefined; + run.status === "starting" && + (run.restartContinuationOfRunId !== undefined || pendingCompactionRequest); if (run.status !== "running" && !preparedContinuation) return; const liveTurnRequired = !preparedContinuation; if (projection.thread.providerInstanceId !== run.providerInstanceId) return; @@ -76,7 +78,7 @@ export function restartContinuationRun( return; if ( liveTurnRequired && - !hasPendingCompactionRequest(run, projection) && + !pendingCompactionRequest && !projection.providerTurns.some( (turn) => turn.providerThreadId === providerThread.id && @@ -94,17 +96,16 @@ export function hasPendingCompactionRequest( ): boolean { let current = source; while (current !== undefined) { - const attemptId = current.activeAttemptId; + const { id: runId, activeAttemptId: attemptId } = current; if ( projection.attempts.some((attempt) => attempt.id === attemptId && attempt.contextCompaction) ) return true; + if (projection.providerTurns.some((turn) => turn.runAttemptId === attemptId)) return false; + if (projection.attempts.some((attempt) => attempt.runId === runId && attempt.contextCompaction)) + return true; const sourceId = current.restartContinuationOfRunId; - if ( - sourceId === undefined || - projection.providerTurns.some((turn) => turn.runAttemptId === attemptId) - ) - return false; + if (sourceId === undefined) return false; current = projection.runs.find((run) => run.id === sourceId); } return false; diff --git a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts index f4b901252862..9c513a2413d4 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts @@ -123,6 +123,7 @@ function makeTestAdapter(input: { readonly workBeforeRejection?: boolean; readonly rejectContextBeforeStart?: boolean; readonly failCompactionResume?: boolean; + readonly failPostCompactionResume?: boolean; readonly responseBeforeRejection?: "completed" | "streaming" | "streaming-message"; readonly getModelContextWindow?: (selection: ModelSelection) => number | undefined; readonly canReuseContextUsage?: ProviderAdapterV2SessionRuntime["canReuseContextUsage"]; @@ -325,6 +326,12 @@ function makeTestAdapter(input: { input.failResumeOnce !== undefined ) yield* Ref.set(input.failResumeOnce, true); + if ( + compacting && + input.failPostCompactionResume && + input.failResumeOnce !== undefined + ) + yield* Ref.set(input.failResumeOnce, true); if (contextRejected && input.workBeforeRejection) yield* PubSub.publish(events, { type: "turn_item.updated", @@ -534,6 +541,7 @@ describe("orchestration v2 provider switching", () => { "exhausted-rejected-repeat-native", "exhausted-rejected-start-error-native", "exhausted-rejected-resume-failure-native", + "exhausted-rejected-resume-failure-after-compaction-native", "exhausted-rejected-unsupported-native", "exhausted-rejected-work-native", "exhausted-rejected-response-native", @@ -672,7 +680,10 @@ describe("orchestration v2 provider switching", () => { repeatContextRejection: scenario.includes("rejected-repeat"), workBeforeRejection: scenario.includes("rejected-work"), rejectContextBeforeStart: scenario.includes("rejected-start-error"), - failCompactionResume: scenario.includes("rejected-resume-failure"), + failCompactionResume: + scenario.includes("rejected-resume-failure") && + !scenario.includes("after-compaction"), + failPostCompactionResume: scenario.includes("after-compaction"), ...(scenario.includes("rejected-response") ? { responseBeforeRejection: scenario.includes("response-streaming-message") @@ -1188,7 +1199,8 @@ describe("orchestration v2 provider switching", () => { scenario === "exhausted-effort-change-native" || scenario === "exhausted-unknown-capacity-native" || scenario.includes("unsupported") || - scenario.includes("rejected-resume-failure") || + (scenario.includes("rejected-resume-failure") && + !scenario.includes("after-compaction")) || scenario.includes("rejected-response") || scenario.includes("rejected-work") ? 0 From 3ae2a0323cad36d3f97115eaf23cecbc86b2afd1 Mon Sep 17 00:00:00 2001 From: Bil0000 Date: Sun, 4 Oct 2026 17:36:41 +0300 Subject: [PATCH 10/10] docs: clarify compaction restart recovery --- docs/user/portable-handoffs.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/user/portable-handoffs.md b/docs/user/portable-handoffs.md index c9807dcdbfac..6312c70db202 100644 --- a/docs/user/portable-handoffs.md +++ b/docs/user/portable-handoffs.md @@ -35,9 +35,10 @@ compaction is unsupported, fails, or still leaves too little room, T3 Code repor without silently replacing the session or shortening your request. Reduce the request or select a larger-context model before trying again. -If the server restarts during automatic compaction, **Settings → General → Continue threads after -restarts** controls whether T3 Code continues with your saved request and attachments. This setting -is off by default. +If the server restarts during automatic compaction or after it finishes but before the retry, +**Settings → General → Continue threads after restarts** controls whether T3 Code continues with +your saved request and attachments. This setting is off by default. When it is off, the run does +not continue; send the request again after the restart. Server operators can set `T3CODE_CONTEXT_HANDOFF_TOKEN_CAP` to change the initial history allowance (default 16,000; clamped to 1,024–64,000). This is an upper bound, not a provider context-window