diff --git a/packages/client-runtime/src/rpc/client.ts b/packages/client-runtime/src/rpc/client.ts index c364b3448d03d..743e2b5f48168 100644 --- a/packages/client-runtime/src/rpc/client.ts +++ b/packages/client-runtime/src/rpc/client.ts @@ -108,6 +108,11 @@ const isEnvironmentAuthorizationError = Schema.is(EnvironmentAuthorizationError) const MAX_EXPECTED_FAILURE_RETRY_DELAY_MS = 30_000; +export const subscriptionRetryDelay = (base: Duration.Input, retries: number) => + Duration.millis( + Math.min(Duration.toMillis(base) * 2 ** retries, MAX_EXPECTED_FAILURE_RETRY_DELAY_MS), + ); + export type EnvironmentRpcInput = Parameters>[0]; export type EnvironmentRpcSuccess = @@ -358,12 +363,9 @@ function subscribeDynamicMapped( ) { return handled; } - const retryDelay = Duration.millis( - Math.min( - Duration.toMillis(options.retryExpectedFailureAfter) * - 2 ** expectedFailureRetries, - MAX_EXPECTED_FAILURE_RETRY_DELAY_MS, - ), + const retryDelay = subscriptionRetryDelay( + options.retryExpectedFailureAfter, + expectedFailureRetries, ); expectedFailureRetries += 1; return handled.pipe( diff --git a/packages/client-runtime/src/state/threads-sync.test.ts b/packages/client-runtime/src/state/threads-sync.test.ts index 3c1c153621e28..fe08d6094485d 100644 --- a/packages/client-runtime/src/state/threads-sync.test.ts +++ b/packages/client-runtime/src/state/threads-sync.test.ts @@ -13,6 +13,7 @@ import { type OrchestrationV2TurnItem, } from "@t3tools/contracts"; import { describe, expect, it } from "@effect/vitest"; +import * as Cause from "effect/Cause"; import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; @@ -78,7 +79,7 @@ const BASE_PROJECTION: OrchestrationV2ThreadProjection = { thread: { ...v2Projection.thread, title: "Cached thread" }, }; -type TestThreadInput = OrchestrationV2ThreadStreamItem | Error; +type TestThreadInput = OrchestrationV2ThreadStreamItem | Error | Cause.Cause; function testSession( client: WsRpcProtocolClient, @@ -147,7 +148,11 @@ const makeHarness = Effect.fn("TestEnvironmentThreads.makeHarness")(function* (o const streamFrom = (queue: Queue.Queue) => Stream.fromQueue(queue).pipe( Stream.mapEffect((input) => - input instanceof Error ? Effect.fail(input) : Effect.succeed(input), + Cause.isCause(input) + ? Effect.failCause(input) + : input instanceof Error + ? Effect.fail(input) + : Effect.succeed(input), ), ); const client = { @@ -1803,6 +1808,46 @@ describe("EnvironmentThreads", () => { }), ); + it.effect( + "resubscribes after a snapshot decode defect ends the thread subscription (#23041)", + () => + Effect.gen(function* () { + const harness = yield* makeHarness({ httpSnapshot: { _tag: "unavailable" } }); + yield* SubscriptionRef.changes(harness.subscriptionCount).pipe( + Stream.filter((count) => count >= 1), + Stream.runHead, + ); + yield* Queue.offer( + harness.inputs, + Cause.die(new Error("Could not decode thread snapshot")), + ); + const failed = yield* awaitThreadState(harness.observed, (value) => + Option.isSome(value.error), + ); + expect(failed.status).toBe("empty"); + expect(Option.getOrThrow(failed.error)).toBe("Could not synchronize the thread."); + + yield* TestClock.adjust("60 seconds"); + expect(yield* SubscriptionRef.get(harness.subscriptionCount)).toBe(2); + + yield* Queue.offer( + harness.inputs, + snapshot({ + ...BASE_PROJECTION, + thread: { ...BASE_PROJECTION.thread, title: "Recovered thread" }, + }), + ); + const recovered = yield* awaitThreadState( + harness.observed, + (value) => + value.status === "live" && + Option.isSome(value.data) && + value.data.value.thread.title === "Recovered thread", + ); + expect(Option.isNone(recovered.error)).toBe(true); + }), + ); + it.effect("keeps replayed updates synchronizing until the completion marker arrives", () => Effect.gen(function* () { const harness = yield* makeHarness({ cached: BASE_PROJECTION, completionMarker: true }); diff --git a/packages/client-runtime/src/state/threads.ts b/packages/client-runtime/src/state/threads.ts index 693af5329fe1b..a53cca16134b2 100644 --- a/packages/client-runtime/src/state/threads.ts +++ b/packages/client-runtime/src/state/threads.ts @@ -32,7 +32,7 @@ import * as ConnectionWakeups from "../connection/wakeups.ts"; import * as Persistence from "../platform/persistence.ts"; import { runCachePersistence } from "./cachePersistence.ts"; import * as ManagedRelay from "../relay/managedRelay.ts"; -import { request, subscribeDynamic } from "../rpc/client.ts"; +import { request, subscribeDynamic, subscriptionRetryDelay } from "../rpc/client.ts"; import { parseThreadKey, threadKey } from "./entities.ts"; import { applyOrchestrationV2ProjectionEvent } from "./orchestrationV2Projection.ts"; import { THREAD_SNAPSHOT_IDLE_TTL_MS } from "./threadRetention.ts"; @@ -55,6 +55,8 @@ import { type EnvironmentThreadStatus, } from "./threadState.ts"; +const THREAD_SUBSCRIPTION_RETRY_DELAY = "250 millis"; + function statusWithoutLiveData( data: Option.Option, ): EnvironmentThreadStatus { @@ -878,7 +880,12 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make }); yield* markSynchronizing; - yield* Effect.forkScoped( + let defectRetries = 0; + const followThread = (): Effect.Effect< + void, + never, + EnvironmentSupervisor.EnvironmentSupervisor + > => subscribeDynamic( ORCHESTRATION_V2_WS_METHODS.subscribeThread, Effect.fn("EnvironmentThreadState.makeSubscribeInput")(function* (session) { @@ -982,11 +989,37 @@ export const makeEnvironmentThreadState = Effect.fn("EnvironmentThreadState.make { onDefect: () => setStreamError("Could not synchronize the thread."), onExpectedFailure: (cause) => setStreamError(formatThreadError(cause)), - retryExpectedFailureAfter: "250 millis", + retryExpectedFailureAfter: THREAD_SUBSCRIPTION_RETRY_DELAY, resubscribe: foregroundResubscriptions, }, - ).pipe(Stream.runForEachArray(applyItems)), - ); + ).pipe( + Stream.runForEachArray((items) => + applyItems(items).pipe( + Effect.andThen( + Effect.sync(() => { + defectRetries = 0; + }), + ), + ), + ), + // A defect (an undecodable snapshot, or one applyItems cannot handle) ends the + // subscription for good; resubscribe the way reselecting the thread would. + Effect.catchCause((cause) => { + if (Cause.hasInterrupts(cause)) return Effect.interrupt; + const retryDelay = subscriptionRetryDelay(THREAD_SUBSCRIPTION_RETRY_DELAY, defectRetries); + defectRetries += 1; + return Effect.logWarning("Thread subscription failed; resubscribing.", { + cause: Cause.pretty(cause), + environmentId, + threadId, + }).pipe( + Effect.andThen(setStreamError("Could not synchronize the thread.")), + Effect.andThen(Effect.sleep(retryDelay)), + Effect.andThen(Effect.suspend(followThread)), + ); + }), + ); + yield* Effect.forkScoped(followThread()); return state; });