fix(server): resume Claude turns after gateway quota recovers

apps/mobile/src/features/threads/UsageLimitRecoveryCard.tsx

@@ -21,7 +21,8 @@ export function UsageLimitRecoveryCard({
   const updateMetadata = useAtomCommand(threadEnvironment.updateMetadata);
   const [pending, setPending] = useState(false);
   const [error, setError] = useState<string | null>(null);
-  const resetAt = thread.runtime?.usageLimitResetAt ?? null;
+  const knownResetAt = thread.runtime?.usageLimitResetAt ?? null;
+  const resetAt = knownResetAt ?? thread.runtime?.usageLimitRetryAt ?? null;
   const canSchedule =
     resetAt !== null &&
     Date.parse(resetAt) > Date.parse(thread.latestRun?.completedAt ?? thread.updatedAt);
@@ -72,19 +73,27 @@ export function UsageLimitRecoveryCard({
     <View className="mx-3 mb-2 gap-2 rounded-xl border border-warning-foreground/25 bg-screen p-3">
       <Text className="text-sm text-warning-foreground">
         {resetAt
-          ? `Usage limit resets ${DateTime.toDateUtc(DateTime.makeUnsafe(resetAt)).toLocaleString()}.`
+          ? `${knownResetAt ? "Usage limit resets" : scheduled ? "Waiting for quota; retrying" : "Next retry"} ${DateTime.toDateUtc(DateTime.makeUnsafe(resetAt)).toLocaleString()}.`
           : "The provider did not report a reset time. Retry manually when your limit is available."}
       </Text>
       {canSchedule ? (
         <View className="flex-row flex-wrap gap-2">
           <Button
-            label={scheduled ? "Cancel auto-resume" : "Resume at reset"}
+            label={
+              scheduled
+                ? "Cancel auto-resume"
+                : knownResetAt
+                  ? "Resume at reset"
+                  : "Resume automatically"
+            }
             tone="secondary"
             disabled={pending}
             onPress={() => void toggle("resume")}
           />
           <Button
-            label={snoozed ? "Wake now" : "Snooze until reset"}
+            label={
+              snoozed ? "Wake now" : knownResetAt ? "Snooze until reset" : "Snooze until retry"
+            }
             tone="secondary"
             disabled={pending || (!snoozed && resetPassed)}
             onPress={() => void toggle("snooze")}

apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts

@@ -5808,6 +5808,122 @@ describe("ClaudeAdapterV2 background wake turns", () => {
     ),
   );
 
+  it.effect.each([
+    { status: 503, code: "claude_quota_exhausted" },
+    { status: 503, code: "claude_quota_unavailable" },
+    { status: 429, code: "claude_quota_exhausted" },
+    { status: 429, code: "claude_quota_exhausted", assistantEvidence: true },
+  ])("schedules gateway quota recovery for $code ($status)", (input) =>
+    Effect.gen(function* () {
+      const harness = yield* makeWakeHarness;
+      const now = yield* DateTime.now;
+      yield* harness.runtime.startTurn(
+        makeClaudeTestTurnInput({
+          threadId: harness.threadId,
+          providerThread: harness.providerThread,
+          now,
+          attemptId: RunAttemptId.make("attempt-gateway-quota"),
+          text: "Continue.",
+          attachments: [],
+        }),
+      );
+      if (input.assistantEvidence) {
+        yield* Queue.offer(
+          harness.sdkMessages,
+          makeAssistantErrorFrame({
+            uuid: "00000000-0000-4000-8000-000000000653",
+            error: "rate_limit",
+          }),
+        );
+      }
+      yield* Queue.offer(
+        harness.sdkMessages,
+        makeResultFrame({
+          uuid: "00000000-0000-4000-8000-000000000654",
+          result: input.assistantEvidence
+            ? `API Error: ${input.status} ${encodeJsonString({ error: { code: input.code, retryable: true, retry_after_seconds: 60 } })}`
+            : `API Error: ${input.status} ${input.code}: No Claude account has quota for this model`,
+          terminalReason: "api_error",
+          isError: true,
+          apiErrorStatus: input.status,
+        }),
+      );
+      const terminal = yield* Queue.take(harness.terminalReceipts);
+      assert.equal(terminal.status, "failed");
+      if (terminal.status !== "failed") return;
+      assert.equal(terminal.failure.class, "usage_limit");
+      assert.equal(terminal.failure.retryable, true);
+      assert.equal(terminal.failure.code, input.code);
+      assert.notExists(terminal.failure.resetAt);
+      assert.isAtLeast(Date.parse(terminal.failure.retryAt!), DateTime.toEpochMillis(now) + 60_000);
+      assert.isAtMost(Date.parse(terminal.failure.retryAt!), DateTime.toEpochMillis(now) + 72_000);
+      const updated = harness.events.findLast((event) => event.type === "provider_thread.updated");
+      assert.equal(
+        updated?.type === "provider_thread.updated"
+          ? updated.providerThread.nativeMetadata?.claudeQuotaRetryCount
+          : null,
+        1,
+      );
+    }).pipe(
+      Effect.provide(
+        Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer),
+      ),
+    ),
+  );
+
+  it.effect("restores quota backoff in a fresh adapter and clears it after success", () =>
+    Effect.gen(function* () {
+      let restored: OrchestrationV2ProviderThread | undefined;
+      let previousDelay = 0;
+      for (const attempt of [0, 1, 2]) {
+        const harness = yield* makeWakeHarness;
+        const now = yield* DateTime.now;
+        yield* harness.runtime.startTurn(
+          makeClaudeTestTurnInput({
+            threadId: harness.threadId,
+            providerThread: restored ?? harness.providerThread,
+            now,
+            attemptId: RunAttemptId.make(`quota-restored-${attempt}`),
+            text: "Continue.",
+            attachments: [],
+          }),
+        );
+        yield* Queue.offer(
+          harness.sdkMessages,
+          makeResultFrame({
+            uuid: `quota-restored-result-${attempt}`,
+            result: attempt === 2 ? "Finished." : "API Error: 503 claude_quota_exhausted: No quota",
+            ...(attempt === 2
+              ? {}
+              : { terminalReason: "api_error", isError: true, apiErrorStatus: 503 }),
+          }),
+        );
+        const terminal = yield* Queue.take(harness.terminalReceipts);
+        const updated = harness.events.findLast(
+          (event) => event.type === "provider_thread.updated",
+        );
+        assert(updated?.type === "provider_thread.updated");
+        restored = yield* decodeProviderThreadJson(
+          yield* encodeProviderThreadJson(updated.providerThread),
+        );
+        if (attempt < 2) {
+          assert(terminal.status === "failed");
+          const delay = Date.parse(terminal.failure.retryAt!) - DateTime.toEpochMillis(now);
+          assert.isAbove(delay, previousDelay);
+          assert.equal(restored.nativeMetadata?.claudeQuotaRetryCount, attempt + 1);
+          previousDelay = delay;
+        } else {
+          assert.equal(terminal.status, "completed");
+          assert.equal(restored.nativeMetadata?.claudeQuotaRetryCount, 0);
+        }
+      }
+    }).pipe(
+      Effect.provide(
+        Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer),
+      ),
+    ),
+  );
+
   it.effect("keeps the measured context when a usage limit ends the turn", () =>
     Effect.gen(function* () {
       const harness = yield* makeWakeHarness;

apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts

@@ -1,3 +1,4 @@
+import { claudeGatewayQuotaFailure } from "./claudeGatewayQuota.ts";
 import * as DocumentText from "../../provider/DocumentText.ts";
 import { makeProviderTextDeltaCoalescer } from "./ProviderTextDeltaCoalescer.ts";
 import { claudeTaskOutputRoot, tailClaudeTaskOutput } from "./ClaudeTaskOutputTail.ts";
@@ -3644,7 +3645,24 @@ function providerFailureFromResult(
   completedAt: DateTime.Utc,
   failureHint?: string,
   usageLimited = false,
+  previousAttempts: unknown = 0,
+  identity = "",
 ): OrchestrationV2ProviderFailure | null {
+  if (
+    (failureHint === undefined || usageLimited) &&
+    message.subtype === "success" &&
+    message.is_error &&
+    message.terminal_reason === "api_error"
+  ) {
+    const quota = claudeGatewayQuotaFailure({
+      result: message.result,
+      status: message.api_error_status,
+      completedAt,
+      previousAttempts,
+      identity,
+    });
+    if (quota !== null) return quota;
+  }
   const gatewayLimit =
     failureHint === undefined &&
     message.subtype === "success" &&
@@ -7457,6 +7475,21 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
                 const providerThread: OrchestrationV2ProviderThread = {
                   ...input.context.input.providerThread,
                   providerSessionId: session.id,
+                  nativeMetadata: {
+                    ...input.context.input.providerThread.nativeMetadata,
+                    claudeQuotaRetryCount:
+                      input.failure?.code === "claude_quota_exhausted" ||
+                      input.failure?.code === "claude_quota_unavailable"
+                        ? Math.min(
+                            5,
+                            (typeof input.context.input.providerThread.nativeMetadata
+                              ?.claudeQuotaRetryCount === "number"
+                              ? input.context.input.providerThread.nativeMetadata
+                                  .claudeQuotaRetryCount
+                              : 0) + 1,
+                          )
+                        : 0,
+                  },
                   ...(clearConversationHead ? { nativeConversationHeadRef: null } : {}),
                   ...(nativeThreadId !== null && goalsByNativeThread.has(nativeThreadId)
                     ? { goal: goalsByNativeThread.get(nativeThreadId) ?? null }
@@ -9216,7 +9249,14 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
                     message: limitedCompaction.message ?? failureHint,
                     code: "compaction_failed",
                   })
-                : providerFailureFromResult(message, completedAt, failureHint, usageLimited);
+                : providerFailureFromResult(
+                    message,
+                    completedAt,
+                    failureHint,
+                    usageLimited,
+                    context.input.providerThread.nativeMetadata?.claudeQuotaRetryCount,
+                    context.input.runId,
+                  );
             const terminalFailure =
               resultFailure?.class === "usage_limit"
                 ? { ...resultFailure, resetAt: resetAt ?? resultFailure.resetAt ?? null }

apps/server/src/orchestration-v2/Adapters/claudeGatewayQuota.test.ts

@@ -0,0 +1,56 @@
+import { describe, expect, it } from "vite-plus/test";
+import * as DateTime from "effect/DateTime";
+import { claudeGatewayQuotaFailure } from "./claudeGatewayQuota.ts";
+
+const completedAt = DateTime.makeUnsafe("2026-10-29T12:00:00Z");
+const nowMs = DateTime.toEpochMillis(completedAt);
+const parse = (error: unknown, previousAttempts: unknown = 0) =>
+  claudeGatewayQuotaFailure({
+    result: `API Error: 429 ${JSON.stringify({ error })}`,
+    status: 429,
+    completedAt,
+    previousAttempts,
+    identity: "run-1",
+  });
+
+describe("claudeGatewayQuotaFailure", () => {
+  it.each([
+    { reset_at: "2026-10-29T13:00:00Z", expectedReset: "2026-10-29T13:00:00.000Z" },
+    { reset_at: "expired", retry_after_seconds: 120, expectedDelay: 120 },
+    { reset_at: "2026-10-29T11:00:00Z", retry_after_seconds: 90, expectedDelay: 90 },
+    { retry_after_seconds: 300, expectedDelay: 300 },
+  ])("separates known resets from retry deadlines ($reset_at)", (input) => {
+    const failure = parse({ code: "claude_quota_exhausted", retryable: true, ...input });
+    expect(failure?.code).toBe("claude_quota_exhausted");
+    if (input.expectedReset) {
+      expect(failure?.resetAt).toBe(input.expectedReset);
+      expect(failure?.retryAt).toBeUndefined();
+    } else {
+      expect(failure?.resetAt).toBeUndefined();
+      expect(Date.parse(failure!.retryAt!) - nowMs).toBe(input.expectedDelay! * 1_000);
+    }
+  });
+
+  it.each([0, 1, 4, 5, 100, -1, "invalid", NaN])(
+    "bounds fallback backoff for %s attempts",
+    (attempts) => {
+      const error = { code: "claude_quota_unavailable", retry_after_seconds: -1 };
+      const failure = parse(error, attempts);
+      const delay = Date.parse(failure!.retryAt!) - nowMs;
+      expect(delay).toBeGreaterThanOrEqual(60_000);
+      expect(delay).toBeLessThanOrEqual(900_000);
+      expect(parse(error, attempts)).toEqual(failure);
+      if (attempts === 1)
+        expect(delay).toBeGreaterThan(Date.parse(parse(error, 0)!.retryAt!) - nowMs);
+    },
+  );
+
+  it.each([
+    { code: "authentication_failed" },
+    { code: "claude_quota_exhausted", retryable: false },
+    { message: "An unrelated error mentions claude_quota_exhausted" },
+    null,
+  ])("leaves unrelated or non-retryable errors unchanged (%j)", (error) => {
+    expect(parse(error)).toBeNull();
+  });
+});

apps/server/src/orchestration-v2/Adapters/claudeGatewayQuota.ts

@@ -0,0 +1,64 @@
+import { field } from "../unknownField.ts";
+import * as DateTime from "effect/DateTime";
+import { makeProviderFailure } from "@t3tools/provider-core/server/failure";
+
+export function claudeGatewayQuotaFailure(input: {
+  readonly result: string;
+  readonly status: number | null | undefined;
+  readonly completedAt: DateTime.Utc;
+  readonly previousAttempts: unknown;
+  readonly identity: string;
+}) {
+  if (input.status !== 429 && input.status !== 503) return null;
+  const body = /^API Error: (?:429|503)\s+([\s\S]+)$/.exec(input.result)?.[1];
+  if (body === undefined) return null;
+  let code: unknown;
+  let reset: unknown;
+  let retryAfter: unknown;
+  try {
+    const parsed: unknown = JSON.parse(body);
+    const error = field(parsed, "error") ?? parsed;
+    code = field(error, "code");
+    const message = field(error, "message");
+    if (code === undefined && typeof message === "string") {
+      code = /^(claude_quota_exhausted|claude_quota_unavailable):/.exec(message)?.[1];
+    }
+    if (field(error, "retryable") === false) return null;
+    reset = field(error, "reset_at");
+    retryAfter = field(error, "retry_after_seconds");
+  } catch {
+    code = /^(claude_quota_exhausted|claude_quota_unavailable):/.exec(body)?.[1];
+  }
+  if (code !== "claude_quota_exhausted" && code !== "claude_quota_unavailable") return null;
+  const now = DateTime.toEpochMillis(input.completedAt);
+  const resetMs = typeof reset === "string" ? Date.parse(reset) : NaN;
+  const attempts =
+    typeof input.previousAttempts === "number" && Number.isSafeInteger(input.previousAttempts)
+      ? Math.max(0, Math.min(4, input.previousAttempts))
+      : 0;
+  const delay = Math.min(750, 60 * 2 ** attempts);
+  // Stable jitter keeps a persisted retry unchanged across process restarts.
+  const hash = Array.from(input.identity).reduce(
+    (value, char) => (value * 31 + char.charCodeAt(0)) >>> 0,
+    0,
+  );
+  const seconds =
+    typeof retryAfter === "number" &&
+    Number.isFinite(retryAfter) &&
+    retryAfter > 0 &&
+    retryAfter <= 604_800
+      ? retryAfter
+      : delay + (hash % (Math.floor(delay / 5) + 1));
+  return makeProviderFailure({
+    class: "usage_limit",
+    code,
+    retryable: true,
+    message:
+      code === "claude_quota_exhausted"
+        ? "Waiting for Claude quota."
+        : "Waiting for Claude quota availability to refresh.",
+    ...(Number.isFinite(resetMs) && resetMs > now && resetMs <= 253402300799000
+      ? { resetAt: DateTime.formatIso(DateTime.makeUnsafe(resetMs)) }
+      : { retryAt: DateTime.formatIso(DateTime.makeUnsafe(now + seconds * 1_000)) }),
+  });
+}