Add gmail retries in more places

apps/web/utils/gmail/draft.ts

@@ -3,16 +3,19 @@ import { createScopedLogger } from "@/utils/logger";
 import { parseMessage } from "@/utils/gmail/message";
 import type { MessageWithPayload } from "@/utils/types";
 import { isGmailError } from "@/utils/error";
+import { withGmailRetry } from "@/utils/gmail/retry";
 
 const logger = createScopedLogger("gmail/draft");
 
 export async function getDraft(draftId: string, gmail: gmail_v1.Gmail) {
   try {
-    const response = await gmail.users.drafts.get({
-      userId: "me",
-      id: draftId,
-      format: "full",
-    });
+    const response = await withGmailRetry(() =>
+      gmail.users.drafts.get({
+        userId: "me",
+        id: draftId,
+        format: "full",
+      }),
+    );
     const message = parseMessage(response.data.message as MessageWithPayload);
     return message;
   } catch (error) {
@@ -24,10 +27,12 @@ export async function getDraft(draftId: string, gmail: gmail_v1.Gmail) {
 export async function deleteDraft(gmail: gmail_v1.Gmail, draftId: string) {
   try {
     logger.info("Deleting draft", { draftId });
-    const response = await gmail.users.drafts.delete({
-      userId: "me",
-      id: draftId,
-    });
+    const response = await withGmailRetry(() =>
+      gmail.users.drafts.delete({
+        userId: "me",
+        id: draftId,
+      }),
+    );
     if (response.status !== 200 && response.status !== 204) {
       logger.error("Failed to delete draft", { draftId, response });
     }

apps/web/utils/gmail/label.ts

@@ -85,14 +85,16 @@ export async function archiveThread({
   actionSource: TinybirdEmailAction["actionSource"];
   labelId?: string;
 }) {
-  const archivePromise = gmail.users.threads.modify({
-    userId: "me",
-    id: threadId,
-    requestBody: {
-      removeLabelIds: [GmailLabel.INBOX],
-      ...(labelId ? { addLabelIds: [labelId] } : {}),
-    },
-  });
+  const archivePromise = withGmailRetry(() =>
+    gmail.users.threads.modify({
+      userId: "me",
+      id: threadId,
+      requestBody: {
+        removeLabelIds: [GmailLabel.INBOX],
+        ...(labelId ? { addLabelIds: [labelId] } : {}),
+      },
+    }),
+  );
 
   const publishPromise = publishArchive({
     ownerEmail,
@@ -137,11 +139,13 @@ export async function labelMessage({
   addLabelIds?: string[];
   removeLabelIds?: string[];
 }) {
-  return gmail.users.messages.modify({
-    userId: "me",
-    id: messageId,
-    requestBody: { addLabelIds, removeLabelIds },
-  });
+  return withGmailRetry(() =>
+    gmail.users.messages.modify({
+      userId: "me",
+      id: messageId,
+      requestBody: { addLabelIds, removeLabelIds },
+    }),
+  );
 }
 
 export async function markReadThread(options: {
@@ -151,17 +155,19 @@ export async function markReadThread(options: {
 }) {
   const { gmail, threadId, read } = options;
 
-  return gmail.users.threads.modify({
-    userId: "me",
-    id: threadId,
-    requestBody: read
-      ? {
-          removeLabelIds: [GmailLabel.UNREAD],
-        }
-      : {
-          addLabelIds: [GmailLabel.UNREAD],
-        },
-  });
+  return withGmailRetry(() =>
+    gmail.users.threads.modify({
+      userId: "me",
+      id: threadId,
+      requestBody: read
+        ? {
+            removeLabelIds: [GmailLabel.UNREAD],
+          }
+        : {
+            addLabelIds: [GmailLabel.UNREAD],
+          },
+    }),
+  );
 }
 
 export async function createLabel({

apps/web/utils/gmail/mail.ts

@@ -112,13 +112,15 @@ export async function sendEmailWithHtml(
   }
 
   const raw = await createRawMailMessage({ ...body, messageText });
-  const result = await gmail.users.messages.send({
-    userId: "me",
-    requestBody: {
-      threadId: body.replyToEmail ? body.replyToEmail.threadId : undefined,
-      raw,
-    },
-  });
+  const result = await withGmailRetry(() =>
+    gmail.users.messages.send({
+      userId: "me",
+      requestBody: {
+        threadId: body.replyToEmail ? body.replyToEmail.threadId : undefined,
+        raw,
+      },
+    }),
+  );
   return result;
 }
 
@@ -160,13 +162,15 @@ export async function replyToEmail(
     from,
   );
 
-  const result = await gmail.users.messages.send({
-    userId: "me",
-    requestBody: {
-      threadId: message.threadId,
-      raw,
-    },
-  });
+  const result = await withGmailRetry(() =>
+    gmail.users.messages.send({
+      userId: "me",
+      requestBody: {
+        threadId: message.threadId,
+        raw,
+      },
+    }),
+  );
 
   return result;
 }
@@ -189,11 +193,13 @@ export async function forwardEmail(
 
   const attachments = await Promise.all(
     message.attachments?.map(async (attachment) => {
-      const attachmentData = await gmail.users.messages.attachments.get({
-        userId: "me",
-        messageId: message.id,
-        id: attachment.attachmentId,
-      });
+      const attachmentData = await withGmailRetry(() =>
+        gmail.users.messages.attachments.get({
+          userId: "me",
+          messageId: message.id,
+          id: attachment.attachmentId,
+        }),
+      );
       return {
         content: Buffer.from(attachmentData.data.data || "", "base64"),
         contentType: attachment.mimeType,
@@ -217,13 +223,15 @@ export async function forwardEmail(
     attachments,
   });
 
-  const result = await gmail.users.messages.send({
-    userId: "me",
-    requestBody: {
-      threadId: message.threadId,
-      raw,
-    },
-  });
+  const result = await withGmailRetry(() =>
+    gmail.users.messages.send({
+      userId: "me",
+      requestBody: {
+        threadId: message.threadId,
+        raw,
+      },
+    }),
+  );
 
   return result;
 }

apps/web/utils/gmail/retry.test.ts

@@ -0,0 +1,142 @@
+import { describe, it, expect } from "vitest";
+import { isRetryableError, calculateRetryDelay } from "./retry";
+
+describe("Gmail retry helpers", () => {
+  describe("isRetryableError", () => {
+    it("should identify 502 status code as retryable server error", () => {
+      const errorInfo = { status: 502, errorMessage: "Server Error" };
+      const result = isRetryableError(errorInfo);
+
+      expect(result.retryable).toBe(true);
+      expect(result.isServerError).toBe(true);
+      expect(result.isRateLimit).toBe(false);
+    });
+
+    it("should identify 502 in error message as retryable (Gmail HTML error)", () => {
+      const errorInfo = {
+        errorMessage: "Error 502 (Server Error)!!1",
+      };
+      const result = isRetryableError(errorInfo);
+
+      expect(result.retryable).toBe(true);
+      expect(result.isServerError).toBe(true);
+      expect(result.isRateLimit).toBe(false);
+    });
+
+    it("should identify 503 in error message as retryable", () => {
+      const errorInfo = {
+        errorMessage: "503 Service Unavailable",
+      };
+      const result = isRetryableError(errorInfo);
+
+      expect(result.retryable).toBe(true);
+      expect(result.isServerError).toBe(true);
+      expect(result.isRateLimit).toBe(false);
+    });
+
+    it("should identify 504 Gateway Timeout as retryable", () => {
+      const errorInfo = { status: 504, errorMessage: "Gateway Timeout" };
+      const result = isRetryableError(errorInfo);
+
+      expect(result.retryable).toBe(true);
+      expect(result.isServerError).toBe(true);
+      expect(result.isRateLimit).toBe(false);
+    });
+
+    it("should identify 429 as a retryable rate limit error", () => {
+      const errorInfo = { status: 429, errorMessage: "Too Many Requests" };
+      const result = isRetryableError(errorInfo);
+
+      expect(result.retryable).toBe(true);
+      expect(result.isRateLimit).toBe(true);
+      expect(result.isServerError).toBe(false);
+    });
+
+    it("should identify 403 with rateLimitExceeded reason as retryable", () => {
+      const errorInfo = {
+        status: 403,
+        reason: "rateLimitExceeded",
+        errorMessage: "Rate limit exceeded",
+      };
+      const result = isRetryableError(errorInfo);
+
+      expect(result.retryable).toBe(true);
+      expect(result.isRateLimit).toBe(true);
+      expect(result.isServerError).toBe(false);
+    });
+
+    it("should identify 404 as non-retryable", () => {
+      const errorInfo = { status: 404, errorMessage: "Not Found" };
+      const result = isRetryableError(errorInfo);
+
+      expect(result.retryable).toBe(false);
+      expect(result.isRateLimit).toBe(false);
+      expect(result.isServerError).toBe(false);
+    });
+
+    it("should identify 403 without rate limit reason as non-retryable", () => {
+      const errorInfo = {
+        status: 403,
+        reason: "forbidden",
+        errorMessage: "Forbidden",
+      };
+      const result = isRetryableError(errorInfo);
+
+      expect(result.retryable).toBe(false);
+      expect(result.isRateLimit).toBe(false);
+      expect(result.isServerError).toBe(false);
+    });
+  });
+
+  describe("calculateRetryDelay", () => {
+    it("should return 30 seconds for rate limit errors", () => {
+      const delay = calculateRetryDelay(true, false, 1);
+      expect(delay).toBe(30_000);
+    });
+
+    it("should use exponential backoff for server errors", () => {
+      expect(calculateRetryDelay(false, true, 1)).toBe(5000); // 5s
+      expect(calculateRetryDelay(false, true, 2)).toBe(10_000); // 10s
+      expect(calculateRetryDelay(false, true, 3)).toBe(20_000); // 20s
+    });
+
+    it("should use fallback delay when retry time is in the past", () => {
+      const pastDate = new Date(Date.now() - 10_000).toISOString();
+      const errorMessage = `Rate limit exceeded. Retry after ${pastDate}`;
+
+      // Should fall back to 30s for rate limit
+      const delay = calculateRetryDelay(
+        true,
+        false,
+        1,
+        undefined,
+        errorMessage,
+      );
+      expect(delay).toBe(30_000);
+    });
+
+    it("should use fallback delay when Retry-After header is stale", () => {
+      // Use HTTP-date format (like "Wed, 21 Oct 2015 07:28:00 GMT")
+      const pastDate = new Date(Date.now() - 5000).toUTCString();
+
+      // Should fall back to exponential backoff for server error
+      const delay = calculateRetryDelay(false, true, 2, pastDate);
+      expect(delay).toBe(10_000); // 2nd attempt = 10s
+    });
+
+    it("should use retry time from error message when valid", () => {
+      const futureDate = new Date(Date.now() + 15_000).toISOString();
+      const errorMessage = `Rate limit exceeded. Retry after ${futureDate}`;
+
+      const delay = calculateRetryDelay(
+        true,
+        false,
+        1,
+        undefined,
+        errorMessage,
+      );
+      expect(delay).toBeGreaterThan(14_000); // Should be ~15s
+      expect(delay).toBeLessThan(16_000);
+    });
+  });
+});