diff --git a/apps/web/utils/gmail/draft.ts b/apps/web/utils/gmail/draft.ts index acde493c6c..f074fa0585 100644 --- a/apps/web/utils/gmail/draft.ts +++ b/apps/web/utils/gmail/draft.ts @@ -3,16 +3,19 @@ import { createScopedLogger } from "@/utils/logger"; import { parseMessage } from "@/utils/gmail/message"; import type { MessageWithPayload } from "@/utils/types"; import { isGmailError } from "@/utils/error"; +import { withGmailRetry } from "@/utils/gmail/retry"; const logger = createScopedLogger("gmail/draft"); export async function getDraft(draftId: string, gmail: gmail_v1.Gmail) { try { - const response = await gmail.users.drafts.get({ - userId: "me", - id: draftId, - format: "full", - }); + const response = await withGmailRetry(() => + gmail.users.drafts.get({ + userId: "me", + id: draftId, + format: "full", + }), + ); const message = parseMessage(response.data.message as MessageWithPayload); return message; } catch (error) { @@ -24,10 +27,12 @@ export async function getDraft(draftId: string, gmail: gmail_v1.Gmail) { export async function deleteDraft(gmail: gmail_v1.Gmail, draftId: string) { try { logger.info("Deleting draft", { draftId }); - const response = await gmail.users.drafts.delete({ - userId: "me", - id: draftId, - }); + const response = await withGmailRetry(() => + gmail.users.drafts.delete({ + userId: "me", + id: draftId, + }), + ); if (response.status !== 200 && response.status !== 204) { logger.error("Failed to delete draft", { draftId, response }); } diff --git a/apps/web/utils/gmail/label.ts b/apps/web/utils/gmail/label.ts index 003c7e7e86..b76879837e 100644 --- a/apps/web/utils/gmail/label.ts +++ b/apps/web/utils/gmail/label.ts @@ -85,14 +85,16 @@ export async function archiveThread({ actionSource: TinybirdEmailAction["actionSource"]; labelId?: string; }) { - const archivePromise = gmail.users.threads.modify({ - userId: "me", - id: threadId, - requestBody: { - removeLabelIds: [GmailLabel.INBOX], - ...(labelId ? { addLabelIds: [labelId] } : {}), - }, - }); + const archivePromise = withGmailRetry(() => + gmail.users.threads.modify({ + userId: "me", + id: threadId, + requestBody: { + removeLabelIds: [GmailLabel.INBOX], + ...(labelId ? { addLabelIds: [labelId] } : {}), + }, + }), + ); const publishPromise = publishArchive({ ownerEmail, @@ -137,11 +139,13 @@ export async function labelMessage({ addLabelIds?: string[]; removeLabelIds?: string[]; }) { - return gmail.users.messages.modify({ - userId: "me", - id: messageId, - requestBody: { addLabelIds, removeLabelIds }, - }); + return withGmailRetry(() => + gmail.users.messages.modify({ + userId: "me", + id: messageId, + requestBody: { addLabelIds, removeLabelIds }, + }), + ); } export async function markReadThread(options: { @@ -151,17 +155,19 @@ export async function markReadThread(options: { }) { const { gmail, threadId, read } = options; - return gmail.users.threads.modify({ - userId: "me", - id: threadId, - requestBody: read - ? { - removeLabelIds: [GmailLabel.UNREAD], - } - : { - addLabelIds: [GmailLabel.UNREAD], - }, - }); + return withGmailRetry(() => + gmail.users.threads.modify({ + userId: "me", + id: threadId, + requestBody: read + ? { + removeLabelIds: [GmailLabel.UNREAD], + } + : { + addLabelIds: [GmailLabel.UNREAD], + }, + }), + ); } export async function createLabel({ diff --git a/apps/web/utils/gmail/mail.ts b/apps/web/utils/gmail/mail.ts index d74243cfb9..107dd2daa7 100644 --- a/apps/web/utils/gmail/mail.ts +++ b/apps/web/utils/gmail/mail.ts @@ -112,13 +112,15 @@ export async function sendEmailWithHtml( } const raw = await createRawMailMessage({ ...body, messageText }); - const result = await gmail.users.messages.send({ - userId: "me", - requestBody: { - threadId: body.replyToEmail ? body.replyToEmail.threadId : undefined, - raw, - }, - }); + const result = await withGmailRetry(() => + gmail.users.messages.send({ + userId: "me", + requestBody: { + threadId: body.replyToEmail ? body.replyToEmail.threadId : undefined, + raw, + }, + }), + ); return result; } @@ -160,13 +162,15 @@ export async function replyToEmail( from, ); - const result = await gmail.users.messages.send({ - userId: "me", - requestBody: { - threadId: message.threadId, - raw, - }, - }); + const result = await withGmailRetry(() => + gmail.users.messages.send({ + userId: "me", + requestBody: { + threadId: message.threadId, + raw, + }, + }), + ); return result; } @@ -189,11 +193,13 @@ export async function forwardEmail( const attachments = await Promise.all( message.attachments?.map(async (attachment) => { - const attachmentData = await gmail.users.messages.attachments.get({ - userId: "me", - messageId: message.id, - id: attachment.attachmentId, - }); + const attachmentData = await withGmailRetry(() => + gmail.users.messages.attachments.get({ + userId: "me", + messageId: message.id, + id: attachment.attachmentId, + }), + ); return { content: Buffer.from(attachmentData.data.data || "", "base64"), contentType: attachment.mimeType, @@ -217,13 +223,15 @@ export async function forwardEmail( attachments, }); - const result = await gmail.users.messages.send({ - userId: "me", - requestBody: { - threadId: message.threadId, - raw, - }, - }); + const result = await withGmailRetry(() => + gmail.users.messages.send({ + userId: "me", + requestBody: { + threadId: message.threadId, + raw, + }, + }), + ); return result; } diff --git a/apps/web/utils/gmail/retry.test.ts b/apps/web/utils/gmail/retry.test.ts new file mode 100644 index 0000000000..c217b5aed1 --- /dev/null +++ b/apps/web/utils/gmail/retry.test.ts @@ -0,0 +1,142 @@ +import { describe, it, expect } from "vitest"; +import { isRetryableError, calculateRetryDelay } from "./retry"; + +describe("Gmail retry helpers", () => { + describe("isRetryableError", () => { + it("should identify 502 status code as retryable server error", () => { + const errorInfo = { status: 502, errorMessage: "Server Error" }; + const result = isRetryableError(errorInfo); + + expect(result.retryable).toBe(true); + expect(result.isServerError).toBe(true); + expect(result.isRateLimit).toBe(false); + }); + + it("should identify 502 in error message as retryable (Gmail HTML error)", () => { + const errorInfo = { + errorMessage: "Error 502 (Server Error)!!1", + }; + const result = isRetryableError(errorInfo); + + expect(result.retryable).toBe(true); + expect(result.isServerError).toBe(true); + expect(result.isRateLimit).toBe(false); + }); + + it("should identify 503 in error message as retryable", () => { + const errorInfo = { + errorMessage: "503 Service Unavailable", + }; + const result = isRetryableError(errorInfo); + + expect(result.retryable).toBe(true); + expect(result.isServerError).toBe(true); + expect(result.isRateLimit).toBe(false); + }); + + it("should identify 504 Gateway Timeout as retryable", () => { + const errorInfo = { status: 504, errorMessage: "Gateway Timeout" }; + const result = isRetryableError(errorInfo); + + expect(result.retryable).toBe(true); + expect(result.isServerError).toBe(true); + expect(result.isRateLimit).toBe(false); + }); + + it("should identify 429 as a retryable rate limit error", () => { + const errorInfo = { status: 429, errorMessage: "Too Many Requests" }; + const result = isRetryableError(errorInfo); + + expect(result.retryable).toBe(true); + expect(result.isRateLimit).toBe(true); + expect(result.isServerError).toBe(false); + }); + + it("should identify 403 with rateLimitExceeded reason as retryable", () => { + const errorInfo = { + status: 403, + reason: "rateLimitExceeded", + errorMessage: "Rate limit exceeded", + }; + const result = isRetryableError(errorInfo); + + expect(result.retryable).toBe(true); + expect(result.isRateLimit).toBe(true); + expect(result.isServerError).toBe(false); + }); + + it("should identify 404 as non-retryable", () => { + const errorInfo = { status: 404, errorMessage: "Not Found" }; + const result = isRetryableError(errorInfo); + + expect(result.retryable).toBe(false); + expect(result.isRateLimit).toBe(false); + expect(result.isServerError).toBe(false); + }); + + it("should identify 403 without rate limit reason as non-retryable", () => { + const errorInfo = { + status: 403, + reason: "forbidden", + errorMessage: "Forbidden", + }; + const result = isRetryableError(errorInfo); + + expect(result.retryable).toBe(false); + expect(result.isRateLimit).toBe(false); + expect(result.isServerError).toBe(false); + }); + }); + + describe("calculateRetryDelay", () => { + it("should return 30 seconds for rate limit errors", () => { + const delay = calculateRetryDelay(true, false, 1); + expect(delay).toBe(30_000); + }); + + it("should use exponential backoff for server errors", () => { + expect(calculateRetryDelay(false, true, 1)).toBe(5000); // 5s + expect(calculateRetryDelay(false, true, 2)).toBe(10_000); // 10s + expect(calculateRetryDelay(false, true, 3)).toBe(20_000); // 20s + }); + + it("should use fallback delay when retry time is in the past", () => { + const pastDate = new Date(Date.now() - 10_000).toISOString(); + const errorMessage = `Rate limit exceeded. Retry after ${pastDate}`; + + // Should fall back to 30s for rate limit + const delay = calculateRetryDelay( + true, + false, + 1, + undefined, + errorMessage, + ); + expect(delay).toBe(30_000); + }); + + it("should use fallback delay when Retry-After header is stale", () => { + // Use HTTP-date format (like "Wed, 21 Oct 2015 07:28:00 GMT") + const pastDate = new Date(Date.now() - 5000).toUTCString(); + + // Should fall back to exponential backoff for server error + const delay = calculateRetryDelay(false, true, 2, pastDate); + expect(delay).toBe(10_000); // 2nd attempt = 10s + }); + + it("should use retry time from error message when valid", () => { + const futureDate = new Date(Date.now() + 15_000).toISOString(); + const errorMessage = `Rate limit exceeded. Retry after ${futureDate}`; + + const delay = calculateRetryDelay( + true, + false, + 1, + undefined, + errorMessage, + ); + expect(delay).toBeGreaterThan(14_000); // Should be ~15s + expect(delay).toBeLessThan(16_000); + }); + }); +}); diff --git a/apps/web/utils/gmail/retry.ts b/apps/web/utils/gmail/retry.ts index 0c48394d2b..51becdd32d 100644 --- a/apps/web/utils/gmail/retry.ts +++ b/apps/web/utils/gmail/retry.ts @@ -4,8 +4,134 @@ import { sleep } from "@/utils/sleep"; const logger = createScopedLogger("gmail-retry"); +interface ErrorInfo { + status?: number; + reason?: string; + errorMessage: string; +} + +/** + * Extracts error information from various error shapes + */ +export function extractErrorInfo(error: unknown): ErrorInfo { + const err = error as Record; + const cause = (err?.cause ?? err) as Record; + const status = + (cause?.status as number) ?? + (cause?.code as number) ?? + ((cause?.response as Record)?.status as number) ?? + undefined; + const reason = + ((cause?.errors as Array>)?.[0] + ?.reason as string) ?? + (( + ( + ( + (cause?.response as Record)?.data as Record< + string, + unknown + > + )?.error as Record + )?.errors as Array> + )?.[0]?.reason as string) ?? + undefined; + const errorMessage = String( + (cause?.message as string) ?? (err?.message as string) ?? "", + ); + + return { status, reason, errorMessage }; +} + +/** + * Determines if an error is retryable (rate limit or server error) + */ +export function isRetryableError(errorInfo: ErrorInfo): { + retryable: boolean; + isRateLimit: boolean; + isServerError: boolean; +} { + const { status, reason, errorMessage } = errorInfo; + + // Broad rate-limit detection: 429, 403 + known reasons, or well-known messages + const isRateLimit = + status === 429 || + (status === 403 && + ["rateLimitExceeded", "userRateLimitExceeded", "quotaExceeded"].includes( + String(reason), + )) || + /(^|[\s-])rate limit exceeded/i.test(errorMessage) || + /quota exceeded/i.test(errorMessage); + + // Temporary server errors that should be retried (502, 503, 504) + const isServerError = + status === 502 || + status === 503 || + status === 504 || + /502|503|504|server error|temporarily unavailable/i.test(errorMessage); + + return { + retryable: isRateLimit || isServerError, + isRateLimit, + isServerError, + }; +} + +/** + * Calculates retry delay based on error type and attempt number + */ +export function calculateRetryDelay( + isRateLimit: boolean, + isServerError: boolean, + attemptNumber: number, + retryAfterHeader?: string, + errorMessage?: string, +): number { + // Try to parse retry time from error message + const retryTime = parseRetryTime(errorMessage || ""); + if (retryTime) { + const delayMs = Math.max(0, retryTime.getTime() - Date.now()); + if (delayMs > 0) { + return delayMs; + } + // If stale, fall through to fallback logic + } + + // Handle Retry-After header + if (retryAfterHeader) { + const retryAfterSeconds = Number.parseInt(retryAfterHeader, 10); + if (!Number.isNaN(retryAfterSeconds)) { + return retryAfterSeconds * 1000; + } + + // Try parsing as HTTP-date + const retryDate = new Date(retryAfterHeader); + if (!Number.isNaN(retryDate.getTime())) { + const delayMs = Math.max(0, retryDate.getTime() - Date.now()); + if (delayMs > 0) { + return delayMs; + } + // If stale, fall through to fallback logic + } + } + + // Use different fallback delays based on error type + if (isServerError) { + // Exponential backoff for server errors: 5s, 10s, 20s, 40s, 80s + return Math.min(5000 * 2 ** (attemptNumber - 1), 80_000); + } + + if (isRateLimit) { + // Fixed delay for rate limits (30 seconds as per Gmail's error message) + return 30_000; + } + + return 0; +} + /** - * Retries a Gmail API operation when rate limits are hit + * Retries a Gmail API operation when rate limits or temporary server errors are encountered + * - Rate limits: 429, 403 with specific reasons + * - Server errors: 502, 503, 504 */ export async function withGmailRetry( operation: () => Promise, @@ -14,84 +140,44 @@ export async function withGmailRetry( return pRetry(operation, { retries: maxRetries, onFailedAttempt: async (error) => { - // Normalized error metadata across common googleapis/gaxios shapes - const err: any = error; - const cause = err?.cause ?? err; - const status = - cause?.status ?? cause?.code ?? cause?.response?.status ?? undefined; - const reason = - cause?.errors?.[0]?.reason ?? - cause?.response?.data?.error?.errors?.[0]?.reason ?? - undefined; - const errorMessage = String(cause?.message ?? err?.message ?? ""); - - // Broad rate-limit detection: 429, 403 + known reasons, or well-known messages - const isRateLimitError = - status === 429 || - (status === 403 && - [ - "rateLimitExceeded", - "userRateLimitExceeded", - "quotaExceeded", - ].includes(String(reason))) || - /(^|[\s-])rate limit exceeded/i.test(errorMessage) || - /quota exceeded/i.test(errorMessage); - - if (!isRateLimitError) { - logger.warn("Non-rate limit error encountered, not retrying", { + const errorInfo = extractErrorInfo(error); + const { retryable, isRateLimit, isServerError } = + isRetryableError(errorInfo); + + if (!retryable) { + logger.warn("Non-retryable error encountered", { error, - status, - reason, + status: errorInfo.status, + reason: errorInfo.reason, }); throw error; } - // Parse retry time from error message or headers - const retryTime = parseRetryTime(errorMessage); - const retryAfterHeader = cause?.response?.headers?.["retry-after"]; - let delayMs = 0; - - if (retryTime) { - // Calculate delay until the specified retry time - delayMs = Math.max(0, retryTime.getTime() - Date.now()); - logger.warn("Gmail rate limit hit. Will retry after specified time", { - retryAfter: retryTime.toISOString(), - delaySeconds: Math.ceil(delayMs / 1000), - attemptNumber: error.attemptNumber, - maxRetries, - }); - } else if (retryAfterHeader) { - // Handle Retry-After header (can be seconds or HTTP-date) - const retryAfterSeconds = Number.parseInt(retryAfterHeader, 10); - if (!Number.isNaN(retryAfterSeconds)) { - delayMs = retryAfterSeconds * 1000; - } else { - // Try parsing as HTTP-date - const retryDate = new Date(retryAfterHeader); - if (!Number.isNaN(retryDate.getTime())) { - delayMs = Math.max(0, retryDate.getTime() - Date.now()); - } - } - - if (delayMs > 0) { - logger.warn("Gmail rate limit hit. Using Retry-After header", { - retryAfterHeader, - delaySeconds: Math.ceil(delayMs / 1000), - attemptNumber: error.attemptNumber, - maxRetries, - }); - } - } + const err = error as Record; + const cause = (err?.cause ?? err) as Record; + const retryAfterHeader = ( + (cause?.response as Record)?.headers as Record< + string, + string + > + )?.["retry-after"]; - if (!delayMs || delayMs <= 0) { - // Fallback to fixed delay if no retry time specified - delayMs = 30_000; - logger.warn("Gmail rate limit hit. Using fallback delay", { - delaySeconds: Math.ceil(delayMs / 1000), - attemptNumber: error.attemptNumber, - maxRetries, - }); - } + const delayMs = calculateRetryDelay( + isRateLimit, + isServerError, + error.attemptNumber, + retryAfterHeader, + errorInfo.errorMessage, + ); + + logger.warn("Gmail error. Will retry", { + delaySeconds: Math.ceil(delayMs / 1000), + attemptNumber: error.attemptNumber, + maxRetries, + status: errorInfo.status, + isRateLimit, + isServerError, + }); // Apply the custom delay if (delayMs > 0) { diff --git a/apps/web/utils/gmail/spam.ts b/apps/web/utils/gmail/spam.ts index ebd116cc68..e64b4ad88d 100644 --- a/apps/web/utils/gmail/spam.ts +++ b/apps/web/utils/gmail/spam.ts @@ -1,5 +1,6 @@ import type { gmail_v1 } from "@googleapis/gmail"; import { GmailLabel } from "@/utils/gmail/label"; +import { withGmailRetry } from "@/utils/gmail/retry"; export async function markSpam(options: { gmail: gmail_v1.Gmail; @@ -7,11 +8,13 @@ export async function markSpam(options: { }) { const { gmail, threadId } = options; - return gmail.users.threads.modify({ - userId: "me", - id: threadId, - requestBody: { - addLabelIds: [GmailLabel.SPAM], - }, - }); + return withGmailRetry(() => + gmail.users.threads.modify({ + userId: "me", + id: threadId, + requestBody: { + addLabelIds: [GmailLabel.SPAM], + }, + }), + ); } diff --git a/apps/web/utils/gmail/trash.ts b/apps/web/utils/gmail/trash.ts index c2357eeb8d..ac9da9c6fb 100644 --- a/apps/web/utils/gmail/trash.ts +++ b/apps/web/utils/gmail/trash.ts @@ -1,6 +1,7 @@ import type { gmail_v1 } from "@googleapis/gmail"; import { publishDelete, type TinybirdEmailAction } from "@inboxzero/tinybird"; import { createScopedLogger } from "@/utils/logger"; +import { withGmailRetry } from "@/utils/gmail/retry"; const logger = createScopedLogger("gmail/trash"); @@ -16,10 +17,12 @@ export async function trashThread(options: { }) { const { gmail, threadId, ownerEmail, actionSource } = options; - const trashPromise = gmail.users.threads.trash({ - userId: "me", - id: threadId, - }); + const trashPromise = withGmailRetry(() => + gmail.users.threads.trash({ + userId: "me", + id: threadId, + }), + ); const publishPromise = publishDelete({ ownerEmail, @@ -71,8 +74,10 @@ export async function trashMessage(options: { }) { const { gmail, messageId } = options; - return gmail.users.messages.trash({ - userId: "me", - id: messageId, - }); + return withGmailRetry(() => + gmail.users.messages.trash({ + userId: "me", + id: messageId, + }), + ); } diff --git a/version.txt b/version.txt index 2d2bfeff71..6d58237c74 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -v2.17.32 +v2.17.33