diff --git a/src/data-layer/fetchers/developer-tools/fetchBuidlGuidl.ts b/src/data-layer/fetchers/developer-tools/fetchBuidlGuidl.ts index c1e1b8fc07a..d575e492ca0 100644 --- a/src/data-layer/fetchers/developer-tools/fetchBuidlGuidl.ts +++ b/src/data-layer/fetchers/developer-tools/fetchBuidlGuidl.ts @@ -1,12 +1,14 @@ import { DeveloperToolsResponse } from "@/lib/types" +import { fetchRetry } from "@/data-layer/fetchers/fetchRetry" + export async function fetchBuidlGuidl(): Promise { const url = "https://raw.githubusercontent.com/BuidlGuidl/Developer-Tooling/refs/heads/main/output/results.json" console.log("Starting BuidlGuidl developer tooling data fetch") - const response = await fetch(url) + const response = await fetchRetry(url) if (!response.ok) { const status = response.status diff --git a/src/data-layer/fetchers/developer-tools/fetchGitHub.ts b/src/data-layer/fetchers/developer-tools/fetchGitHub.ts index 32d105cc857..b55dc66b1b4 100644 --- a/src/data-layer/fetchers/developer-tools/fetchGitHub.ts +++ b/src/data-layer/fetchers/developer-tools/fetchGitHub.ts @@ -1,6 +1,6 @@ import type { DeveloperToolsResponse } from "@/lib/types" -import { retry, sleep } from "@/lib/utils/fetch" +import { fetchRetry, sleep } from "@/data-layer/fetchers/fetchRetry" import type { DeveloperTool } from "./utils" @@ -66,7 +66,7 @@ async function fetchReposBatch( const query = buildGraphQLQuery(repos) - const response = await fetch("https://api.github.com/graphql", { + const response = await fetchRetry("https://api.github.com/graphql", { method: "POST", headers: { Authorization: `Bearer ${process.env.GITHUB_TOKEN_READ_ONLY}`, @@ -129,17 +129,13 @@ export async function fetchGitHub( const batch = allRepos.slice(i, i + BATCH_SIZE) try { - // Retry with exponential backoff (3 attempts: 0ms, 1s, 2s) - const batchResults = await retry(() => fetchReposBatch(batch)) + const batchResults = await fetchReposBatch(batch) for (const [href, data] of batchResults) { repoDataMap.set(href, data) } } catch (error) { - console.error( - `Failed to fetch batch ${i / BATCH_SIZE + 1} after retries:`, - error - ) + console.error(`Failed to fetch batch ${i / BATCH_SIZE + 1}:`, error) // Continue with next batch instead of failing entirely } diff --git a/src/data-layer/fetchers/developer-tools/fetchNpmJs.ts b/src/data-layer/fetchers/developer-tools/fetchNpmJs.ts index 78e096178f5..548dca32d49 100644 --- a/src/data-layer/fetchers/developer-tools/fetchNpmJs.ts +++ b/src/data-layer/fetchers/developer-tools/fetchNpmJs.ts @@ -1,4 +1,4 @@ -import { retry, sleep } from "@/lib/utils/fetch" +import { fetchRetry, sleep } from "@/data-layer/fetchers/fetchRetry" import type { DeveloperTool } from "./utils" @@ -37,7 +37,7 @@ async function fetchSinglePackageDownloads( packageName: string ): Promise { try { - const response = await fetch( + const response = await fetchRetry( `https://api.npmjs.org/downloads/point/last-week/${encodeURIComponent(packageName)}` ) @@ -94,35 +94,32 @@ async function fetchBulkDownloads( const packageList = batch.join(",") try { - // Retry with exponential backoff (3 attempts: 0ms, 1s, 2s) - await retry(async () => { - const response = await fetch( - `https://api.npmjs.org/downloads/point/last-week/${packageList}` + const response = await fetchRetry( + `https://api.npmjs.org/downloads/point/last-week/${packageList}` + ) + + if (!response.ok) { + throw new Error( + `npm downloads API returned ${response.status} for unscoped batch` ) + } - if (!response.ok) { - throw new Error( - `npm downloads API returned ${response.status} for unscoped batch` - ) - } + const data = await response.json() - const data = await response.json() - - // Handle single-package response: { downloads: number, package: string } - // vs multi-package response: { "pkg1": { downloads: n }, "pkg2": { downloads: m } } - if ("downloads" in data && "package" in data) { - results.set(data.package, data.downloads) - } else { - for (const [pkg, info] of Object.entries(data)) { - if (info && typeof info === "object" && "downloads" in info) { - results.set(pkg, (info as { downloads: number }).downloads) - } + // Handle single-package response: { downloads: number, package: string } + // vs multi-package response: { "pkg1": { downloads: n }, "pkg2": { downloads: m } } + if ("downloads" in data && "package" in data) { + results.set(data.package, data.downloads) + } else { + for (const [pkg, info] of Object.entries(data)) { + if (info && typeof info === "object" && "downloads" in info) { + results.set(pkg, (info as { downloads: number }).downloads) } } - }) + } } catch (err) { console.error( - `Failed to fetch bulk npm downloads for batch ${i / BATCH_SIZE + 1} after retries:`, + `Failed to fetch bulk npm downloads for batch ${i / BATCH_SIZE + 1}:`, err ) // Continue with next batch instead of failing entirely diff --git a/src/data-layer/fetchers/fetchAccountHolders.ts b/src/data-layer/fetchers/fetchAccountHolders.ts index 99a6f8ef01c..e13ee618560 100644 --- a/src/data-layer/fetchers/fetchAccountHolders.ts +++ b/src/data-layer/fetchers/fetchAccountHolders.ts @@ -2,6 +2,8 @@ import type { MetricReturnData } from "@/lib/types" import { DUNE_API_URL } from "@/lib/constants" +import { fetchRetry } from "./fetchRetry" + export const FETCH_ACCOUNT_HOLDERS_TASK_ID = "fetch-account-holders" // Dune query: https://dune.com/queries/6676254 @@ -30,7 +32,7 @@ export async function fetchAccountHolders(): Promise { console.log("Starting account holders data fetch from Dune Analytics") - const response = await fetch(url, { + const response = await fetchRetry(url, { headers: { "X-Dune-API-Key": duneApiKey }, }) diff --git a/src/data-layer/fetchers/fetchApps.ts b/src/data-layer/fetchers/fetchApps.ts index 57f970a7c71..49f68051650 100644 --- a/src/data-layer/fetchers/fetchApps.ts +++ b/src/data-layer/fetchers/fetchApps.ts @@ -2,6 +2,8 @@ import { AppCategoryEnum, AppData } from "@/lib/types" import { uploadToS3 } from "@/data-layer/s3" +import { fetchRetry } from "./fetchRetry" + export const FETCH_APPS_TASK_ID = "fetch-apps" /** @@ -25,7 +27,7 @@ export async function fetchApps(): Promise> { // First, get the spreadsheet metadata to see what sheets exist const metadataUrl = `https://sheets.googleapis.com/v4/spreadsheets/${sheetId}?key=${googleApiKey}` - const metadataResponse = await fetch(metadataUrl) + const metadataResponse = await fetchRetry(metadataUrl) if (!metadataResponse.ok) { const errorText = await metadataResponse.text() @@ -52,7 +54,7 @@ export async function fetchApps(): Promise> { console.log(`Found ${appCategorySheetNames.length} app category sheets`) - const appsOfTheWeek = await fetch( + const appsOfTheWeek = await fetchRetry( `https://sheets.googleapis.com/v4/spreadsheets/${sheetId}/values/App%20of%20the%20day!A2:C?majorDimension=ROWS&key=${googleApiKey}` ) @@ -69,7 +71,7 @@ export async function fetchApps(): Promise> { // Fetch and process data from each sheet for (const sheetName of appCategorySheetNames) { const dataUrl = `https://sheets.googleapis.com/v4/spreadsheets/${sheetId}/values/${sheetName}!A:Z?majorDimension=ROWS&key=${googleApiKey}` - const dataResponse = await fetch(dataUrl) + const dataResponse = await fetchRetry(dataUrl) if (!dataResponse.ok) { console.warn( diff --git a/src/data-layer/fetchers/fetchBlobscanStats.ts b/src/data-layer/fetchers/fetchBlobscanStats.ts index 38c0c1ab796..a6cb125ff56 100644 --- a/src/data-layer/fetchers/fetchBlobscanStats.ts +++ b/src/data-layer/fetchers/fetchBlobscanStats.ts @@ -1,3 +1,5 @@ +import { fetchRetry } from "./fetchRetry" + export const FETCH_BLOBSCAN_STATS_TASK_ID = "fetch-blobscan-stats" export type BlobscanStats = { @@ -26,7 +28,7 @@ export async function fetchBlobscanStats(): Promise { console.log("Starting blobscan stats data fetch") - const response = await fetch(url) + const response = await fetchRetry(url) if (!response.ok) { const status = response.status diff --git a/src/data-layer/fetchers/fetchCalendarEvents.ts b/src/data-layer/fetchers/fetchCalendarEvents.ts index 12da06d42e7..f0b810f3039 100644 --- a/src/data-layer/fetchers/fetchCalendarEvents.ts +++ b/src/data-layer/fetchers/fetchCalendarEvents.ts @@ -3,6 +3,8 @@ import type { ReqCommunityEvent, } from "@/lib/interfaces" +import { fetchRetry } from "./fetchRetry" + export const FETCH_CALENDAR_EVENTS_TASK_ID = "fetch-calendar-events" /** @@ -13,13 +15,13 @@ export async function fetchCalendarEvents(): Promise const apiKey = process.env.GOOGLE_API_KEY const calendarId = process.env.GOOGLE_CALENDAR_ID - const futureEventsReq = await fetch( + const futureEventsReq = await fetchRetry( `https://content.googleapis.com/calendar/v3/calendars/${calendarId}/events?key=${apiKey}&timeMin=${new Date().toISOString()}&maxResults=3&singleEvents=true&orderby=starttime` ) const futureEvents = await futureEventsReq.json() const futureEventsReqData: ReqCommunityEvent[] = futureEvents.items - const pastEventsReq = await fetch( + const pastEventsReq = await fetchRetry( `https://content.googleapis.com/calendar/v3/calendars/${calendarId}/events?key=${apiKey}&timeMax=${new Date().toISOString()}&orderby=starttime` ) const pastEvents = await pastEventsReq.json() diff --git a/src/data-layer/fetchers/fetchCommunityPicks.ts b/src/data-layer/fetchers/fetchCommunityPicks.ts index 49881248bbb..65ef2ea5420 100644 --- a/src/data-layer/fetchers/fetchCommunityPicks.ts +++ b/src/data-layer/fetchers/fetchCommunityPicks.ts @@ -1,5 +1,7 @@ import type { CommunityPick } from "@/lib/types" +import { fetchRetry } from "./fetchRetry" + export const FETCH_COMMUNITY_PICKS_TASK_ID = "fetch-community-picks" /** @@ -20,7 +22,7 @@ export async function fetchCommunityPicks(): Promise { console.log("Starting community picks data fetch from Google Sheets") - const response = await fetch( + const response = await fetchRetry( `https://sheets.googleapis.com/v4/spreadsheets/${sheetId}/values/community_picks!A:Z?majorDimension=ROWS&key=${googleApiKey}` ) diff --git a/src/data-layer/fetchers/fetchEthPrice.ts b/src/data-layer/fetchers/fetchEthPrice.ts index 6bb5eb1b3e1..6ee2b72a268 100644 --- a/src/data-layer/fetchers/fetchEthPrice.ts +++ b/src/data-layer/fetchers/fetchEthPrice.ts @@ -1,5 +1,7 @@ import type { MetricReturnData } from "@/lib/types" +import { fetchRetry } from "./fetchRetry" + export const FETCH_ETH_PRICE_TASK_ID = "fetch-eth-price" /** @@ -12,7 +14,7 @@ export async function fetchEthPrice(): Promise { console.log("Starting Ethereum price data fetch") - const response = await fetch(url) + const response = await fetchRetry(url) if (!response.ok) { const status = response.status diff --git a/src/data-layer/fetchers/fetchEthereumMarketcap.ts b/src/data-layer/fetchers/fetchEthereumMarketcap.ts index cb692e742f7..77d4ecd80c3 100644 --- a/src/data-layer/fetchers/fetchEthereumMarketcap.ts +++ b/src/data-layer/fetchers/fetchEthereumMarketcap.ts @@ -1,5 +1,7 @@ import type { MetricReturnData } from "@/lib/types" +import { fetchRetry } from "./fetchRetry" + export const FETCH_ETHEREUM_MARKETCAP_TASK_ID = "fetch-ethereum-marketcap" /** @@ -12,7 +14,7 @@ export async function fetchEthereumMarketcap(): Promise { console.log("Starting Ethereum market cap data fetch") - const response = await fetch(url) + const response = await fetchRetry(url) if (!response.ok) { const status = response.status diff --git a/src/data-layer/fetchers/fetchEthereumStablecoinsMcap.ts b/src/data-layer/fetchers/fetchEthereumStablecoinsMcap.ts index 1618a56e3a8..de24fbfe149 100644 --- a/src/data-layer/fetchers/fetchEthereumStablecoinsMcap.ts +++ b/src/data-layer/fetchers/fetchEthereumStablecoinsMcap.ts @@ -1,5 +1,7 @@ import type { MetricReturnData } from "@/lib/types" +import { fetchRetry } from "./fetchRetry" + export const FETCH_ETHEREUM_STABLECOINS_MCAP_TASK_ID = "fetch-ethereum-stablecoins-mcap" @@ -19,7 +21,7 @@ export async function fetchEthereumStablecoinsMcap(): Promise console.log("Starting Ethereum stablecoins market cap data fetch") - const response = await fetch(url) + const response = await fetchRetry(url) if (!response.ok) { const status = response.status diff --git a/src/data-layer/fetchers/fetchEvents.ts b/src/data-layer/fetchers/fetchEvents.ts index 9cdaaded402..87144bacb02 100644 --- a/src/data-layer/fetchers/fetchEvents.ts +++ b/src/data-layer/fetchers/fetchEvents.ts @@ -5,6 +5,8 @@ import { slugify } from "@/lib/utils/url" import { uploadToS3 } from "../s3" +import { fetchRetry } from "./fetchRetry" + export const FETCH_EVENTS_TASK_ID = "fetch-events" // Priority order for eventTypes @@ -66,7 +68,7 @@ export async function fetchEvents(): Promise { console.log("Starting events data fetch from Geode Labs API") try { - const response = await fetch(`${url}?select=*`, { + const response = await fetchRetry(`${url}?select=*`, { headers: { apikey: key, Authorization: `Bearer ${key}`, diff --git a/src/data-layer/fetchers/fetchGFIs.ts b/src/data-layer/fetchers/fetchGFIs.ts index fa53d4c8b3c..3c60e1332ea 100644 --- a/src/data-layer/fetchers/fetchGFIs.ts +++ b/src/data-layer/fetchers/fetchGFIs.ts @@ -1,5 +1,7 @@ import type { GHIssue } from "@/lib/types" +import { fetchRetry } from "./fetchRetry" + export const FETCH_GFIS_TASK_ID = "fetch-gfis" const owner = "ethereum" @@ -27,7 +29,7 @@ export async function fetchGFIs(): Promise { console.log("Starting GitHub good first issues data fetch") - const response = await fetch(url, { + const response = await fetchRetry(url, { headers: { Authorization: `token ${githubToken}`, Accept: "application/vnd.github.v3+json", diff --git a/src/data-layer/fetchers/fetchGasPrice.ts b/src/data-layer/fetchers/fetchGasPrice.ts index 6fafb18c4da..29aae3965e2 100644 --- a/src/data-layer/fetchers/fetchGasPrice.ts +++ b/src/data-layer/fetchers/fetchGasPrice.ts @@ -1,3 +1,5 @@ +import { fetchRetry } from "./fetchRetry" + export interface GasPriceData { gasPrice: number timestamp: number @@ -8,7 +10,7 @@ export async function fetchGasPrice(): Promise { console.log("Starting gas price data fetch") - const response = await fetch( + const response = await fetchRetry( `https://api.etherscan.io/v2/api?chainid=1&module=gastracker&action=gasoracle&apikey=${etherscanApiKey}` ) diff --git a/src/data-layer/fetchers/fetchGitHistory.ts b/src/data-layer/fetchers/fetchGitHistory.ts index b5576c33009..4d411061fa0 100644 --- a/src/data-layer/fetchers/fetchGitHistory.ts +++ b/src/data-layer/fetchers/fetchGitHistory.ts @@ -1,5 +1,7 @@ import type { Commit } from "@/lib/types" +import { fetchRetry } from "./fetchRetry" + export const FETCH_GIT_HISTORY_TASK_ID = "fetch-git-history" const owner = "ethereum" @@ -22,7 +24,7 @@ export async function fetchGitHistory(): Promise { console.log("Starting GitHub commit history data fetch") - const response = await fetch(url.href, { + const response = await fetchRetry(url.href, { headers: { Authorization: `token ${githubToken}`, Accept: "application/vnd.github.v3+json", diff --git a/src/data-layer/fetchers/fetchGitHubContributors.ts b/src/data-layer/fetchers/fetchGitHubContributors.ts index 4743eb00029..cd04385dfd1 100644 --- a/src/data-layer/fetchers/fetchGitHubContributors.ts +++ b/src/data-layer/fetchers/fetchGitHubContributors.ts @@ -2,6 +2,8 @@ import type { FileContributor, GitHubContributorsData } from "@/lib/types" import { CONTENT_DIR, OLD_CONTENT_DIR } from "@/lib/constants" +import { fetchRetry } from "./fetchRetry" + const GITHUB_API_BASE = "https://api.github.com/repos/ethereum/ethereum-org-website" @@ -26,7 +28,7 @@ type NameLookup = Map async function fetchNameLookup(): Promise { const url = "https://raw.githubusercontent.com/ethereum/ethereum-org-website/master/.all-contributorsrc" - const response = await fetch(url) + const response = await fetchRetry(url) if (!response.ok) { console.warn("Failed to fetch .all-contributorsrc:", response.status) @@ -250,14 +252,14 @@ async function fetchCommitsForPath( url.searchParams.set("path", filepath) url.searchParams.set("sha", "master") - const response = await fetch(url.href, { + const response = await fetchRetry(url.href, { headers: { Authorization: `Bearer ${token}`, Accept: "application/vnd.github.v3+json", }, }) - // Handle rate limiting + // Handle GitHub-specific rate limiting (403, not 429) if ( response.status === 403 && response.headers.get("X-RateLimit-Remaining") === "0" @@ -368,7 +370,7 @@ async function discoverPathsFromTree(token: string): Promise<{ }> { const url = `${GITHUB_API_BASE}/git/trees/master?recursive=1` - const response = await fetch(url, { + const response = await fetchRetry(url, { headers: { Authorization: `Bearer ${token}`, Accept: "application/vnd.github.v3+json", diff --git a/src/data-layer/fetchers/fetchGithubRepoData.ts b/src/data-layer/fetchers/fetchGithubRepoData.ts index 93d3dc2c8a3..07ebe4c9535 100644 --- a/src/data-layer/fetchers/fetchGithubRepoData.ts +++ b/src/data-layer/fetchers/fetchGithubRepoData.ts @@ -1,5 +1,7 @@ import type { GithubRepoData } from "@/lib/types" +import { fetchRetry } from "./fetchRetry" + export const FETCH_GITHUB_REPO_DATA_TASK_ID = "fetch-github-repo-data" // GitHub repository URLs for local environment frameworks @@ -42,7 +44,7 @@ export async function fetchGithubRepoData(): Promise { const repoName = split[split.length - 1] // Fetch repo data - const repoReq = await fetch( + const repoReq = await fetchRetry( `https://api.github.com/repos/${repoOwner}/${repoName}`, { headers: { @@ -62,7 +64,7 @@ export async function fetchGithubRepoData(): Promise { const repoData = await repoReq.json() // Fetch language data - const languageReq = await fetch( + const languageReq = await fetchRetry( `https://api.github.com/repos/${repoOwner}/${repoName}/languages`, { headers: { diff --git a/src/data-layer/fetchers/fetchGrowThePie.ts b/src/data-layer/fetchers/fetchGrowThePie.ts index 7bab56b08d3..de697d279e3 100644 --- a/src/data-layer/fetchers/fetchGrowThePie.ts +++ b/src/data-layer/fetchers/fetchGrowThePie.ts @@ -2,6 +2,8 @@ import type { GrowThePieData } from "@/lib/types" import { LAYER2_GROWTHEPIE_IDS } from "@/data/networks/growthepieIds" +import { fetchRetry } from "./fetchRetry" + export const FETCH_GROW_THE_PIE_TASK_ID = "fetch-grow-the-pie" type DataItem = { @@ -24,7 +26,7 @@ export async function fetchGrowThePie(): Promise { console.log("Starting GrowThePie data fetch") - const response = await fetch(url) + const response = await fetchRetry(url) if (!response.ok) { const status = response.status diff --git a/src/data-layer/fetchers/fetchGrowThePieBlockspace.ts b/src/data-layer/fetchers/fetchGrowThePieBlockspace.ts index f3797b6426d..b6666614f78 100644 --- a/src/data-layer/fetchers/fetchGrowThePieBlockspace.ts +++ b/src/data-layer/fetchers/fetchGrowThePieBlockspace.ts @@ -2,6 +2,8 @@ import type { BlockspaceData } from "@/lib/types" import { LAYER2_GROWTHEPIE_IDS } from "@/data/networks/growthepieIds" +import { fetchRetry } from "./fetchRetry" + export const FETCH_GROW_THE_PIE_BLOCKSPACE_TASK_ID = "fetch-grow-the-pie-blockspace" @@ -33,7 +35,7 @@ export async function fetchGrowThePieBlockspace(): Promise< for (const networkId of LAYER2_GROWTHEPIE_IDS) { try { const url = `https://api.growthepie.com/v1/chains/blockspace/${networkId}.json` - const response = await fetch(url) + const response = await fetchRetry(url) if (!response.ok) { const status = response.status diff --git a/src/data-layer/fetchers/fetchGrowThePieMaster.ts b/src/data-layer/fetchers/fetchGrowThePieMaster.ts index 7c78e58c3d0..f935fbd475c 100644 --- a/src/data-layer/fetchers/fetchGrowThePieMaster.ts +++ b/src/data-layer/fetchers/fetchGrowThePieMaster.ts @@ -1,5 +1,7 @@ import type { GrowThePieMasterData } from "@/lib/types" +import { fetchRetry } from "./fetchRetry" + export const FETCH_GROW_THE_PIE_MASTER_TASK_ID = "fetch-grow-the-pie-master" interface Chain { @@ -20,7 +22,7 @@ export async function fetchGrowThePieMaster(): Promise { console.log("Starting GrowThePie master data fetch") - const response = await fetch(url) + const response = await fetchRetry(url) if (!response.ok) { const status = response.status diff --git a/src/data-layer/fetchers/fetchL2beat.ts b/src/data-layer/fetchers/fetchL2beat.ts index dc48c4b467e..64c6e88f2f3 100644 --- a/src/data-layer/fetchers/fetchL2beat.ts +++ b/src/data-layer/fetchers/fetchL2beat.ts @@ -1,3 +1,5 @@ +import { fetchRetry } from "./fetchRetry" + export const FETCH_L2BEAT_TASK_ID = "fetch-l2beat" /** @@ -11,7 +13,7 @@ export async function fetchL2beat(): Promise { console.log("Starting L2BEAT data fetch") - const response = await fetch(url) + const response = await fetchRetry(url) if (!response.ok) { const status = response.status diff --git a/src/data-layer/fetchers/fetchRSS.ts b/src/data-layer/fetchers/fetchRSS.ts index b4787153ed5..f89da78bbb3 100644 --- a/src/data-layer/fetchers/fetchRSS.ts +++ b/src/data-layer/fetchers/fetchRSS.ts @@ -6,6 +6,8 @@ import { isValidDate } from "@/lib/utils/date" import { ATTESTANT_BLOG, BLOG_FEEDS } from "@/lib/constants" +import { fetchRetry } from "./fetchRetry" + export const FETCH_RSS_TASK_ID = "fetch-rss" /** @@ -14,18 +16,12 @@ export const FETCH_RSS_TASK_ID = "fetch-rss" * Exported for use by other data-layer modules (e.g., fetchPosts) */ export async function fetchXml(url: string): Promise> { - const response = await fetch(url, { + const response = await fetchRetry(url, { headers: { Cookie: "", DNT: "1" }, // Empty cookie header and do-not-track credentials: "omit", // Don't send or receive cookies }) if (!response.ok) { - // Provide more specific error messages - if (response.status === 429) { - throw new Error( - `Rate limited (429) when fetching ${url}. The server is temporarily limiting requests.` - ) - } throw new Error(`Failed to fetch XML from ${url}: ${response.status}`) } diff --git a/src/data-layer/fetchers/fetchRetry.ts b/src/data-layer/fetchers/fetchRetry.ts index aeec72a9c8d..a9ec2fa849c 100644 --- a/src/data-layer/fetchers/fetchRetry.ts +++ b/src/data-layer/fetchers/fetchRetry.ts @@ -2,31 +2,50 @@ * Shared retry.fetch configuration for all data-layer fetchers. * * Uses Trigger.dev's built-in retry.fetch which automatically handles - * 429 (rate-limit) and 5xx (server error) responses with exponential backoff. + * retryable HTTP responses with exponential backoff. + * + * Retried status codes (up to 3 attempts each): + * - 408 Request Timeout + * - 409 Conflict + * - 429 Too Many Requests (rate-limit) + * - 500-599 Server errors */ import { retry } from "@trigger.dev/sdk/v3" -const RETRY_BY_STATUS = { - "429": { - strategy: "backoff" as const, - maxAttempts: 2, - }, - "500-599": { - strategy: "backoff" as const, - maxAttempts: 2, - }, +type RetryFetchInit = NonNullable[1]> +type RetryByStatus = NonNullable< + NonNullable["byStatus"] +> + +/** Sleep for specified milliseconds. */ +export const sleep = (ms: number) => + new Promise((resolve) => setTimeout(resolve, ms)) + +const RETRY_STRATEGY = { strategy: "backoff" as const, maxAttempts: 3 } + +const RETRY_BY_STATUS: RetryByStatus = { + "408": RETRY_STRATEGY, + "409": RETRY_STRATEGY, + "429": RETRY_STRATEGY, + "500-599": RETRY_STRATEGY, } /** - * Drop-in replacement for `fetch()` with built-in 429 and 5xx retry. + * Drop-in replacement for `fetch()` with built-in retry on transient errors. * Delegates to Trigger.dev's `retry.fetch` under the hood. + * + * Callers can override retry behavior for specific status codes: + * ```ts + * fetchRetry(url, { headers }, { "429": { strategy: "headers", ... } }) + * ``` */ export function fetchRetry( url: string | URL, - init?: RequestInit + init?: RequestInit, + retryByStatus?: RetryByStatus ): Promise { return retry.fetch(url, { ...init, - retry: { byStatus: RETRY_BY_STATUS }, + retry: { byStatus: { ...RETRY_BY_STATUS, ...retryByStatus } }, }) } diff --git a/src/data-layer/fetchers/fetchStablecoinsData.ts b/src/data-layer/fetchers/fetchStablecoinsData.ts index 1b0ecdcc77f..82080d6eeae 100644 --- a/src/data-layer/fetchers/fetchStablecoinsData.ts +++ b/src/data-layer/fetchers/fetchStablecoinsData.ts @@ -1,3 +1,5 @@ +import { fetchRetry } from "./fetchRetry" + const COINGECKO_API_BASE_URL = "https://api.coingecko.com/api/v3/coins/markets?vs_currency=usd&category=" const COINGECKO_API_URL_PARAMS = `&order=market_cap_desc&per_page=250&page=1&sparkline=false&x_cg_demo_api_key=${process.env.COINGECKO_API_KEY}` @@ -23,7 +25,7 @@ export async function fetchStablecoinsData(): Promise { console.log("Starting total ETH staked data fetch from Dune Analytics") - const response = await fetch(url, { + const response = await fetchRetry(url, { headers: { "X-Dune-API-Key": duneApiKey }, }) diff --git a/src/data-layer/fetchers/fetchTotalValueLocked.ts b/src/data-layer/fetchers/fetchTotalValueLocked.ts index 5da934ccfab..56a29d48c94 100644 --- a/src/data-layer/fetchers/fetchTotalValueLocked.ts +++ b/src/data-layer/fetchers/fetchTotalValueLocked.ts @@ -1,5 +1,7 @@ import type { DefiLlamaTVLResponse, MetricReturnData } from "@/lib/types" +import { fetchRetry } from "./fetchRetry" + export const FETCH_TOTAL_VALUE_LOCKED_TASK_ID = "fetch-total-value-locked" /** @@ -11,7 +13,7 @@ export async function fetchTotalValueLocked(): Promise { console.log("Starting total value locked data fetch from DefiLlama") - const response = await fetch(url) + const response = await fetchRetry(url) if (!response.ok) { const status = response.status diff --git a/src/data-layer/fetchers/fetchTranslationGlossary.ts b/src/data-layer/fetchers/fetchTranslationGlossary.ts index 13fc99db036..16b98708848 100644 --- a/src/data-layer/fetchers/fetchTranslationGlossary.ts +++ b/src/data-layer/fetchers/fetchTranslationGlossary.ts @@ -1,3 +1,5 @@ +import { fetchRetry } from "./fetchRetry" + export type GlossaryEntry = { string_term: string translation_text: string @@ -27,7 +29,7 @@ export async function fetchTranslationGlossary(): Promise { let offset = 0 while (hasMore) { const url = `${baseUrl}&limit=${pageSize}&offset=${offset}` - const response = await fetch(url, { + const response = await fetchRetry(url, { headers: { apikey: supabaseKey, Authorization: `Bearer ${supabaseKey}`, diff --git a/src/lib/utils/fetch.ts b/src/lib/utils/fetch.ts deleted file mode 100644 index e72c2aab998..00000000000 --- a/src/lib/utils/fetch.ts +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Utility functions for fetching data with retry logic and rate limiting. - */ - -/** - * Sleep for specified milliseconds. - */ -export const sleep = (ms: number) => - new Promise((resolve) => setTimeout(resolve, ms)) - -/** - * Retry a function with exponential backoff. - * - * @param fn - The async function to retry - * @param maxAttempts - Maximum number of attempts (default: 3) - * @param baseDelay - Base delay in milliseconds for exponential backoff (default: 1000) - * @returns The result of the function if successful - * @throws The last error if all attempts fail - */ -export async function retry( - fn: () => Promise, - maxAttempts = 3, - baseDelay = 1000 -): Promise { - let lastError: Error | undefined - - for (let attempt = 1; attempt <= maxAttempts; attempt++) { - try { - return await fn() - } catch (error) { - lastError = error as Error - if (attempt < maxAttempts) { - const delay = baseDelay * Math.pow(2, attempt - 1) - console.log( - `Attempt ${attempt} failed, retrying in ${delay}ms...`, - error - ) - await sleep(delay) - } - } - } - - throw lastError -}