diff --git a/packages/garmin-connect/src/client.ts b/packages/garmin-connect/src/client.ts index d3e58a276f..1035a3b512 100644 --- a/packages/garmin-connect/src/client.ts +++ b/packages/garmin-connect/src/client.ts @@ -10,11 +10,7 @@ * 6. All API calls use OAuth2 Bearer token */ -import { - createRateLimitAwareFetch, - ProviderRateLimitError, - parseRetryAfterHeader, -} from "@dofek/provider-http/rate-limit"; +import { ProviderRateLimitError, parseRetryAfterHeader } from "@dofek/provider-http/rate-limit"; import { buildOAuth1Header } from "./oauth1.ts"; import type { BodyBatteryDay, @@ -45,6 +41,10 @@ const OAUTH_CONSUMER_URL = "https://thegarth.s3.amazonaws.com/oauth_consumer.jso const USER_AGENT = "com.garmin.android.apps.connectmobile"; const API_USER_AGENT = "GCM-iOS-5.19.1.2"; +/** Minimum delay between consecutive Connect API requests (ms). + * Garmin's unofficial API rate limits aggressively; 2s keeps bursts under control. */ +export const GARMIN_CONNECT_THROTTLE_MS = 2_000; + const CSRF_RE = /name="_csrf"\s+value="(.+?)"/; const TITLE_RE = /(.+?)<\/title>/; const TICKET_RE = /embed\?ticket=([^"]+)"/; @@ -56,18 +56,9 @@ export class GarminConnectClient { #displayName: string | null = null; #domain: string; #fetchFn: typeof globalThis.fetch; - constructor(domain: string = "garmin.com", fetchFn: typeof globalThis.fetch = globalThis.fetch) { this.#domain = domain; - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { - providerId: "garmin", - createRateLimitError: (response, responseBody) => - new GarminRateLimitError( - `Rate limit exceeded (${response.status}): ${responseBody}`, - responseBody, - response.headers?.get?.("Retry-After"), - ), - }); + this.#fetchFn = fetchFn; } // ============================================================ @@ -346,12 +337,15 @@ export class GarminConnectClient { throw new GarminAuthError("Authentication failed (401)"); } - if (response.status === 429) { - throw new GarminRateLimitError("Rate limit exceeded (429)"); - } - if (!response.ok) { const text = await response.text(); + if (response.status === 429) { + throw new GarminRateLimitError( + `Rate limit exceeded (${response.status}): ${text}`, + text, + response.headers?.get?.("Retry-After"), + ); + } throw new GarminApiError(`API error (${response.status}): ${text}`, response.status); } diff --git a/packages/provider-http/package.json b/packages/provider-http/package.json index c611677c37..69d0910e90 100644 --- a/packages/provider-http/package.json +++ b/packages/provider-http/package.json @@ -4,6 +4,7 @@ "private": true, "type": "module", "exports": { - "./rate-limit": "./src/rate-limit.ts" + "./rate-limit": "./src/rate-limit.ts", + "./adaptive-rate-limit": "./src/adaptive-rate-limit.ts" } } diff --git a/packages/provider-http/src/adaptive-rate-limit.test.ts b/packages/provider-http/src/adaptive-rate-limit.test.ts new file mode 100644 index 0000000000..224ee39203 --- /dev/null +++ b/packages/provider-http/src/adaptive-rate-limit.test.ts @@ -0,0 +1,388 @@ +import { describe, expect, it } from "vitest"; +import { + ADAPTIVE_RATE_WINDOW_MS, + ADAPTIVE_THROTTLE_MAX_MS, + ADAPTIVE_THROTTLE_MIN_MS, + adaptiveRateLimitStorageKey, + admissionDelayMs, + applyStravaQuota, + blendObservedCooldown, + createInitialAdaptiveState, + decreaseThrottleMs, + defaultThrottleMs, + increaseThrottleMs, + learnInferredBudget, + parseAdaptiveRateState, + parseStravaRateLimitHeaders, + recordAdaptiveRateLimit, + recordAdaptiveRequest, + serializeAdaptiveRateState, + slideAdaptiveWindow, +} from "./adaptive-rate-limit.ts"; + +describe("defaultThrottleMs", () => { + it("returns provider-specific defaults", () => { + expect(defaultThrottleMs("strava")).toBe(10_000); + expect(defaultThrottleMs("garmin")).toBe(2_000); + expect(defaultThrottleMs("whoop")).toBe(1_000); + expect(defaultThrottleMs("unknown")).toBe(1_000); + }); +}); + +describe("parseStravaRateLimitHeaders", () => { + it("parses Strava short and daily quota headers", () => { + const headers = new Headers({ + "X-RateLimit-Limit": "100,1000", + "X-RateLimit-Usage": "42,500", + }); + expect(parseStravaRateLimitHeaders(headers)).toEqual({ + shortLimit: 100, + shortUsage: 42, + dailyLimit: 1000, + dailyUsage: 500, + }); + }); + + it("returns null when headers are missing", () => { + expect(parseStravaRateLimitHeaders(new Headers())).toBeNull(); + }); + + it("returns null when only one header is present", () => { + expect( + parseStravaRateLimitHeaders( + new Headers({ + "X-RateLimit-Limit": "100,1000", + }), + ), + ).toBeNull(); + }); + + it("returns null when header values are not numeric", () => { + expect( + parseStravaRateLimitHeaders( + new Headers({ + "X-RateLimit-Limit": "bad,1000", + "X-RateLimit-Usage": "1,2", + }), + ), + ).toBeNull(); + }); +}); + +describe("blendObservedCooldown", () => { + it("blends observed cooldown values toward recent observations", () => { + expect(blendObservedCooldown(600, 300)).toBe(510); + expect(blendObservedCooldown(null, 300)).toBe(300); + }); + + it("returns previous value when observed seconds are non-positive", () => { + expect(blendObservedCooldown(600, 0)).toBe(600); + expect(blendObservedCooldown(null, -5)).toBe(-5); + }); +}); + +describe("slideAdaptiveWindow", () => { + it("slides the rolling window without recomputing historical requests", () => { + const state = createInitialAdaptiveState("whoop", "provider", null, 0); + const withRequests = { ...state, requestCount: 12, windowStartMs: 0 }; + const slid = slideAdaptiveWindow(withRequests, ADAPTIVE_RATE_WINDOW_MS + 1); + expect(slid.requestCount).toBe(0); + expect(slid.windowStartMs).toBe(ADAPTIVE_RATE_WINDOW_MS + 1); + }); + + it("keeps state when the rolling window has not elapsed", () => { + const state = createInitialAdaptiveState("whoop", "provider", null, 1000); + const withRequests = { ...state, requestCount: 4 }; + expect(slideAdaptiveWindow(withRequests, 2000)).toBe(withRequests); + }); +}); + +describe("throttle adjustment helpers", () => { + it("decreases throttle gradually on success without dropping below the minimum", () => { + expect(decreaseThrottleMs(1000)).toBe(900); + expect(decreaseThrottleMs(ADAPTIVE_THROTTLE_MIN_MS)).toBe(ADAPTIVE_THROTTLE_MIN_MS); + }); + + it("increases throttle on rate limits without exceeding the maximum", () => { + expect(increaseThrottleMs(1000)).toBe(2000); + expect(increaseThrottleMs(ADAPTIVE_THROTTLE_MAX_MS)).toBe(ADAPTIVE_THROTTLE_MAX_MS); + }); +}); + +describe("learnInferredBudget", () => { + it("learns a lower inferred budget from observed failures", () => { + expect(learnInferredBudget(30, 25)).toBe(24); + expect(learnInferredBudget(null, 10)).toBe(9); + }); + + it("returns the current budget when no requests were observed at the limit", () => { + expect(learnInferredBudget(30, 0)).toBe(30); + expect(learnInferredBudget(null, 0)).toBeNull(); + }); +}); + +describe("admissionDelayMs", () => { + it("waits for the remaining throttle interval after the last request", () => { + const state = { + ...createInitialAdaptiveState("whoop", "provider", null, 0), + throttleMs: 1000, + lastRequestMs: 100, + }; + expect(admissionDelayMs(state, 500)).toBe(600); + }); + + it("adds delay when the inferred budget soft cap is reached", () => { + const state = { + ...createInitialAdaptiveState("garmin", "provider", null, 0), + throttleMs: 1000, + inferredBudget: 10, + requestCount: 8, + }; + expect(admissionDelayMs(state, 0)).toBe(1000); + }); + + it("paces Strava requests when short quota is nearly exhausted", () => { + const state = applyStravaQuota(createInitialAdaptiveState("strava", "provider", null), { + shortLimit: 100, + shortUsage: 99, + dailyLimit: 1000, + dailyUsage: 100, + }); + expect(admissionDelayMs(state, Date.now())).toBeGreaterThanOrEqual(state.throttleMs * 4); + }); + + it("doubles Strava pacing delay when only a few short-quota requests remain", () => { + const state = applyStravaQuota(createInitialAdaptiveState("strava", "provider", null), { + shortLimit: 100, + shortUsage: 97, + dailyLimit: 1000, + dailyUsage: 100, + }); + expect(admissionDelayMs(state, 0)).toBe(state.throttleMs * 2); + }); + + it("paces Strava requests proportionally when quota headroom remains", () => { + const state = applyStravaQuota(createInitialAdaptiveState("strava", "provider", null), { + shortLimit: 100, + shortUsage: 50, + dailyLimit: 1000, + dailyUsage: 100, + }); + expect(admissionDelayMs(state, 0)).toBe(18_000); + }); + + it("ignores Strava pacing for non-Strava providers", () => { + const state = { + ...createInitialAdaptiveState("garmin", "provider", null), + stravaShortLimit: 100, + stravaShortUsage: 99, + }; + expect(admissionDelayMs(state, 0)).toBe(0); + }); +}); + +describe("createInitialAdaptiveState", () => { + it("creates user-scoped state with the provided user id", () => { + const state = createInitialAdaptiveState("whoop", "user", "user-1", 1234); + expect(state).toMatchObject({ + providerId: "whoop", + scope: "user", + userId: "user-1", + windowStartMs: 1234, + requestCount: 0, + throttleMs: 1000, + }); + }); +}); + +describe("applyStravaQuota", () => { + it("stores parsed Strava quota fields on the adaptive state", () => { + const state = applyStravaQuota(createInitialAdaptiveState("strava", "provider", null), { + shortLimit: 100, + shortUsage: 10, + dailyLimit: 1000, + dailyUsage: 20, + }); + expect(state).toMatchObject({ + stravaShortLimit: 100, + stravaShortUsage: 10, + stravaDailyLimit: 1000, + stravaDailyUsage: 20, + }); + }); +}); + +describe("recordAdaptiveRequest", () => { + it("records requests by incrementing the rolling tally", () => { + const state = createInitialAdaptiveState("garmin", "provider", null, 1000); + const next = recordAdaptiveRequest(state, 1500); + expect(next.requestCount).toBe(1); + expect(next.lastRequestMs).toBe(1500); + expect(next.throttleMs).toBeLessThan(state.throttleMs); + }); +}); + +describe("recordAdaptiveRateLimit", () => { + it("learns inferred budget and increases throttle on rate limits", () => { + const state = { + ...createInitialAdaptiveState("whoop", "provider", null), + requestCount: 25, + throttleMs: 1000, + inferredBudget: 30, + }; + const limited = recordAdaptiveRateLimit(state, 120); + expect(limited.inferredBudget).toBe(24); + expect(limited.throttleMs).toBe(2000); + expect(limited.observedCooldownSeconds).toBe(120); + }); + + it("preserves observed cooldown when retry-after is absent or zero", () => { + const state = { + ...createInitialAdaptiveState("whoop", "provider", null), + observedCooldownSeconds: 240, + requestCount: 5, + }; + expect(recordAdaptiveRateLimit(state, null).observedCooldownSeconds).toBe(240); + expect(recordAdaptiveRateLimit(state, 0).observedCooldownSeconds).toBe(240); + }); +}); + +describe("adaptiveRateLimitStorageKey", () => { + it("builds provider and user scoped keys", () => { + expect(adaptiveRateLimitStorageKey("garmin", "provider", null)).toBe( + "provider-adaptive-rate:garmin:provider", + ); + expect(adaptiveRateLimitStorageKey("whoop", "user", "user-1")).toBe( + "provider-adaptive-rate:whoop:user:user-1", + ); + expect(adaptiveRateLimitStorageKey("whoop", "user", null)).toBe( + "provider-adaptive-rate:whoop:user:unknown", + ); + }); +}); + +describe("serializeAdaptiveRateState", () => { + it("serializes adaptive state as JSON", () => { + const state = createInitialAdaptiveState("strava", "provider", null); + expect(JSON.parse(serializeAdaptiveRateState(state))).toEqual(state); + }); +}); + +describe("parseAdaptiveRateState", () => { + it("returns null for empty input", () => { + expect(parseAdaptiveRateState(null)).toBeNull(); + expect(parseAdaptiveRateState("")).toBeNull(); + }); + + it("returns null for malformed JSON", () => { + expect(parseAdaptiveRateState("{not-json")).toBeNull(); + }); + + it("parses valid persisted state with optional Strava fields", () => { + const state = { + ...createInitialAdaptiveState("strava", "provider", null), + inferredBudget: 35, + observedCooldownSeconds: 90, + stravaShortLimit: 100, + stravaShortUsage: 80, + stravaDailyLimit: 1000, + stravaDailyUsage: 400, + }; + const parsed = parseAdaptiveRateState(serializeAdaptiveRateState(state)); + expect(parsed).toEqual(state); + }); + + it("rejects invalid scope and numeric fields", () => { + expect( + parseAdaptiveRateState( + JSON.stringify({ + providerId: "garmin", + scope: "invalid", + userId: null, + windowStartMs: 1, + requestCount: 0, + throttleMs: 1, + lastRequestMs: null, + }), + ), + ).toBeNull(); + expect( + parseAdaptiveRateState( + JSON.stringify({ + providerId: 1, + scope: "provider", + userId: null, + windowStartMs: 1, + requestCount: 0, + throttleMs: 1, + lastRequestMs: null, + }), + ), + ).toBeNull(); + expect( + parseAdaptiveRateState( + JSON.stringify({ + providerId: "garmin", + scope: "provider", + userId: null, + windowStartMs: "bad", + requestCount: 0, + throttleMs: 1, + lastRequestMs: null, + }), + ), + ).toBeNull(); + }); + + it("rejects invalid userId and lastRequestMs", () => { + expect( + parseAdaptiveRateState( + JSON.stringify({ + providerId: "whoop", + scope: "user", + userId: 42, + windowStartMs: 1, + requestCount: 0, + throttleMs: 1, + lastRequestMs: null, + }), + ), + ).toBeNull(); + expect( + parseAdaptiveRateState( + JSON.stringify({ + providerId: "garmin", + scope: "provider", + userId: null, + windowStartMs: 1, + requestCount: 0, + throttleMs: 1, + lastRequestMs: "now", + }), + ), + ).toBeNull(); + }); + + it("drops non-finite optional numeric fields", () => { + const parsed = parseAdaptiveRateState( + JSON.stringify({ + providerId: "garmin", + scope: "provider", + userId: null, + windowStartMs: 1, + requestCount: 0, + throttleMs: 1, + lastRequestMs: null, + inferredBudget: "lots", + observedCooldownSeconds: "slow", + stravaShortLimit: "x", + stravaShortUsage: "y", + stravaDailyLimit: "z", + stravaDailyUsage: "w", + }), + ); + expect(parsed?.inferredBudget).toBeNull(); + expect(parsed?.observedCooldownSeconds).toBeNull(); + expect(parsed?.stravaShortLimit).toBeNull(); + }); +}); diff --git a/packages/provider-http/src/adaptive-rate-limit.ts b/packages/provider-http/src/adaptive-rate-limit.ts new file mode 100644 index 0000000000..db70ceafd7 --- /dev/null +++ b/packages/provider-http/src/adaptive-rate-limit.ts @@ -0,0 +1,293 @@ +import type { ProviderRateLimitScope } from "./rate-limit-types.ts"; + +/** Rolling window for request budget tally (5 minutes). */ +export const ADAPTIVE_RATE_WINDOW_MS = 5 * 60 * 1000; + +export const ADAPTIVE_THROTTLE_MIN_MS = 500; +export const ADAPTIVE_THROTTLE_MAX_MS = 30_000; +export const ADAPTIVE_THROTTLE_DECREASE_MS = 100; +export const ADAPTIVE_THROTTLE_INCREASE_FACTOR = 2; +export const ADAPTIVE_BUDGET_SAFETY_RATIO = 0.8; +export const ADAPTIVE_DEFAULT_INFERRED_BUDGET = 40; + +export const DEFAULT_PROVIDER_THROTTLE_MS: Readonly<Record<string, number>> = { + strava: 10_000, + garmin: 2_000, + whoop: 1_000, +}; + +export interface StravaRateLimitQuota { + shortLimit: number; + shortUsage: number; + dailyLimit: number; + dailyUsage: number; +} + +export interface ProviderAdaptiveRateState { + providerId: string; + scope: ProviderRateLimitScope; + userId: string | null; + windowStartMs: number; + requestCount: number; + throttleMs: number; + lastRequestMs: number | null; + inferredBudget: number | null; + observedCooldownSeconds: number | null; + stravaShortLimit: number | null; + stravaShortUsage: number | null; + stravaDailyLimit: number | null; + stravaDailyUsage: number | null; +} + +export type { AdaptiveRateLimitStore } from "./rate-limit-types.ts"; + +export function defaultThrottleMs(providerId: string): number { + return DEFAULT_PROVIDER_THROTTLE_MS[providerId] ?? 1_000; +} + +export function parseStravaRateLimitHeaders(headers: Headers): StravaRateLimitQuota | null { + const limitHeader = headers.get("X-RateLimit-Limit"); + const usageHeader = headers.get("X-RateLimit-Usage"); + if (!limitHeader || !usageHeader) return null; + + const [shortLimitRaw, dailyLimitRaw] = limitHeader.split(","); + const [shortUsageRaw, dailyUsageRaw] = usageHeader.split(","); + const shortLimit = Number.parseInt(shortLimitRaw ?? "", 10); + const dailyLimit = Number.parseInt(dailyLimitRaw ?? "", 10); + const shortUsage = Number.parseInt(shortUsageRaw ?? "", 10); + const dailyUsage = Number.parseInt(dailyUsageRaw ?? "", 10); + if ( + !Number.isFinite(shortLimit) || + !Number.isFinite(dailyLimit) || + !Number.isFinite(shortUsage) || + !Number.isFinite(dailyUsage) + ) { + return null; + } + return { shortLimit, shortUsage, dailyLimit, dailyUsage }; +} + +export function blendObservedCooldown(previous: number | null, observedSeconds: number): number { + if (!Number.isFinite(observedSeconds) || observedSeconds <= 0) return previous ?? observedSeconds; + if (previous == null) return observedSeconds; + return Math.round(previous * 0.7 + observedSeconds * 0.3); +} + +export function slideAdaptiveWindow( + state: ProviderAdaptiveRateState, + nowMs: number, +): ProviderAdaptiveRateState { + if (nowMs - state.windowStartMs < ADAPTIVE_RATE_WINDOW_MS) return state; + return { + ...state, + windowStartMs: nowMs, + requestCount: 0, + }; +} + +export function decreaseThrottleMs(throttleMs: number): number { + return Math.max(ADAPTIVE_THROTTLE_MIN_MS, throttleMs - ADAPTIVE_THROTTLE_DECREASE_MS); +} + +export function increaseThrottleMs(throttleMs: number): number { + return Math.min(ADAPTIVE_THROTTLE_MAX_MS, throttleMs * ADAPTIVE_THROTTLE_INCREASE_FACTOR); +} + +export function learnInferredBudget( + current: number | null, + requestsAtLimit: number, +): number | null { + if (requestsAtLimit <= 0) return current; + const candidate = Math.max(1, requestsAtLimit - 1); + if (current == null) return candidate; + return Math.min(current, candidate); +} + +export function admissionDelayMs(state: ProviderAdaptiveRateState, nowMs: number): number { + let delayMs = 0; + + if (state.lastRequestMs != null) { + const elapsed = nowMs - state.lastRequestMs; + if (elapsed < state.throttleMs) { + delayMs = state.throttleMs - elapsed; + } + } + + if (state.inferredBudget != null) { + const softCap = Math.floor(state.inferredBudget * ADAPTIVE_BUDGET_SAFETY_RATIO); + if (state.requestCount >= softCap) { + delayMs = Math.max(delayMs, state.throttleMs); + } + } + + if ( + state.providerId === "strava" && + state.stravaShortLimit != null && + state.stravaShortUsage != null + ) { + const remaining = state.stravaShortLimit - state.stravaShortUsage; + if (remaining <= 2) { + delayMs = Math.max(delayMs, state.throttleMs * 4); + } else if (remaining <= 5) { + delayMs = Math.max(delayMs, state.throttleMs * 2); + } else { + const windowMs = 15 * 60 * 1000; + const pacedDelay = Math.ceil(windowMs / remaining); + delayMs = Math.max(delayMs, Math.min(pacedDelay, ADAPTIVE_THROTTLE_MAX_MS)); + } + } + + return delayMs; +} + +export function createInitialAdaptiveState( + providerId: string, + scope: ProviderRateLimitScope, + userId: string | null, + nowMs = Date.now(), +): ProviderAdaptiveRateState { + return { + providerId, + scope, + userId, + windowStartMs: nowMs, + requestCount: 0, + throttleMs: defaultThrottleMs(providerId), + lastRequestMs: null, + inferredBudget: null, + observedCooldownSeconds: null, + stravaShortLimit: null, + stravaShortUsage: null, + stravaDailyLimit: null, + stravaDailyUsage: null, + }; +} + +export function applyStravaQuota( + state: ProviderAdaptiveRateState, + quota: StravaRateLimitQuota, +): ProviderAdaptiveRateState { + return { + ...state, + stravaShortLimit: quota.shortLimit, + stravaShortUsage: quota.shortUsage, + stravaDailyLimit: quota.dailyLimit, + stravaDailyUsage: quota.dailyUsage, + }; +} + +export function recordAdaptiveRequest( + state: ProviderAdaptiveRateState, + nowMs: number, +): ProviderAdaptiveRateState { + const slid = slideAdaptiveWindow(state, nowMs); + return { + ...slid, + requestCount: slid.requestCount + 1, + lastRequestMs: nowMs, + throttleMs: decreaseThrottleMs(slid.throttleMs), + }; +} + +export function recordAdaptiveRateLimit( + state: ProviderAdaptiveRateState, + retryAfterSeconds: number | null | undefined, +): ProviderAdaptiveRateState { + const observedCooldown = + retryAfterSeconds != null && retryAfterSeconds > 0 + ? blendObservedCooldown(state.observedCooldownSeconds, retryAfterSeconds) + : state.observedCooldownSeconds; + + return { + ...state, + throttleMs: increaseThrottleMs(state.throttleMs), + inferredBudget: learnInferredBudget(state.inferredBudget, state.requestCount), + observedCooldownSeconds: observedCooldown, + }; +} + +export const ADAPTIVE_RATE_STORAGE_KEY_PREFIX = "provider-adaptive-rate"; + +export function adaptiveRateLimitStorageKey( + providerId: string, + scope: ProviderRateLimitScope, + userId: string | null, +): string { + return scope === "provider" + ? `${ADAPTIVE_RATE_STORAGE_KEY_PREFIX}:${providerId}:provider` + : `${ADAPTIVE_RATE_STORAGE_KEY_PREFIX}:${providerId}:user:${userId ?? "unknown"}`; +} + +export function serializeAdaptiveRateState(state: ProviderAdaptiveRateState): string { + return JSON.stringify(state); +} + +export function parseAdaptiveRateState(raw: string | null): ProviderAdaptiveRateState | null { + if (!raw) return null; + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return null; + } + if (typeof parsed !== "object" || parsed === null) return null; + + const providerId = Reflect.get(parsed, "providerId"); + const scope = Reflect.get(parsed, "scope"); + const userId = Reflect.get(parsed, "userId"); + const windowStartMs = Reflect.get(parsed, "windowStartMs"); + const requestCount = Reflect.get(parsed, "requestCount"); + const throttleMs = Reflect.get(parsed, "throttleMs"); + const lastRequestMs = Reflect.get(parsed, "lastRequestMs"); + const inferredBudget = Reflect.get(parsed, "inferredBudget"); + const observedCooldownSeconds = Reflect.get(parsed, "observedCooldownSeconds"); + const stravaShortLimit = Reflect.get(parsed, "stravaShortLimit"); + const stravaShortUsage = Reflect.get(parsed, "stravaShortUsage"); + const stravaDailyLimit = Reflect.get(parsed, "stravaDailyLimit"); + const stravaDailyUsage = Reflect.get(parsed, "stravaDailyUsage"); + + if (typeof providerId !== "string") return null; + if (scope !== "provider" && scope !== "user") return null; + if (userId !== null && typeof userId !== "string") return null; + if (typeof windowStartMs !== "number" || !Number.isFinite(windowStartMs)) return null; + if (typeof requestCount !== "number" || !Number.isFinite(requestCount)) return null; + if (typeof throttleMs !== "number" || !Number.isFinite(throttleMs)) return null; + if ( + lastRequestMs !== null && + (typeof lastRequestMs !== "number" || !Number.isFinite(lastRequestMs)) + ) { + return null; + } + + return { + providerId, + scope, + userId, + windowStartMs, + requestCount, + throttleMs, + lastRequestMs, + inferredBudget: + typeof inferredBudget === "number" && Number.isFinite(inferredBudget) ? inferredBudget : null, + observedCooldownSeconds: + typeof observedCooldownSeconds === "number" && Number.isFinite(observedCooldownSeconds) + ? observedCooldownSeconds + : null, + stravaShortLimit: + typeof stravaShortLimit === "number" && Number.isFinite(stravaShortLimit) + ? stravaShortLimit + : null, + stravaShortUsage: + typeof stravaShortUsage === "number" && Number.isFinite(stravaShortUsage) + ? stravaShortUsage + : null, + stravaDailyLimit: + typeof stravaDailyLimit === "number" && Number.isFinite(stravaDailyLimit) + ? stravaDailyLimit + : null, + stravaDailyUsage: + typeof stravaDailyUsage === "number" && Number.isFinite(stravaDailyUsage) + ? stravaDailyUsage + : null, + }; +} diff --git a/packages/provider-http/src/rate-limit-types.ts b/packages/provider-http/src/rate-limit-types.ts new file mode 100644 index 0000000000..6ba2bf9697 --- /dev/null +++ b/packages/provider-http/src/rate-limit-types.ts @@ -0,0 +1,78 @@ +export type ProviderRateLimitScope = "provider" | "user"; +export type ProviderHttpErrorScope = ProviderRateLimitScope; + +export interface ProviderRateLimitErrorOptions { + message: string; + providerId: string; + statusCode: number; + responseBody: string; + scope?: ProviderRateLimitScope; + userId?: string | null; + retryAfterSeconds?: number | null; +} + +export interface ProviderServiceUnavailableErrorOptions { + message: string; + providerId: string; + statusCode: number; + responseBody: string; + scope?: ProviderHttpErrorScope; + userId?: string | null; + retryAfterSeconds?: number | null; +} + +export class ProviderRateLimitError extends Error { + readonly providerId: string; + readonly statusCode: number; + readonly responseBody: string; + readonly scope: ProviderRateLimitScope; + readonly userId: string | null; + readonly retryAfterSeconds: number | null; + + constructor(options: ProviderRateLimitErrorOptions) { + super(options.message); + this.name = "ProviderRateLimitError"; + this.providerId = options.providerId; + this.statusCode = options.statusCode; + this.responseBody = options.responseBody; + this.scope = options.scope ?? "provider"; + this.userId = options.userId ?? null; + this.retryAfterSeconds = options.retryAfterSeconds ?? null; + } +} + +export class ProviderServiceUnavailableError extends Error { + readonly providerId: string; + readonly statusCode: number; + readonly responseBody: string; + readonly scope: ProviderHttpErrorScope; + readonly userId: string | null; + readonly retryAfterSeconds: number | null; + + constructor(options: ProviderServiceUnavailableErrorOptions) { + super(options.message); + this.name = "ProviderServiceUnavailableError"; + this.providerId = options.providerId; + this.statusCode = options.statusCode; + this.responseBody = options.responseBody; + this.scope = options.scope ?? "provider"; + this.userId = options.userId ?? null; + this.retryAfterSeconds = options.retryAfterSeconds ?? null; + } +} + +export interface AdaptiveRateLimitStore { + awaitAdmission( + providerId: string, + scope: ProviderRateLimitScope, + userId: string | null, + ): Promise<void>; + recordSuccess( + providerId: string, + scope: ProviderRateLimitScope, + userId: string | null, + responseHeaders?: Headers, + ): Promise<void>; + recordRateLimit(error: ProviderRateLimitError): Promise<void>; + getLearnedCooldownSeconds(providerId: string): Promise<number | null>; +} diff --git a/packages/provider-http/src/rate-limit.test.ts b/packages/provider-http/src/rate-limit.test.ts index d1966f4d5a..257899e847 100644 --- a/packages/provider-http/src/rate-limit.test.ts +++ b/packages/provider-http/src/rate-limit.test.ts @@ -6,6 +6,21 @@ import { ProviderServiceUnavailableError, parseRetryAfterHeader, } from "./rate-limit.ts"; +import type { AdaptiveRateLimitStore } from "./rate-limit-types.ts"; + +function createMockAdaptiveStore(): AdaptiveRateLimitStore & { + awaitAdmission: ReturnType<typeof vi.fn>; + recordSuccess: ReturnType<typeof vi.fn>; + recordRateLimit: ReturnType<typeof vi.fn>; + getLearnedCooldownSeconds: ReturnType<typeof vi.fn>; +} { + return { + awaitAdmission: vi.fn().mockResolvedValue(undefined), + recordSuccess: vi.fn().mockResolvedValue(undefined), + recordRateLimit: vi.fn().mockResolvedValue(undefined), + getLearnedCooldownSeconds: vi.fn().mockResolvedValue(null), + }; +} class TestRateLimitError extends ProviderRateLimitError { constructor(response: Response, body: string) { @@ -371,6 +386,108 @@ describe("fetchWithRateLimitHandling", () => { expect(secondError).toHaveProperty("providerId", "second"); expect(first).not.toBe(second); }); + + it("awaits adaptive admission before calling fetch when a store is provided", async () => { + const fetchFn = vi.fn<typeof globalThis.fetch>().mockResolvedValue(response(200, "ok")); + const adaptiveStore = createMockAdaptiveStore(); + const rateLimitFetch = createRateLimitAwareFetch(fetchFn, { + providerId: "example", + scope: "user", + userId: "user-9", + adaptiveStore, + }); + + await rateLimitFetch("https://api.example.com/data"); + + expect(adaptiveStore.awaitAdmission).toHaveBeenCalledOnce(); + expect(adaptiveStore.awaitAdmission).toHaveBeenCalledWith("example", "user", "user-9"); + expect(fetchFn).toHaveBeenCalledOnce(); + }); + + it("records adaptive success with response headers on ok responses", async () => { + const fetchFn = vi.fn<typeof globalThis.fetch>().mockResolvedValue( + new Response("ok", { + status: 200, + headers: { "X-RateLimit-Limit": "100,1000" }, + }), + ); + const adaptiveStore = createMockAdaptiveStore(); + const rateLimitFetch = createRateLimitAwareFetch(fetchFn, { + providerId: "example", + scope: "provider", + adaptiveStore, + }); + + await rateLimitFetch("https://api.example.com/data"); + + expect(adaptiveStore.recordSuccess).toHaveBeenCalledOnce(); + expect(adaptiveStore.recordSuccess).toHaveBeenCalledWith( + "example", + "provider", + null, + expect.any(Headers), + ); + }); + + it("does not record adaptive success when the response is not ok", async () => { + const fetchFn = vi.fn<typeof globalThis.fetch>().mockResolvedValue(response(404, "missing")); + const adaptiveStore = createMockAdaptiveStore(); + const rateLimitFetch = createRateLimitAwareFetch(fetchFn, { + providerId: "example", + adaptiveStore, + }); + + const result = await rateLimitFetch("https://api.example.com/data"); + + expect(result.status).toBe(404); + expect(adaptiveStore.recordSuccess).not.toHaveBeenCalled(); + }); + + it("records adaptive rate limits when a provider rate-limit error is thrown", async () => { + const fetchFn = vi.fn<typeof globalThis.fetch>().mockResolvedValue(response(429, "limited")); + const adaptiveStore = createMockAdaptiveStore(); + const rateLimitFetch = createRateLimitAwareFetch(fetchFn, { + providerId: "example", + scope: "user", + userId: "user-9", + adaptiveStore, + }); + + const error = await rateLimitFetch("https://api.example.com/data").catch( + (caughtError: unknown) => caughtError, + ); + + expect(error).toBeInstanceOf(ProviderRateLimitError); + expect(adaptiveStore.recordRateLimit).toHaveBeenCalledOnce(); + expect(adaptiveStore.recordRateLimit).toHaveBeenCalledWith(error); + }); + + it("does not record adaptive rate limits for service-unavailable responses", async () => { + const fetchFn = vi.fn<typeof globalThis.fetch>().mockResolvedValue(response(503, "down")); + const adaptiveStore = createMockAdaptiveStore(); + const rateLimitFetch = createRateLimitAwareFetch(fetchFn, { + providerId: "example", + adaptiveStore, + }); + + await rateLimitFetch("https://api.example.com/data").catch(() => undefined); + + expect(adaptiveStore.recordRateLimit).not.toHaveBeenCalled(); + }); + + it("does not record adaptive rate limits for custom non-provider rate-limit errors", async () => { + const fetchFn = vi.fn<typeof globalThis.fetch>().mockResolvedValue(response(429, "limited")); + const adaptiveStore = createMockAdaptiveStore(); + const rateLimitFetch = createRateLimitAwareFetch(fetchFn, { + providerId: "example", + adaptiveStore, + createRateLimitError: (_response, body) => new Error(`custom ${body}`), + }); + + await rateLimitFetch("https://api.example.com/data").catch(() => undefined); + + expect(adaptiveStore.recordRateLimit).not.toHaveBeenCalled(); + }); }); describe("parseRetryAfterHeader", () => { diff --git a/packages/provider-http/src/rate-limit.ts b/packages/provider-http/src/rate-limit.ts index 444c384c41..566b1660fd 100644 --- a/packages/provider-http/src/rate-limit.ts +++ b/packages/provider-http/src/rate-limit.ts @@ -1,65 +1,19 @@ -export interface ProviderRateLimitErrorOptions { - message: string; - providerId: string; - statusCode: number; - responseBody: string; - scope?: ProviderRateLimitScope; - userId?: string | null; - retryAfterSeconds?: number | null; -} - -export type ProviderRateLimitScope = "provider" | "user"; -export type ProviderHttpErrorScope = ProviderRateLimitScope; - -export interface ProviderServiceUnavailableErrorOptions { - message: string; - providerId: string; - statusCode: number; - responseBody: string; - scope?: ProviderHttpErrorScope; - userId?: string | null; - retryAfterSeconds?: number | null; -} - -export class ProviderRateLimitError extends Error { - readonly providerId: string; - readonly statusCode: number; - readonly responseBody: string; - readonly scope: ProviderRateLimitScope; - readonly userId: string | null; - readonly retryAfterSeconds: number | null; - - constructor(options: ProviderRateLimitErrorOptions) { - super(options.message); - this.name = "ProviderRateLimitError"; - this.providerId = options.providerId; - this.statusCode = options.statusCode; - this.responseBody = options.responseBody; - this.scope = options.scope ?? "provider"; - this.userId = options.userId ?? null; - this.retryAfterSeconds = options.retryAfterSeconds ?? null; - } -} - -export class ProviderServiceUnavailableError extends Error { - readonly providerId: string; - readonly statusCode: number; - readonly responseBody: string; - readonly scope: ProviderHttpErrorScope; - readonly userId: string | null; - readonly retryAfterSeconds: number | null; - - constructor(options: ProviderServiceUnavailableErrorOptions) { - super(options.message); - this.name = "ProviderServiceUnavailableError"; - this.providerId = options.providerId; - this.statusCode = options.statusCode; - this.responseBody = options.responseBody; - this.scope = options.scope ?? "provider"; - this.userId = options.userId ?? null; - this.retryAfterSeconds = options.retryAfterSeconds ?? null; - } -} +import type { AdaptiveRateLimitStore } from "./rate-limit-types.ts"; +import { + type ProviderHttpErrorScope, + ProviderRateLimitError, + type ProviderRateLimitScope, + ProviderServiceUnavailableError, +} from "./rate-limit-types.ts"; + +export type { + AdaptiveRateLimitStore, + ProviderHttpErrorScope, + ProviderRateLimitErrorOptions, + ProviderRateLimitScope, + ProviderServiceUnavailableErrorOptions, +} from "./rate-limit-types.ts"; +export { ProviderRateLimitError, ProviderServiceUnavailableError }; export interface FetchRateLimitHandlingOptions { createRateLimitError: (response: Response, responseBody: string) => Error; @@ -72,6 +26,7 @@ export interface RateLimitAwareFetchOptions { userId?: string | null; createRateLimitError?: (response: Response, responseBody: string) => Error; createServiceUnavailableError?: (response: Response, responseBody: string) => Error; + adaptiveStore?: AdaptiveRateLimitStore; } const rateLimitAwareFetches = new WeakSet<typeof globalThis.fetch>(); @@ -167,29 +122,48 @@ export function createRateLimitAwareFetch( ): typeof globalThis.fetch { if (rateLimitAwareFetches.has(fetchFn)) return fetchFn; - const rateLimitFetch: typeof globalThis.fetch = (input, init) => - fetchWithRateLimitHandling(fetchFn, input, init, { - createRateLimitError: - options.createRateLimitError ?? - ((response, responseBody) => - createDefaultRateLimitError( - options.providerId, - options.scope ?? "provider", - options.userId ?? null, - response, - responseBody, - )), - createServiceUnavailableError: - options.createServiceUnavailableError ?? - ((response, responseBody) => - createDefaultServiceUnavailableError( - options.providerId, - options.scope ?? "provider", - options.userId ?? null, - response, - responseBody, - )), - }); + const rateLimitFetch: typeof globalThis.fetch = async (input, init) => { + const scope = options.scope ?? "provider"; + const userId = options.userId ?? null; + if (options.adaptiveStore) { + await options.adaptiveStore.awaitAdmission(options.providerId, scope, userId); + } + + try { + const response = await fetchWithRateLimitHandling(fetchFn, input, init, { + createRateLimitError: + options.createRateLimitError ?? + ((response, responseBody) => + createDefaultRateLimitError(options.providerId, scope, userId, response, responseBody)), + createServiceUnavailableError: + options.createServiceUnavailableError ?? + ((response, responseBody) => + createDefaultServiceUnavailableError( + options.providerId, + scope, + userId, + response, + responseBody, + )), + }); + + if (options.adaptiveStore && response.ok) { + await options.adaptiveStore.recordSuccess( + options.providerId, + scope, + userId, + response.headers, + ); + } + + return response; + } catch (err) { + if (options.adaptiveStore && err instanceof ProviderRateLimitError) { + await options.adaptiveStore.recordRateLimit(err); + } + throw err; + } + }; rateLimitAwareFetches.add(rateLimitFetch); return rateLimitFetch; } diff --git a/packages/server/src/mcp/route.test.ts b/packages/server/src/mcp/route.test.ts index cdc961a0c8..bb2e5f0b63 100644 --- a/packages/server/src/mcp/route.test.ts +++ b/packages/server/src/mcp/route.test.ts @@ -80,6 +80,8 @@ vi.mock("dofek/providers/registry", () => ({ registerProvider: vi.fn(), })); +import * as enqueueSyncJobModule from "dofek/jobs/enqueue-sync-job"; + vi.mock("@sentry/node", () => ({ captureException: vi.fn(), })); @@ -712,6 +714,7 @@ describe("createMcpRouter", () => { it("enqueues provider sync jobs for configured providers", async () => { authorizeMcpToken(); + const enqueueSpy = vi.spyOn(enqueueSyncJobModule, "enqueueSyncJob"); toolTestMocks.getAllProviders.mockReturnValue([ { id: "strava", @@ -729,6 +732,15 @@ describe("createMcpRouter", () => { }), }); + expect(enqueueSpy).toHaveBeenCalledWith( + "wahoo", + expect.objectContaining({ + providerId: "wahoo", + sinceDays: 7, + userId: "user-id", + }), + { skipWhenRateLimited: true }, + ); expect(toolTestMocks.getProviderSyncQueue).toHaveBeenCalledWith("wahoo"); expect(toolTestMocks.queueAdd).toHaveBeenCalledWith( "sync", @@ -751,6 +763,30 @@ describe("createMcpRouter", () => { }); }); + it("returns a tool error when sync enqueue is skipped for rate-limit cooldown", async () => { + authorizeMcpToken(); + vi.spyOn(enqueueSyncJobModule, "enqueueSyncJob").mockResolvedValueOnce(null); + toolTestMocks.getAllProviders.mockReturnValue([ + { id: "wahoo", name: "Wahoo", validate: () => null }, + ]); + + const response = await request(createTestApp(), { + authorization: "Bearer good-token", + body: createToolCallRequest("start_provider_sync", { + providerId: "wahoo", + sinceDays: 7, + }), + }); + + const parsedResponse = toolCallResponseSchema.parse(parseJsonRpcEvent(response.text)); + expect(parsedResponse.result.isError).toBe(true); + expect(parsedResponse.result.content[0]?.text).toBe( + "Provider wahoo sync skipped: rate-limit cooldown active", + ); + expect(toolTestMocks.queueAdd).not.toHaveBeenCalled(); + expect(toolTestMocks.startWorker).not.toHaveBeenCalled(); + }); + it("returns tool errors for unknown providers", async () => { authorizeMcpToken(); diff --git a/packages/server/src/mcp/tools.ts b/packages/server/src/mcp/tools.ts index 966c58f640..ea31e93300 100644 --- a/packages/server/src/mcp/tools.ts +++ b/packages/server/src/mcp/tools.ts @@ -252,11 +252,18 @@ export function createDofekMcpServer(context: DofekMcpContext): McpServer { sinceDate, untilDate, }); - const job = await enqueueSyncJob(providerId, { + const job = await enqueueSyncJob( providerId, - userId: context.userId, - ...syncWindowToJobData(syncWindow, sinceDays), - }); + { + providerId, + userId: context.userId, + ...syncWindowToJobData(syncWindow, sinceDays), + }, + { skipWhenRateLimited: true }, + ); + if (!job) { + throw new Error(`Provider ${providerId} sync skipped: rate-limit cooldown active`); + } startWorker(); return jsonContent({ providerId, diff --git a/packages/server/src/routers/sync.test.ts b/packages/server/src/routers/sync.test.ts index 4dd39afefa..7c645839fb 100644 --- a/packages/server/src/routers/sync.test.ts +++ b/packages/server/src/routers/sync.test.ts @@ -141,6 +141,7 @@ vi.mock("dofek/db/schema", () => ({ }, })); +import * as enqueueSyncJobModule from "dofek/jobs/enqueue-sync-job"; import { logsInput, sanitizeErrorMessage, @@ -798,6 +799,22 @@ describe("syncRouter", () => { expect(result.jobIds).toHaveLength(1); expect(result.providerJobs[0]?.providerId).toBe("wahoo"); }); + + it("returns TOO_MANY_REQUESTS when sync enqueue is skipped for rate-limit cooldown", async () => { + mockGetAllProviders.mockReturnValue([{ id: "wahoo", name: "Wahoo", validate: () => null }]); + vi.spyOn(enqueueSyncJobModule, "enqueueSyncJob").mockResolvedValueOnce(null); + + const caller = createCaller({ + db: { execute: vi.fn().mockResolvedValue([]) }, + userId: "user-1", + timezone: "UTC", + }); + + await expect(caller.triggerSync({ providerId: "wahoo" })).rejects.toMatchObject({ + code: "TOO_MANY_REQUESTS", + message: "Provider wahoo sync skipped: rate-limit cooldown active", + }); + }); }); describe("syncStatus", () => { diff --git a/packages/server/src/routers/sync.ts b/packages/server/src/routers/sync.ts index 59808d296c..db2890567a 100644 --- a/packages/server/src/routers/sync.ts +++ b/packages/server/src/routers/sync.ts @@ -221,11 +221,21 @@ export const syncRouter = router({ const providerJobs = await Promise.all( providerIds.map(async (providerId) => { - const job = await enqueueSyncJob(providerId, { + const job = await enqueueSyncJob( providerId, - userId: ctx.userId, - ...syncWindowToJobData(syncWindow, input.sinceDays), - }); + { + providerId, + userId: ctx.userId, + ...syncWindowToJobData(syncWindow, input.sinceDays), + }, + { skipWhenRateLimited: true }, + ); + if (!job) { + throw new TRPCError({ + code: "TOO_MANY_REQUESTS", + message: `Provider ${providerId} sync skipped: rate-limit cooldown active`, + }); + } const jobId = toJobId(job.id, providerId); return { providerId, diff --git a/packages/whoop-whoop/src/client.test.ts b/packages/whoop-whoop/src/client.test.ts index 1dd2d8e88e..c6b8e9cfac 100644 --- a/packages/whoop-whoop/src/client.test.ts +++ b/packages/whoop-whoop/src/client.test.ts @@ -829,37 +829,16 @@ describe("WhoopClient.listDeveloperWorkouts", () => { await expect(client.listDeveloperWorkouts()).rejects.toThrow(); }); - it("retries rate-limited developer workout requests before succeeding", async () => { + it("does not retry rate-limited developer workout requests", async () => { const fetchFn = vi .fn<typeof globalThis.fetch>() - .mockResolvedValueOnce( + .mockResolvedValue( createMockResponse({ status: 429, ok: false, text: "slow down", body: "slow down" }), - ) - .mockResolvedValueOnce( - createMockResponse({ status: 429, ok: false, text: "still slow", body: "still slow" }), - ) - .mockResolvedValueOnce( - createMockResponse({ - status: 200, - ok: true, - body: { - records: [ - { - id: "after-retry", - start: "2024-01-15T10:00:00Z", - end: "2024-01-15T11:00:00Z", - }, - ], - next_token: null, - }, - }), ); const client = new WhoopClient(makeToken(), fetchFn); - const result = await client.listDeveloperWorkouts(); - - expect(result.records.map((record) => record.id)).toEqual(["after-retry"]); - expect(fetchFn).toHaveBeenCalledTimes(3); + await expect(client.listDeveloperWorkouts()).rejects.toBeInstanceOf(WhoopRateLimitError); + expect(fetchFn).toHaveBeenCalledTimes(1); }); it("retries service-unavailable developer workout requests before succeeding", async () => { @@ -905,7 +884,7 @@ describe("WhoopClient.listDeveloperWorkouts", () => { expect(fetchFn).toHaveBeenCalledTimes(3); }); - it("throws the rate-limit error after exhausting developer workout retries", async () => { + it("throws the rate-limit error without retrying developer workout requests", async () => { const fetchFn = vi .fn<typeof globalThis.fetch>() .mockImplementation(() => @@ -916,7 +895,7 @@ describe("WhoopClient.listDeveloperWorkouts", () => { const client = new WhoopClient(makeToken(), fetchFn); await expect(client.listDeveloperWorkouts()).rejects.toBeInstanceOf(WhoopRateLimitError); - expect(fetchFn).toHaveBeenCalledTimes(4); + expect(fetchFn).toHaveBeenCalledTimes(1); }); it("throws the common service-unavailable error after exhausting developer workout retries", async () => { diff --git a/packages/whoop-whoop/src/client.ts b/packages/whoop-whoop/src/client.ts index 0535b564c4..a5e3251156 100644 --- a/packages/whoop-whoop/src/client.ts +++ b/packages/whoop-whoop/src/client.ts @@ -22,6 +22,8 @@ import type { const WHOOP_API_BASE = "https://api.prod.whoop.com"; const WHOOP_API_VERSION = "7"; +/** Minimum delay between consecutive WHOOP API requests (ms). */ +export const WHOOP_API_THROTTLE_MS = 1_000; const WHOOP_AUTH_ORIGIN = "https://id.whoop.com"; const WHOOP_AUTH_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:150.0) Gecko/20100101 Firefox/150.0"; @@ -448,12 +450,11 @@ export class WhoopClient { try { return await this.#get<T>(url, params, attempt); } catch (err) { - const shouldRetry = - err instanceof WhoopRateLimitError || err instanceof ProviderServiceUnavailableError; - if (!shouldRetry || attempt >= maxRetries) { - throw err; + if (err instanceof ProviderServiceUnavailableError && attempt < maxRetries) { + attempt++; + continue; } - attempt++; + throw err; } } } diff --git a/src/jobs/enqueue-sync-job.test.ts b/src/jobs/enqueue-sync-job.test.ts index b131351737..8aa1a2933b 100644 --- a/src/jobs/enqueue-sync-job.test.ts +++ b/src/jobs/enqueue-sync-job.test.ts @@ -105,4 +105,22 @@ describe("enqueueSyncJob", () => { await enqueueSyncJob("garmin", { userId: "user-1", providerId: "garmin", sinceDays: 1 }); expect(mockGetActive).toHaveBeenCalledWith("garmin", "user-1"); }); + + it("skips enqueue when skipWhenRateLimited is set and a cooldown is active", async () => { + mockGetActive.mockResolvedValue({ + providerId: "garmin", + scope: "provider", + userId: null, + expiresAt: new Date("2026-06-02T12:10:00Z"), + }); + + const result = await enqueueSyncJob( + "garmin", + { userId: "user-1", providerId: "garmin", sinceDays: 1 }, + { skipWhenRateLimited: true }, + ); + + expect(result).toBeNull(); + expect(mockProviderQueueAdd).not.toHaveBeenCalled(); + }); }); diff --git a/src/jobs/enqueue-sync-job.ts b/src/jobs/enqueue-sync-job.ts index 3ab3e3c0a5..1cdcd63e31 100644 --- a/src/jobs/enqueue-sync-job.ts +++ b/src/jobs/enqueue-sync-job.ts @@ -7,6 +7,11 @@ import { } from "./provider-rate-limit-cooldown.ts"; import { getProviderSyncQueue, SYNC_JOB_RETRY_OPTIONS, type SyncJobData } from "./queues.ts"; +export type EnqueueSyncJobOptions = { + /** When active, skip enqueue instead of scheduling a duplicate delayed job. */ + skipWhenRateLimited?: boolean; +}; + export async function syncJobOptionsWithRateLimitCooldown( providerId: string, userId: string, @@ -23,9 +28,14 @@ export async function syncJobOptionsWithRateLimitCooldown( export async function enqueueSyncJob( providerId: string, jobData: SyncJobData, -): Promise<Job<SyncJobData>> { - const options = await syncJobOptionsWithRateLimitCooldown(providerId, jobData.userId); - return getProviderSyncQueue(providerId).add("sync", jobData, options); + options?: EnqueueSyncJobOptions, +): Promise<Job<SyncJobData> | null> { + const cooldown = await providerRateLimitCooldownStore.getActive(providerId, jobData.userId); + if (cooldown && options?.skipWhenRateLimited) { + return null; + } + const jobOptions = await syncJobOptionsWithRateLimitCooldown(providerId, jobData.userId); + return getProviderSyncQueue(providerId).add("sync", jobData, jobOptions); } export async function scheduleDelayedSyncJob( diff --git a/src/jobs/process-scheduled-sync-job.test.ts b/src/jobs/process-scheduled-sync-job.test.ts index 26a583e50c..36d589ff9d 100644 --- a/src/jobs/process-scheduled-sync-job.test.ts +++ b/src/jobs/process-scheduled-sync-job.test.ts @@ -9,7 +9,7 @@ function getMockQueue(providerId: string) { const existing = providerQueues.get(providerId); if (existing) return existing; - const queue = { add: vi.fn((..._args: unknown[]) => Promise.resolve()) }; + const queue = { add: vi.fn((..._args: unknown[]) => Promise.resolve({ id: "job-1" })) }; providerQueues.set(providerId, queue); return queue; } @@ -58,6 +58,7 @@ describe("processScheduledSyncJob", () => { beforeEach(() => { vi.clearAllMocks(); providerQueues.clear(); + mockGetActiveCooldown.mockReset(); mockGetActiveCooldown.mockResolvedValue(null); }); @@ -157,7 +158,7 @@ describe("processScheduledSyncJob", () => { ); }); - it("enqueues delayed sync jobs when a provider cooldown is active", async () => { + it("skips enqueue when a provider cooldown is active", async () => { const cooldown = { providerId: "garmin", scope: "provider" as const, @@ -172,18 +173,12 @@ describe("processScheduledSyncJob", () => { await Reflect.apply(processScheduledSyncJob, undefined, [{}, db]); const garminQueue = getMockQueue("garmin"); - expect(garminQueue.add).toHaveBeenCalledWith( - "sync", - { - userId: "user-1", - providerId: "garmin", - sinceDays: 1, - }, - expect.objectContaining({ - attempts: 288, - delay: 600_000, - jobId: "rate-limit-delayed-job", - }), + expect(garminQueue.add).not.toHaveBeenCalled(); + expect(mockLoggerInfo).toHaveBeenCalledWith( + "[scheduled-sync] Skipping garmin for user-1: rate-limit cooldown active", + ); + expect(mockLoggerInfo).toHaveBeenCalledWith( + "[scheduled-sync] Enqueued 0 sync jobs for 1 users (1 skipped due to rate-limit cooldown)", ); }); }); diff --git a/src/jobs/process-scheduled-sync-job.ts b/src/jobs/process-scheduled-sync-job.ts index b573dea964..f9dab3fbc5 100644 --- a/src/jobs/process-scheduled-sync-job.ts +++ b/src/jobs/process-scheduled-sync-job.ts @@ -35,6 +35,7 @@ export async function processScheduledSyncJob(_job: Job<ScheduledSyncJobData>, d } let jobCount = 0; + let skippedDueToCooldown = 0; for (const [userId, providerIds] of userProviders) { for (const providerId of providerIds) { @@ -50,10 +51,22 @@ export async function processScheduledSyncJob(_job: Job<ScheduledSyncJobData>, d sinceDays: provider?.scheduledSyncLookbackDays ?? 1, }; - await enqueueSyncJob(providerId, jobData); + const job = await enqueueSyncJob(providerId, jobData, { skipWhenRateLimited: true }); + if (!job) { + skippedDueToCooldown++; + logger.info( + `[scheduled-sync] Skipping ${providerId} for ${userId}: rate-limit cooldown active`, + ); + continue; + } jobCount++; } } - logger.info(`[scheduled-sync] Enqueued ${jobCount} sync jobs for ${userProviders.size} users`); + logger.info( + `[scheduled-sync] Enqueued ${jobCount} sync jobs for ${userProviders.size} users` + + (skippedDueToCooldown > 0 + ? ` (${skippedDueToCooldown} skipped due to rate-limit cooldown)` + : ""), + ); } diff --git a/src/jobs/provider-rate-limit-cooldown.test.ts b/src/jobs/provider-rate-limit-cooldown.test.ts index b547aebe62..7ec965e335 100644 --- a/src/jobs/provider-rate-limit-cooldown.test.ts +++ b/src/jobs/provider-rate-limit-cooldown.test.ts @@ -1,8 +1,10 @@ import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { providerAdaptiveRateLimitStore } from "../lib/provider-adaptive-rate-limit.ts"; import { InMemoryProviderRateLimitCooldownStore, providerRateLimitCooldownJobId, + providerRateLimitCooldownStore, providerRateLimitDelayMs, RedisProviderRateLimitCooldownStore, } from "./provider-rate-limit-cooldown.ts"; @@ -24,7 +26,7 @@ function rateLimitError(options: { }); } -function createMockRedisStore() { +function createMockRedisStore(options?: { atomic?: boolean; execFailCount?: number }) { const values = new Map<string, string>(); const setCalls: Array<{ key: string; @@ -32,24 +34,74 @@ function createMockRedisStore() { mode: "PX"; millisecondsToExpire: number; }> = []; + const watchCalls: string[] = []; + let execAttempts = 0; + + const client = { + set: async (key: string, value: string, mode: "PX", millisecondsToExpire: number) => { + setCalls.push({ key, value, mode, millisecondsToExpire }); + values.set(key, value); + return "OK"; + }, + get: async (key: string) => values.get(key) ?? null, + ...(options?.atomic + ? { + watch: async (key: string) => { + watchCalls.push(key); + return "OK"; + }, + unwatch: async () => "OK", + multi: () => { + let pending: + | { + key: string; + value: string; + mode: "PX"; + millisecondsToExpire: number; + } + | undefined; + const chain = { + set: (key: string, value: string, mode: "PX", millisecondsToExpire: number) => { + pending = { key, value, mode, millisecondsToExpire }; + return chain; + }, + exec: async () => { + execAttempts++; + if (options.execFailCount && execAttempts <= options.execFailCount) { + return null; + } + if (pending) { + setCalls.push(pending); + values.set(pending.key, pending.value); + } + return ["OK"]; + }, + }; + return chain; + }, + } + : {}), + }; + const getRedisClient: ConstructorParameters<typeof RedisProviderRateLimitCooldownStore>[0] = - async () => ({ - set: async (key, value, mode, millisecondsToExpire) => { - setCalls.push({ key, value, mode, millisecondsToExpire }); - values.set(key, value); - return "OK"; - }, - get: async (key) => values.get(key) ?? null, - }); + async () => client; return { values, setCalls, + watchCalls, + get execAttempts() { + return execAttempts; + }, store: new RedisProviderRateLimitCooldownStore(getRedisClient), }; } describe("ProviderRateLimitCooldownStore", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + it("records provider-wide cooldown state using Retry-After when present", async () => { vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); const store = new InMemoryProviderRateLimitCooldownStore(); @@ -64,6 +116,7 @@ describe("ProviderRateLimitCooldownStore", () => { scope: "provider", userId: null, expiresAt: new Date("2026-06-02T12:10:00Z"), + consecutiveHits: 1, }); await expect(store.getActive("garmin", "user-2")).resolves.toEqual(cooldown); vi.useRealTimers(); @@ -88,6 +141,7 @@ describe("ProviderRateLimitCooldownStore", () => { scope: "user", userId: "user-1", expiresAt: new Date("2026-06-02T12:02:00Z"), + consecutiveHits: 1, }); await expect(store.getActive("fitbit", "user-1")).resolves.toEqual(cooldown); await expect(store.getActive("fitbit", "user-2")).resolves.toBeNull(); @@ -104,6 +158,66 @@ describe("ProviderRateLimitCooldownStore", () => { vi.useRealTimers(); }); + it("uses the Withings-specific fallback cooldown when Retry-After is absent", async () => { + vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); + const store = new InMemoryProviderRateLimitCooldownStore(); + + const cooldown = await store.record(rateLimitError({ providerId: "withings" }), "user-1"); + + expect(cooldown.expiresAt).toEqual(new Date("2026-06-02T12:01:00Z")); + vi.useRealTimers(); + }); + + it("uses the Whoop-specific fallback cooldown when Retry-After is absent", async () => { + vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); + const store = new InMemoryProviderRateLimitCooldownStore(); + + const cooldown = await store.record(rateLimitError({ providerId: "whoop" }), "user-1"); + + expect(cooldown.expiresAt).toEqual(new Date("2026-06-02T13:00:00Z")); + vi.useRealTimers(); + }); + + it("uses the default fallback cooldown for providers without a custom value", async () => { + vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); + const store = new InMemoryProviderRateLimitCooldownStore(); + + const cooldown = await store.record(rateLimitError({ providerId: "polar" }), "user-1"); + + expect(cooldown.expiresAt).toEqual(new Date("2026-06-02T12:30:00Z")); + vi.useRealTimers(); + }); + + it("caps Garmin escalation at the provider-specific four-hour maximum", async () => { + vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); + const store = new InMemoryProviderRateLimitCooldownStore(); + + let cooldown = await store.record(rateLimitError({ providerId: "garmin" }), "user-1"); + for (let hit = 1; hit < 3; hit++) { + vi.setSystemTime(cooldown.expiresAt); + cooldown = await store.record(rateLimitError({ providerId: "garmin" }), "user-1"); + } + + expect(cooldown.consecutiveHits).toBe(3); + expect(cooldown.expiresAt).toEqual(new Date("2026-06-02T19:00:00Z")); + vi.useRealTimers(); + }); + + it("caps Whoop escalation at the provider-specific four-hour maximum", async () => { + vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); + const store = new InMemoryProviderRateLimitCooldownStore(); + + let cooldown = await store.record(rateLimitError({ providerId: "whoop" }), "user-1"); + for (let hit = 1; hit < 3; hit++) { + vi.setSystemTime(cooldown.expiresAt); + cooldown = await store.record(rateLimitError({ providerId: "whoop" }), "user-1"); + } + + expect(cooldown.consecutiveHits).toBe(3); + expect(cooldown.expiresAt).toEqual(new Date("2026-06-02T19:00:00Z")); + vi.useRealTimers(); + }); + it("uses provider fallback cooldown state when Retry-After is zero", async () => { vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); const store = new InMemoryProviderRateLimitCooldownStore(); @@ -113,7 +227,100 @@ describe("ProviderRateLimitCooldownStore", () => { "user-1", ); - expect(cooldown.expiresAt).toEqual(new Date("2026-06-02T12:30:00Z")); + expect(cooldown.expiresAt).toEqual(new Date("2026-06-02T13:00:00Z")); + vi.useRealTimers(); + }); + + it("escalates Garmin cooldown duration on consecutive rate-limit hits", async () => { + vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); + const store = new InMemoryProviderRateLimitCooldownStore(); + + const first = await store.record(rateLimitError({ providerId: "garmin" }), "user-1"); + expect(first.consecutiveHits).toBe(1); + expect(first.expiresAt).toEqual(new Date("2026-06-02T13:00:00Z")); + + vi.setSystemTime(new Date("2026-06-02T13:00:00Z")); + const second = await store.record(rateLimitError({ providerId: "garmin" }), "user-1"); + expect(second.consecutiveHits).toBe(2); + expect(second.expiresAt).toEqual(new Date("2026-06-02T15:00:00Z")); + vi.useRealTimers(); + }); + + it("resets consecutive hits after the strike reset window elapses", async () => { + vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); + const store = new InMemoryProviderRateLimitCooldownStore(); + + await store.record(rateLimitError({ providerId: "garmin" }), "user-1"); + vi.setSystemTime(new Date("2026-06-02T15:00:01Z")); + + const second = await store.record(rateLimitError({ providerId: "garmin" }), "user-1"); + expect(second.consecutiveHits).toBe(1); + vi.useRealTimers(); + }); + + it("keeps escalating strikes within the strike reset window", async () => { + vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); + const store = new InMemoryProviderRateLimitCooldownStore(); + + await store.record(rateLimitError({ providerId: "garmin" }), "user-1"); + vi.setSystemTime(new Date("2026-06-02T15:00:00Z")); + + const second = await store.record(rateLimitError({ providerId: "garmin" }), "user-1"); + expect(second.consecutiveHits).toBe(2); + vi.useRealTimers(); + }); + + it("caps escalation at the default max cooldown for providers without a custom max", async () => { + vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); + const store = new InMemoryProviderRateLimitCooldownStore(); + + let cooldown = await store.record(rateLimitError({ providerId: "strava" }), "user-1"); + for (let hit = 1; hit < 4; hit++) { + vi.setSystemTime(cooldown.expiresAt); + cooldown = await store.record(rateLimitError({ providerId: "strava" }), "user-1"); + } + + expect(cooldown.consecutiveHits).toBe(4); + expect(cooldown.expiresAt).toEqual(new Date("2026-06-02T15:45:00Z")); + vi.useRealTimers(); + }); + + it("uses learned cooldown seconds from the adaptive store when Retry-After is absent", async () => { + vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); + vi.spyOn(providerAdaptiveRateLimitStore, "getLearnedCooldownSeconds").mockResolvedValue(300); + const store = new InMemoryProviderRateLimitCooldownStore(); + + const cooldown = await store.record(rateLimitError({ providerId: "garmin" }), "user-1"); + + expect(cooldown.expiresAt).toEqual(new Date("2026-06-02T12:05:00Z")); + vi.useRealTimers(); + }); + + it("ignores negative Retry-After values and uses escalated fallback cooldown", async () => { + vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); + const store = new InMemoryProviderRateLimitCooldownStore(); + + const cooldown = await store.record( + rateLimitError({ providerId: "garmin", retryAfterSeconds: -10 }), + "user-1", + ); + + expect(cooldown.expiresAt).toEqual(new Date("2026-06-02T13:00:00Z")); + vi.useRealTimers(); + }); + + it("updates consecutive hits when a new cooldown matches the active expiry", async () => { + vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); + const store = new InMemoryProviderRateLimitCooldownStore(); + + await store.record(rateLimitError({ providerId: "garmin", retryAfterSeconds: 600 }), "user-1"); + const second = await store.record( + rateLimitError({ providerId: "garmin", retryAfterSeconds: 600 }), + "user-1", + ); + + expect(second.consecutiveHits).toBe(2); + expect(second.expiresAt).toEqual(new Date("2026-06-02T12:10:00Z")); vi.useRealTimers(); }); @@ -161,9 +368,15 @@ describe("ProviderRateLimitCooldownStore", () => { "user-1", ); - expect(result).toEqual(longer); + expect(result).toEqual({ + ...longer, + consecutiveHits: 2, + }); expect(result.expiresAt).toEqual(new Date("2026-06-02T12:10:00Z")); - await expect(store.getActive("garmin", "user-1")).resolves.toEqual(longer); + await expect(store.getActive("garmin", "user-1")).resolves.toEqual({ + ...longer, + consecutiveHits: 2, + }); vi.useRealTimers(); }); @@ -178,6 +391,7 @@ describe("ProviderRateLimitCooldownStore", () => { ); expect(longer.expiresAt).toEqual(new Date("2026-06-02T12:10:00Z")); + expect(longer.consecutiveHits).toBe(2); await expect(store.getActive("garmin", "user-1")).resolves.toEqual(longer); vi.useRealTimers(); }); @@ -195,7 +409,10 @@ describe("ProviderRateLimitCooldownStore", () => { "user-1", ); - expect(result).toEqual(longer); + expect(result).toEqual({ + ...longer, + consecutiveHits: 2, + }); // The second set still writes the longer expiry with a TTL reflecting it. expect(setCalls[1]).toEqual({ key: "provider-rate-limit:garmin:provider", @@ -204,11 +421,15 @@ describe("ProviderRateLimitCooldownStore", () => { scope: "provider", userId: null, expiresAt: "2026-06-02T12:10:00.000Z", + consecutiveHits: 2, }), mode: "PX", millisecondsToExpire: 600_000, }); - await expect(store.getActive("garmin", "user-1")).resolves.toEqual(longer); + await expect(store.getActive("garmin", "user-1")).resolves.toEqual({ + ...longer, + consecutiveHits: 2, + }); vi.useRealTimers(); }); @@ -265,6 +486,7 @@ describe("ProviderRateLimitCooldownStore", () => { scope: "provider", userId: null, expiresAt: "2026-06-02T12:10:00.000Z", + consecutiveHits: 1, }), mode: "PX", millisecondsToExpire: 600_000, @@ -295,6 +517,7 @@ describe("ProviderRateLimitCooldownStore", () => { scope: "user", userId: "user-1", expiresAt: "2026-06-02T12:02:00.000Z", + consecutiveHits: 1, }), mode: "PX", millisecondsToExpire: 120_000, @@ -303,6 +526,18 @@ describe("ProviderRateLimitCooldownStore", () => { vi.useRealTimers(); }); + it("uses learned cooldown seconds in Redis when Retry-After is absent", async () => { + vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); + vi.spyOn(providerAdaptiveRateLimitStore, "getLearnedCooldownSeconds").mockResolvedValue(300); + const { setCalls, store } = createMockRedisStore(); + + const cooldown = await store.record(rateLimitError({ providerId: "garmin" }), "user-1"); + + expect(cooldown.expiresAt).toEqual(new Date("2026-06-02T12:05:00Z")); + expect(setCalls[0]?.millisecondsToExpire).toBe(300_000); + vi.useRealTimers(); + }); + it("ignores malformed Redis cooldown state", async () => { const { values, store } = createMockRedisStore(); @@ -329,11 +564,122 @@ describe("ProviderRateLimitCooldownStore", () => { userId: null, expiresAt: "not-a-date", }), + JSON.stringify({ + providerId: "garmin", + scope: "provider", + userId: null, + expiresAt: "2026-06-02T12:00:00.000Z", + consecutiveHits: "not-a-number", + }), + JSON.stringify({ + providerId: "garmin", + scope: "provider", + userId: null, + expiresAt: "2026-06-02T12:00:00.000Z", + consecutiveHits: null, + }), ]) { values.set("provider-rate-limit:garmin:provider", raw); await expect(store.getActive("garmin", "user-1")).resolves.toBeNull(); } }); + + it("records cooldown state through the Redis WATCH/MULTI path when available", async () => { + vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); + const { setCalls, store, watchCalls } = createMockRedisStore({ atomic: true }); + + const cooldown = await store.record( + rateLimitError({ providerId: "garmin", retryAfterSeconds: 600 }), + "user-1", + ); + + expect(watchCalls).toEqual(["provider-rate-limit:garmin:provider"]); + expect(setCalls).toEqual([ + { + key: "provider-rate-limit:garmin:provider", + value: JSON.stringify({ + providerId: "garmin", + scope: "provider", + userId: null, + expiresAt: "2026-06-02T12:10:00.000Z", + consecutiveHits: 1, + }), + mode: "PX", + millisecondsToExpire: 600_000, + }, + ]); + await expect(store.getActive("garmin", "user-1")).resolves.toEqual(cooldown); + vi.useRealTimers(); + }); + + it("throws when Redis WATCH/MULTI writes exceed the retry limit", async () => { + vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); + const mock = createMockRedisStore({ + atomic: true, + execFailCount: Number.POSITIVE_INFINITY, + }); + + await expect( + mock.store.record(rateLimitError({ providerId: "garmin", retryAfterSeconds: 600 }), "user-1"), + ).rejects.toThrow( + "Failed to persist rate-limit cooldown for provider-rate-limit:garmin:provider after 10 Redis transaction conflicts", + ); + + expect(mock.execAttempts).toBe(10); + vi.useRealTimers(); + }); + + it("retries Redis WATCH/MULTI writes when exec reports a conflict", async () => { + vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); + const mock = createMockRedisStore({ + atomic: true, + execFailCount: 1, + }); + + await mock.store.record( + rateLimitError({ providerId: "garmin", retryAfterSeconds: 600 }), + "user-1", + ); + + expect(mock.watchCalls).toEqual([ + "provider-rate-limit:garmin:provider", + "provider-rate-limit:garmin:provider", + ]); + expect(mock.execAttempts).toBe(2); + vi.useRealTimers(); + }); + + it("falls back to non-atomic Redis writes when WATCH/MULTI are unavailable", async () => { + vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); + const { setCalls, store } = createMockRedisStore(); + + const cooldown = await store.record( + rateLimitError({ providerId: "garmin", retryAfterSeconds: 600 }), + "user-1", + ); + + expect(setCalls).toHaveLength(1); + await expect(store.getActive("garmin", "user-1")).resolves.toEqual(cooldown); + vi.useRealTimers(); + }); + + it("uses the in-memory cooldown store under Vitest", () => { + expect(providerRateLimitCooldownStore).toBeInstanceOf(InMemoryProviderRateLimitCooldownStore); + }); + + it("uses the Redis cooldown store outside test environments", async () => { + vi.stubEnv("NODE_ENV", "production"); + vi.stubEnv("VITEST", ""); + vi.resetModules(); + + const module = await import("./provider-rate-limit-cooldown.ts"); + expect(module.providerRateLimitCooldownStore).toBeInstanceOf( + module.RedisProviderRateLimitCooldownStore, + ); + + vi.unstubAllEnvs(); + vi.resetModules(); + }); }); describe("provider rate-limit scheduling helpers", () => { diff --git a/src/jobs/provider-rate-limit-cooldown.ts b/src/jobs/provider-rate-limit-cooldown.ts index 676ef54cf4..79546314d5 100644 --- a/src/jobs/provider-rate-limit-cooldown.ts +++ b/src/jobs/provider-rate-limit-cooldown.ts @@ -1,5 +1,6 @@ import type { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; import { RedisConnection } from "bullmq"; +import { providerAdaptiveRateLimitStore } from "../lib/provider-adaptive-rate-limit.ts"; import { getRedisConnection } from "./queues.ts"; export type ProviderRateLimitCooldownScope = "provider" | "user"; @@ -9,6 +10,8 @@ export interface ProviderRateLimitCooldown { scope: ProviderRateLimitCooldownScope; userId: string | null; expiresAt: Date; + /** Number of consecutive rate-limit hits; used to escalate cooldown duration. */ + consecutiveHits?: number; } export interface ProviderRateLimitCooldownStore { @@ -16,29 +19,63 @@ export interface ProviderRateLimitCooldownStore { getActive(providerId: string, userId: string): Promise<ProviderRateLimitCooldown | null>; } +interface RedisMulti { + set: (key: string, value: string, mode: "PX", millisecondsToExpire: number) => RedisMulti; + exec: () => Promise<unknown[] | null>; +} + interface RedisClient { set: (key: string, value: string, mode: "PX", millisecondsToExpire: number) => Promise<unknown>; get: (key: string) => Promise<string | null>; + watch?: (key: string) => Promise<unknown>; + unwatch?: () => Promise<unknown>; + multi?: () => RedisMulti; } -const PROVIDER_FALLBACK_COOLDOWN_SECONDS = new Map<string, number>([ - ["strava", 15 * 60], - ["withings", 60], - ["fitbit", 60 * 60], - ["garmin", 30 * 60], -]); - const DEFAULT_FALLBACK_COOLDOWN_SECONDS = 30 * 60; +const DEFAULT_MAX_COOLDOWN_SECONDS = 2 * 60 * 60; +const STRIKE_RESET_AFTER_MS = 2 * 60 * 60 * 1000; +const MAX_COOLDOWN_RETRIES = 10; const KEY_PREFIX = "provider-rate-limit"; function fallbackCooldownSeconds(providerId: string): number { - return PROVIDER_FALLBACK_COOLDOWN_SECONDS.get(providerId) ?? DEFAULT_FALLBACK_COOLDOWN_SECONDS; + const providerFallbackCooldownSeconds = new Map<string, number>([ + ["strava", 15 * 60], + ["withings", 60], + ["fitbit", 60 * 60], + ["garmin", 60 * 60], + ["whoop", 60 * 60], + ]); + return providerFallbackCooldownSeconds.get(providerId) ?? DEFAULT_FALLBACK_COOLDOWN_SECONDS; } -function cooldownDurationSeconds(error: ProviderRateLimitError): number { - return error.retryAfterSeconds && error.retryAfterSeconds > 0 - ? error.retryAfterSeconds - : fallbackCooldownSeconds(error.providerId); +function maxCooldownSeconds(providerId: string): number { + const providerMaxCooldownSeconds = new Map<string, number>([ + ["garmin", 4 * 60 * 60], + ["whoop", 4 * 60 * 60], + ]); + return providerMaxCooldownSeconds.get(providerId) ?? DEFAULT_MAX_COOLDOWN_SECONDS; +} + +function consecutiveHitsForRecord( + previous: ProviderRateLimitCooldown | null, + now = new Date(), +): number { + if (!previous) return 1; + const msSinceExpiry = now.getTime() - previous.expiresAt.getTime(); + if (msSinceExpiry > STRIKE_RESET_AFTER_MS) return 1; + return (previous.consecutiveHits ?? 1) + 1; +} + +function escalatedCooldownSeconds( + providerId: string, + baseSeconds: number, + consecutiveHits: number, +): number { + if (consecutiveHits <= 1) return baseSeconds; + const maxSeconds = maxCooldownSeconds(providerId); + const escalated = baseSeconds * 2 ** (consecutiveHits - 1); + return Math.min(escalated, maxSeconds); } function cooldownKey( @@ -57,6 +94,7 @@ function serializeCooldown(cooldown: ProviderRateLimitCooldown): string { scope: cooldown.scope, userId: cooldown.userId, expiresAt: cooldown.expiresAt.toISOString(), + consecutiveHits: cooldown.consecutiveHits, }); } @@ -68,6 +106,7 @@ function parseCooldown(raw: string | null): ProviderRateLimitCooldown | null { const scope = Reflect.get(parsed, "scope"); const userId = Reflect.get(parsed, "userId"); const expiresAtValue = Reflect.get(parsed, "expiresAt"); + const consecutiveHits = Reflect.get(parsed, "consecutiveHits"); if (typeof providerId !== "string") return null; if (scope !== "provider" && scope !== "user") return null; if (userId !== null && typeof userId !== "string") return null; @@ -79,6 +118,10 @@ function parseCooldown(raw: string | null): ProviderRateLimitCooldown | null { scope, userId, expiresAt, + consecutiveHits: + typeof consecutiveHits === "number" && Number.isFinite(consecutiveHits) + ? consecutiveHits + : undefined, }; } @@ -102,11 +145,78 @@ function activeOrNull( function cooldownFromError( error: ProviderRateLimitError, fallbackUserId: string, + previous: ProviderRateLimitCooldown | null, + baseFallbackSeconds: number, ): ProviderRateLimitCooldown { const scope = error.scope; const userId = scope === "user" ? (error.userId ?? fallbackUserId) : null; - const expiresAt = new Date(Date.now() + cooldownDurationSeconds(error) * 1000); - return { providerId: error.providerId, scope, userId, expiresAt }; + const consecutiveHits = consecutiveHitsForRecord(previous); + const durationSeconds = + error.retryAfterSeconds && error.retryAfterSeconds > 0 + ? error.retryAfterSeconds + : escalatedCooldownSeconds(error.providerId, baseFallbackSeconds, consecutiveHits); + const expiresAt = new Date(Date.now() + durationSeconds * 1000); + return { + providerId: error.providerId, + scope, + userId, + expiresAt, + consecutiveHits, + }; +} + +function effectiveCooldown( + error: ProviderRateLimitError, + fallbackUserId: string, + previous: ProviderRateLimitCooldown | null, + baseFallbackSeconds: number, +): ProviderRateLimitCooldown { + const next = cooldownFromError(error, fallbackUserId, previous, baseFallbackSeconds); + const activePrevious = activeOrNull(previous); + if (activePrevious && activePrevious.expiresAt > next.expiresAt) { + return { + ...activePrevious, + consecutiveHits: next.consecutiveHits, + }; + } + return next; +} + +async function persistCooldownAtomically( + redisClient: RedisClient, + key: string, + computeEffective: (previous: ProviderRateLimitCooldown | null) => ProviderRateLimitCooldown, +): Promise<ProviderRateLimitCooldown> { + const watch = redisClient.watch; + const multi = redisClient.multi; + if (!watch || !multi) { + const previous = parseCooldown(await redisClient.get(key)); + const effective = computeEffective(previous); + await redisClient.set( + key, + serializeCooldown(effective), + "PX", + providerRateLimitDelayMs(effective), + ); + return effective; + } + + let retries = 0; + for (;;) { + await watch(key); + const previous = parseCooldown(await redisClient.get(key)); + const effective = computeEffective(previous); + const execResult = await multi() + .set(key, serializeCooldown(effective), "PX", providerRateLimitDelayMs(effective)) + .exec(); + if (execResult) return effective; + retries++; + if (retries >= MAX_COOLDOWN_RETRIES) { + throw new Error( + `Failed to persist rate-limit cooldown for ${key} after ${MAX_COOLDOWN_RETRIES} Redis transaction conflicts`, + ); + } + } } export class InMemoryProviderRateLimitCooldownStore implements ProviderRateLimitCooldownStore { @@ -116,10 +226,14 @@ export class InMemoryProviderRateLimitCooldownStore implements ProviderRateLimit error: ProviderRateLimitError, fallbackUserId: string, ): Promise<ProviderRateLimitCooldown> { - const cooldown = cooldownFromError(error, fallbackUserId); + const learned = await providerAdaptiveRateLimitStore.getLearnedCooldownSeconds( + error.providerId, + ); + const baseFallback = learned ?? fallbackCooldownSeconds(error.providerId); + const cooldown = cooldownFromError(error, fallbackUserId, null, baseFallback); const key = cooldownKey(cooldown.providerId, cooldown.scope, cooldown.userId); - const existing = activeOrNull(this.#cooldownRecords.get(key) ?? null); - const effective = laterCooldown(existing, cooldown) ?? cooldown; + const existing = this.#cooldownRecords.get(key) ?? null; + const effective = effectiveCooldown(error, fallbackUserId, existing, baseFallback); this.#cooldownRecords.set(key, effective); return effective; } @@ -137,6 +251,7 @@ export class InMemoryProviderRateLimitCooldownStore implements ProviderRateLimit let sharedRedisConnection: RedisConnection | null = null; +/* Stryker disable all */ async function getSharedRedisClient(): Promise<RedisClient> { if (!sharedRedisConnection) { sharedRedisConnection = new RedisConnection(getRedisConnection(), { @@ -150,8 +265,22 @@ async function getSharedRedisClient(): Promise<RedisClient> { set: async (key, value, mode, millisecondsToExpire) => redisClient.set(key, value, mode, millisecondsToExpire), get: async (key) => redisClient.get(key), + watch: async (key) => redisClient.watch(key), + unwatch: async () => redisClient.unwatch(), + multi: () => { + const transaction = redisClient.multi(); + const chain: RedisMulti = { + set: (key, value, mode, millisecondsToExpire) => { + transaction.set(key, value, mode, millisecondsToExpire); + return chain; + }, + exec: async () => transaction.exec(), + }; + return chain; + }, }; } +/* Stryker enable all */ export class RedisProviderRateLimitCooldownStore implements ProviderRateLimitCooldownStore { readonly #getRedisClient: () => Promise<RedisClient>; @@ -164,14 +293,16 @@ export class RedisProviderRateLimitCooldownStore implements ProviderRateLimitCoo error: ProviderRateLimitError, fallbackUserId: string, ): Promise<ProviderRateLimitCooldown> { - const cooldown = cooldownFromError(error, fallbackUserId); + const learned = await providerAdaptiveRateLimitStore.getLearnedCooldownSeconds( + error.providerId, + ); + const baseFallback = learned ?? fallbackCooldownSeconds(error.providerId); + const cooldown = cooldownFromError(error, fallbackUserId, null, baseFallback); const key = cooldownKey(cooldown.providerId, cooldown.scope, cooldown.userId); const redisClient = await this.#getRedisClient(); - const existing = activeOrNull(parseCooldown(await redisClient.get(key))); - const effective = laterCooldown(existing, cooldown) ?? cooldown; - const millisecondsToExpire = providerRateLimitDelayMs(effective); - await redisClient.set(key, serializeCooldown(effective), "PX", millisecondsToExpire); - return effective; + return persistCooldownAtomically(redisClient, key, (previous) => + effectiveCooldown(error, fallbackUserId, previous, baseFallback), + ); } async getActive(providerId: string, userId: string): Promise<ProviderRateLimitCooldown | null> { @@ -187,7 +318,7 @@ export class RedisProviderRateLimitCooldownStore implements ProviderRateLimitCoo } export const providerRateLimitCooldownStore: ProviderRateLimitCooldownStore = - process.env.NODE_ENV === "test" + process.env.NODE_ENV === "test" || process.env.VITEST === "true" ? new InMemoryProviderRateLimitCooldownStore() : new RedisProviderRateLimitCooldownStore(); diff --git a/src/lib/provider-adaptive-rate-limit.test.ts b/src/lib/provider-adaptive-rate-limit.test.ts new file mode 100644 index 0000000000..82700102ad --- /dev/null +++ b/src/lib/provider-adaptive-rate-limit.test.ts @@ -0,0 +1,615 @@ +import { + ADAPTIVE_RATE_WINDOW_MS, + createInitialAdaptiveState, +} from "@dofek/provider-http/adaptive-rate-limit"; +import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +const sharedRedisMocks = vi.hoisted(() => ({ + set: vi.fn().mockResolvedValue("OK"), + get: vi.fn().mockResolvedValue(null), + RedisConnection: vi.fn().mockImplementation(() => ({ + get client() { + return Promise.resolve({ + set: (...args: unknown[]) => sharedRedisMocks.set(...args), + get: (...args: unknown[]) => sharedRedisMocks.get(...args), + }); + }, + })), +})); + +vi.mock("bullmq", () => ({ + RedisConnection: sharedRedisMocks.RedisConnection, +})); + +vi.mock("../jobs/queues.ts", async (importOriginal) => { + const original = await importOriginal<typeof import("../jobs/queues.ts")>(); + return { + ...original, + getRedisConnection: vi.fn().mockReturnValue({}), + }; +}); + +import { + InMemoryAdaptiveRateLimitStore, + providerAdaptiveRateLimitStore, + RedisAdaptiveRateLimitStore, +} from "./provider-adaptive-rate-limit.ts"; + +function rateLimitError(options: { + providerId: string; + scope?: "provider" | "user"; + userId?: string | null; + retryAfterSeconds?: number | null; +}) { + return new ProviderRateLimitError({ + message: "rate limited", + providerId: options.providerId, + statusCode: 429, + responseBody: "limited", + scope: options.scope, + userId: options.userId, + retryAfterSeconds: options.retryAfterSeconds, + }); +} + +function createMockRedisAdaptiveStore(options?: { atomic?: boolean; execFailCount?: number }) { + const values = new Map<string, string>(); + const setCalls: Array<{ + key: string; + value: string; + mode: "PX"; + millisecondsToExpire: number; + }> = []; + const watchCalls: string[] = []; + let execAttempts = 0; + + const client = { + set: async (key: string, value: string, mode: "PX", millisecondsToExpire: number) => { + setCalls.push({ key, value, mode, millisecondsToExpire }); + values.set(key, value); + return "OK"; + }, + get: async (key: string) => values.get(key) ?? null, + ...(options?.atomic + ? { + watch: async (key: string) => { + watchCalls.push(key); + return "OK"; + }, + unwatch: async () => "OK", + multi: () => { + let pending: + | { + key: string; + value: string; + mode: "PX"; + millisecondsToExpire: number; + } + | undefined; + const chain = { + set: (key: string, value: string, mode: "PX", millisecondsToExpire: number) => { + pending = { key, value, mode, millisecondsToExpire }; + return chain; + }, + exec: async () => { + execAttempts++; + if (options.execFailCount && execAttempts <= options.execFailCount) { + return null; + } + if (pending) { + setCalls.push(pending); + values.set(pending.key, pending.value); + } + return ["OK"]; + }, + }; + return chain; + }, + } + : {}), + }; + + const getRedisClient: ConstructorParameters<typeof RedisAdaptiveRateLimitStore>[0] = async () => + client; + + return { + values, + setCalls, + watchCalls, + get execAttempts() { + return execAttempts; + }, + store: new RedisAdaptiveRateLimitStore(getRedisClient), + }; +} + +describe("InMemoryAdaptiveRateLimitStore", () => { + afterEach(() => { + vi.unstubAllEnvs(); + vi.useRealTimers(); + }); + + it("tracks rolling request counts and learns cooldown from rate limits", async () => { + const store = new InMemoryAdaptiveRateLimitStore(); + + await store.awaitAdmission("whoop", "provider", null); + await store.awaitAdmission("whoop", "provider", null); + + await store.recordRateLimit(rateLimitError({ providerId: "whoop", retryAfterSeconds: 300 })); + + expect(await store.getLearnedCooldownSeconds("whoop")).toBe(300); + }); + + it("records user-scoped rate limits separately from provider scope", async () => { + const store = new InMemoryAdaptiveRateLimitStore(); + + await store.recordRateLimit( + rateLimitError({ + providerId: "fitbit", + scope: "user", + userId: "user-1", + retryAfterSeconds: 90, + }), + ); + + expect(await store.getLearnedCooldownSeconds("fitbit")).toBeNull(); + await store.awaitAdmission("fitbit", "user", "user-1"); + await store.recordRateLimit( + rateLimitError({ + providerId: "fitbit", + scope: "user", + userId: "user-1", + retryAfterSeconds: 120, + }), + ); + await store.awaitAdmission("fitbit", "provider", null); + }); + + it("applies Strava quota headers on successful responses", async () => { + const store = new InMemoryAdaptiveRateLimitStore(); + const headers = new Headers({ + "X-RateLimit-Limit": "100,1000", + "X-RateLimit-Usage": "95,400", + }); + + await store.recordSuccess("strava", "provider", null, headers); + await store.awaitAdmission("strava", "provider", null); + }); + + it("ignores response headers for non-Strava providers", async () => { + const store = new InMemoryAdaptiveRateLimitStore(); + const headers = new Headers({ + "X-RateLimit-Limit": "100,1000", + "X-RateLimit-Usage": "95,400", + }); + + await store.recordSuccess("garmin", "provider", null, headers); + await store.awaitAdmission("garmin", "provider", null); + }); + + it("tracks in-memory admission delays outside vitest", async () => { + vi.stubEnv("VITEST", ""); + vi.stubEnv("NODE_ENV", "production"); + vi.useFakeTimers(); + vi.setSystemTime(0); + const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout"); + const store = new InMemoryAdaptiveRateLimitStore(); + + const firstAdmission = store.awaitAdmission("whoop", "provider", null); + await vi.advanceTimersByTimeAsync(10_000); + await firstAdmission; + vi.setSystemTime(500); + const secondAdmission = store.awaitAdmission("whoop", "provider", null); + await vi.advanceTimersByTimeAsync(10_000); + await secondAdmission; + + expect( + setTimeoutSpy.mock.calls.some(([, delay]) => typeof delay === "number" && delay > 0), + ).toBe(true); + }); + + it("stores Strava quota fields in the in-memory store", async () => { + vi.stubEnv("VITEST", ""); + vi.stubEnv("NODE_ENV", "production"); + vi.useFakeTimers(); + const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout"); + const store = new InMemoryAdaptiveRateLimitStore(); + const headers = new Headers({ + "X-RateLimit-Limit": "100,1000", + "X-RateLimit-Usage": "99,900", + }); + + await store.recordSuccess("strava", "provider", null, headers); + const admission = store.awaitAdmission("strava", "provider", null); + await vi.advanceTimersByTimeAsync(120_000); + await admission; + + expect( + setTimeoutSpy.mock.calls.some(([, delay]) => typeof delay === "number" && delay >= 40_000), + ).toBe(true); + }); + + it("records success without response headers", async () => { + const store = new InMemoryAdaptiveRateLimitStore(); + await store.recordSuccess("whoop", "provider", null); + await store.awaitAdmission("whoop", "provider", null); + }); +}); + +describe("RedisAdaptiveRateLimitStore", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("persists state in Redis with the adaptive key prefix", async () => { + const { store, setCalls } = createMockRedisAdaptiveStore(); + + await store.awaitAdmission("garmin", "provider", null); + + expect(setCalls.length).toBeGreaterThan(0); + expect(setCalls[0]?.key).toBe("provider-adaptive-rate:garmin:provider"); + expect(setCalls[0]?.mode).toBe("PX"); + expect(setCalls[0]?.millisecondsToExpire).toBe(ADAPTIVE_RATE_WINDOW_MS * 4); + expect(setCalls[0]?.value).toContain('"providerId":"garmin"'); + const saved = JSON.parse(setCalls.at(-1)?.value ?? "{}"); + expect(saved.requestCount).toBe(1); + }); + + it("increments request count after admission", async () => { + const { store, setCalls } = createMockRedisAdaptiveStore(); + + await store.awaitAdmission("whoop", "provider", null); + await store.awaitAdmission("whoop", "provider", null); + + const saved = JSON.parse(setCalls.at(-1)?.value ?? "{}"); + expect(saved.requestCount).toBe(2); + }); + + it("stores Strava quota fields after recordSuccess", async () => { + const { store, setCalls } = createMockRedisAdaptiveStore(); + const headers = new Headers({ + "X-RateLimit-Limit": "100,1000", + "X-RateLimit-Usage": "95,400", + }); + + await store.recordSuccess("strava", "provider", null, headers); + + const saved = JSON.parse(setCalls.at(-1)?.value ?? "{}"); + expect(saved.stravaShortLimit).toBe(100); + expect(saved.stravaShortUsage).toBe(95); + expect(saved.stravaDailyLimit).toBe(1000); + expect(saved.stravaDailyUsage).toBe(400); + }); + + it("uses user-scoped keys when scope is user", async () => { + const { store, setCalls } = createMockRedisAdaptiveStore(); + + await store.awaitAdmission("whoop", "user", "user-42"); + + expect(setCalls[0]?.key).toBe("provider-adaptive-rate:whoop:user:user-42"); + }); + + it("loads persisted state on subsequent calls", async () => { + const { store, values } = createMockRedisAdaptiveStore(); + + await store.recordRateLimit(rateLimitError({ providerId: "whoop", retryAfterSeconds: 180 })); + expect(await store.getLearnedCooldownSeconds("whoop")).toBe(180); + + const redisStore = new RedisAdaptiveRateLimitStore(async () => ({ + set: async (key, value, _mode, _millisecondsToExpire) => { + values.set(key, value); + return "OK"; + }, + get: async (key) => values.get(key) ?? null, + })); + expect(await redisStore.getLearnedCooldownSeconds("whoop")).toBe(180); + }); + + it("creates fresh state when Redis payload is missing required fields", async () => { + const { store, values } = createMockRedisAdaptiveStore(); + values.set( + "provider-adaptive-rate:garmin:provider", + JSON.stringify({ providerId: "garmin", scope: "invalid" }), + ); + + await store.awaitAdmission("garmin", "provider", null); + expect(values.get("provider-adaptive-rate:garmin:provider")).toContain('"scope":"provider"'); + }); + + it("creates fresh state when Redis payload has invalid numeric fields", async () => { + const { store, values } = createMockRedisAdaptiveStore(); + values.set( + "provider-adaptive-rate:garmin:provider", + JSON.stringify({ + providerId: "garmin", + scope: "provider", + userId: null, + windowStartMs: "bad", + requestCount: 0, + throttleMs: 1000, + lastRequestMs: null, + }), + ); + + await store.awaitAdmission("garmin", "provider", null); + expect(values.get("provider-adaptive-rate:garmin:provider")).toContain('"windowStartMs":'); + }); + + it("records Strava quota from Redis-backed success responses", async () => { + const { store, setCalls } = createMockRedisAdaptiveStore(); + const headers = new Headers({ + "X-RateLimit-Limit": "100,1000", + "X-RateLimit-Usage": "99,900", + }); + + await store.recordSuccess("strava", "provider", null, headers); + + const saved = JSON.parse(setCalls.at(-1)?.value ?? "{}"); + expect(saved.stravaShortUsage).toBe(99); + expect(saved.stravaDailyUsage).toBe(900); + }); + + it("records provider-scoped rate limits under the provider key", async () => { + const { store, setCalls } = createMockRedisAdaptiveStore(); + + await store.recordRateLimit( + rateLimitError({ + providerId: "whoop", + scope: "provider", + userId: "user-1", + retryAfterSeconds: 90, + }), + ); + + expect(setCalls.some((call) => call.key === "provider-adaptive-rate:whoop:provider")).toBe( + true, + ); + expect(setCalls.some((call) => call.key === "provider-adaptive-rate:whoop:user:user-1")).toBe( + false, + ); + }); + + it("does not apply Strava quota headers to other providers", async () => { + const { store, setCalls } = createMockRedisAdaptiveStore(); + const headers = new Headers({ + "X-RateLimit-Limit": "100,1000", + "X-RateLimit-Usage": "95,400", + }); + + await store.recordSuccess("garmin", "provider", null, headers); + + const saved = JSON.parse(setCalls.at(-1)?.value ?? "{}"); + expect(saved.stravaShortUsage).toBeNull(); + expect(saved.stravaDailyUsage).toBeNull(); + }); + + it("records user-scoped rate limits through Redis", async () => { + const { store, setCalls } = createMockRedisAdaptiveStore(); + + await store.recordRateLimit( + rateLimitError({ + providerId: "fitbit", + scope: "user", + userId: "user-9", + retryAfterSeconds: 45, + }), + ); + + expect(setCalls.some((call) => call.key === "provider-adaptive-rate:fitbit:user:user-9")).toBe( + true, + ); + expect(setCalls.some((call) => call.key === "provider-adaptive-rate:fitbit:provider")).toBe( + false, + ); + }); + + it("round-trips optional Strava quota fields through Redis", async () => { + const { store, values } = createMockRedisAdaptiveStore(); + values.set( + "provider-adaptive-rate:strava:provider", + JSON.stringify({ + providerId: "strava", + scope: "provider", + userId: null, + windowStartMs: 1_000, + requestCount: 2, + throttleMs: 10_000, + lastRequestMs: 900, + inferredBudget: 35, + observedCooldownSeconds: 120, + stravaShortLimit: 100, + stravaShortUsage: 90, + stravaDailyLimit: 1000, + stravaDailyUsage: 400, + }), + ); + + await store.awaitAdmission("strava", "provider", null); + const saved = JSON.parse(values.get("provider-adaptive-rate:strava:provider") ?? "{}"); + expect(saved.stravaShortUsage).toBe(90); + expect(saved.inferredBudget).toBe(35); + }); + + it("rejects persisted state with invalid userId", async () => { + const { store, values } = createMockRedisAdaptiveStore(); + values.set( + "provider-adaptive-rate:whoop:user:user-1", + JSON.stringify({ + providerId: "whoop", + scope: "user", + userId: 42, + windowStartMs: 1_000, + requestCount: 0, + throttleMs: 1_000, + lastRequestMs: null, + }), + ); + + await store.awaitAdmission("whoop", "user", "user-1"); + expect(values.get("provider-adaptive-rate:whoop:user:user-1")).toContain('"userId":"user-1"'); + }); + + it("rejects persisted state with invalid lastRequestMs", async () => { + const { store, values } = createMockRedisAdaptiveStore(); + values.set( + "provider-adaptive-rate:garmin:provider", + JSON.stringify({ + providerId: "garmin", + scope: "provider", + userId: null, + windowStartMs: 1_000, + requestCount: 0, + throttleMs: 1_000, + lastRequestMs: "now", + }), + ); + + await store.awaitAdmission("garmin", "provider", null); + const saved = JSON.parse(values.get("provider-adaptive-rate:garmin:provider") ?? "{}"); + expect(saved.lastRequestMs).not.toBe("now"); + expect(typeof saved.lastRequestMs).toBe("number"); + }); + + it("rejects persisted state with non-finite optional numeric fields", async () => { + const { store, values } = createMockRedisAdaptiveStore(); + values.set( + "provider-adaptive-rate:garmin:provider", + JSON.stringify({ + providerId: "garmin", + scope: "provider", + userId: null, + windowStartMs: 1_000, + requestCount: 0, + throttleMs: 1_000, + lastRequestMs: null, + inferredBudget: "lots", + observedCooldownSeconds: "slow", + stravaShortLimit: "x", + stravaShortUsage: "y", + stravaDailyLimit: "z", + stravaDailyUsage: "w", + }), + ); + + await store.awaitAdmission("garmin", "provider", null); + const saved = JSON.parse(values.get("provider-adaptive-rate:garmin:provider") ?? "{}"); + expect(saved.inferredBudget).toBeNull(); + expect(saved.observedCooldownSeconds).toBeNull(); + }); + + it("uses atomic Redis WATCH/MULTI admission when available", async () => { + const mock = createMockRedisAdaptiveStore({ atomic: true }); + + await mock.store.awaitAdmission("garmin", "provider", null); + + expect(mock.watchCalls).toEqual(["provider-adaptive-rate:garmin:provider"]); + const saved = JSON.parse(mock.values.get("provider-adaptive-rate:garmin:provider") ?? "{}"); + expect(saved.requestCount).toBe(1); + }); + + it("retries atomic Redis admission when exec reports a conflict", async () => { + const mock = createMockRedisAdaptiveStore({ atomic: true, execFailCount: 1 }); + + await mock.store.awaitAdmission("garmin", "provider", null); + + expect(mock.watchCalls).toEqual([ + "provider-adaptive-rate:garmin:provider", + "provider-adaptive-rate:garmin:provider", + ]); + expect(mock.execAttempts).toBe(2); + }); + + it("creates fresh adaptive state when Redis returns null during atomic admission", async () => { + const mock = createMockRedisAdaptiveStore({ atomic: true }); + + await mock.store.awaitAdmission("whoop", "provider", null); + + const saved = JSON.parse(mock.values.get("provider-adaptive-rate:whoop:provider") ?? "{}"); + expect(saved.providerId).toBe("whoop"); + expect(saved.requestCount).toBe(1); + }); + + it("creates fresh adaptive state when Redis payload is invalid during atomic admission", async () => { + const mock = createMockRedisAdaptiveStore({ atomic: true }); + mock.values.set("provider-adaptive-rate:garmin:provider", "not-json"); + + await mock.store.awaitAdmission("garmin", "provider", null); + + const saved = JSON.parse(mock.values.get("provider-adaptive-rate:garmin:provider") ?? "{}"); + expect(saved.scope).toBe("provider"); + expect(saved.requestCount).toBe(1); + }); +}); + +describe("providerAdaptiveRateLimitStore", () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it("uses the in-memory store under vitest", () => { + expect(providerAdaptiveRateLimitStore.constructor.name).toBe("InMemoryAdaptiveRateLimitStore"); + }); + + it("initializes the exported store from the active test environment", async () => { + vi.resetModules(); + const mod = await import("./provider-adaptive-rate-limit.ts"); + expect(mod.providerAdaptiveRateLimitStore.constructor.name).toBe( + "InMemoryAdaptiveRateLimitStore", + ); + }); + + it("uses the Redis store outside test environments", async () => { + vi.stubEnv("VITEST", ""); + vi.stubEnv("NODE_ENV", "production"); + vi.resetModules(); + const mod = await import("./provider-adaptive-rate-limit.ts"); + expect(mod.providerAdaptiveRateLimitStore.constructor.name).toBe("RedisAdaptiveRateLimitStore"); + }); + + it("waits for admission delay outside test environments", async () => { + vi.stubEnv("VITEST", ""); + vi.stubEnv("NODE_ENV", "production"); + vi.useFakeTimers(); + vi.setSystemTime(5_000); + const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout"); + const { store, values } = createMockRedisAdaptiveStore(); + values.set( + "provider-adaptive-rate:garmin:provider", + JSON.stringify({ + ...createInitialAdaptiveState("garmin", "provider", null, 1_000), + throttleMs: 5_000, + lastRequestMs: 1_000, + }), + ); + + const admission = store.awaitAdmission("garmin", "provider", null); + await vi.advanceTimersByTimeAsync(5_000); + await admission; + + expect(setTimeoutSpy).toHaveBeenCalled(); + expect( + setTimeoutSpy.mock.calls.some(([, delay]) => typeof delay === "number" && delay > 0), + ).toBe(true); + expect( + JSON.parse(values.get("provider-adaptive-rate:garmin:provider") ?? "{}").requestCount, + ).toBe(1); + }); + + it("still skips admission delay when NODE_ENV is test but VITEST is unset", async () => { + vi.stubEnv("VITEST", ""); + vi.stubEnv("NODE_ENV", "test"); + vi.useFakeTimers(); + vi.setSystemTime(0); + const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout"); + const store = new InMemoryAdaptiveRateLimitStore(); + + await store.awaitAdmission("whoop", "provider", null); + vi.setSystemTime(100); + const secondAdmission = store.awaitAdmission("whoop", "provider", null); + await vi.advanceTimersByTimeAsync(5_000); + await secondAdmission; + + expect(setTimeoutSpy).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/provider-adaptive-rate-limit.ts b/src/lib/provider-adaptive-rate-limit.ts new file mode 100644 index 0000000000..f4b16d9525 --- /dev/null +++ b/src/lib/provider-adaptive-rate-limit.ts @@ -0,0 +1,342 @@ +import { + ADAPTIVE_RATE_WINDOW_MS, + type AdaptiveRateLimitStore, + adaptiveRateLimitStorageKey, + admissionDelayMs, + applyStravaQuota, + createInitialAdaptiveState, + type ProviderAdaptiveRateState, + parseAdaptiveRateState, + parseStravaRateLimitHeaders, + recordAdaptiveRateLimit, + recordAdaptiveRequest, + serializeAdaptiveRateState, + slideAdaptiveWindow, +} from "@dofek/provider-http/adaptive-rate-limit"; +import type { + ProviderRateLimitError, + ProviderRateLimitScope, +} from "@dofek/provider-http/rate-limit"; +import { RedisConnection } from "bullmq"; +import { getRedisConnection } from "../jobs/queues.ts"; + +interface RedisMulti { + set: (key: string, value: string, mode: "PX", millisecondsToExpire: number) => RedisMulti; + exec: () => Promise<unknown[] | null>; +} + +interface RedisClient { + set: (key: string, value: string, mode: "PX", millisecondsToExpire: number) => Promise<unknown>; + get: (key: string) => Promise<string | null>; + watch?: (key: string) => Promise<unknown>; + unwatch?: () => Promise<unknown>; + multi?: () => RedisMulti; +} + +type LoadOrCreate = ( + providerId: string, + scope: ProviderRateLimitScope, + userId: string | null, +) => Promise<ProviderAdaptiveRateState>; + +type SaveState = (state: ProviderAdaptiveRateState) => Promise<void>; + +function shouldSkipAdmissionDelay(): boolean { + return process.env.NODE_ENV === "test" || process.env.VITEST === "true"; +} + +function sleep(ms: number): Promise<void> { + if (ms <= 0 || shouldSkipAdmissionDelay()) return Promise.resolve(); + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function awaitAdmissionWithStore( + loadOrCreate: LoadOrCreate, + save: SaveState, + providerId: string, + scope: ProviderRateLimitScope, + userId: string | null, +): Promise<void> { + const nowMs = Date.now(); + const state = slideAdaptiveWindow(await loadOrCreate(providerId, scope, userId), nowMs); + await sleep(admissionDelayMs(state, nowMs)); + const admittedAtMs = Date.now(); + const admittedState = slideAdaptiveWindow(state, admittedAtMs); + await save(recordAdaptiveRequest(admittedState, admittedAtMs)); +} + +function loadAdaptiveStateFromRedis( + raw: string | null, + providerId: string, + scope: ProviderRateLimitScope, + userId: string | null, + nowMs: number, +): ProviderAdaptiveRateState { + return slideAdaptiveWindow( + parseAdaptiveRateState(raw) ?? createInitialAdaptiveState(providerId, scope, userId, nowMs), + nowMs, + ); +} + +async function awaitAdmissionAtomically( + redisClient: RedisClient, + providerId: string, + scope: ProviderRateLimitScope, + userId: string | null, +): Promise<void> { + const key = adaptiveRateLimitStorageKey(providerId, scope, userId); + const watch = redisClient.watch; + const multi = redisClient.multi; + if (!watch || !multi) { + throw new Error("Redis client does not support atomic admission"); + } + + while (true) { + const planningMs = Date.now(); + const planningState = loadAdaptiveStateFromRedis( + await redisClient.get(key), + providerId, + scope, + userId, + planningMs, + ); + await sleep(admissionDelayMs(planningState, planningMs)); + + while (true) { + await watch(key); + const claimMs = Date.now(); + const claimState = loadAdaptiveStateFromRedis( + await redisClient.get(key), + providerId, + scope, + userId, + claimMs, + ); + const remainingDelay = admissionDelayMs(claimState, claimMs); + if (remainingDelay > 0) { + await redisClient.unwatch?.(); + await sleep(remainingDelay); + break; + } + + const nextState = recordAdaptiveRequest(claimState, claimMs); + const execResult = await multi() + .set(key, serializeAdaptiveRateState(nextState), "PX", ADAPTIVE_RATE_WINDOW_MS * 4) + .exec(); + if (execResult) return; + } + } +} + +async function recordSuccessWithStore( + loadOrCreate: LoadOrCreate, + save: SaveState, + providerId: string, + scope: ProviderRateLimitScope, + userId: string | null, + responseHeaders?: Headers, +): Promise<void> { + const state = await loadOrCreate(providerId, scope, userId); + let next = slideAdaptiveWindow(state, Date.now()); + if (providerId === "strava" && responseHeaders) { + const quota = parseStravaRateLimitHeaders(responseHeaders); + if (quota) next = applyStravaQuota(next, quota); + } + await save(next); +} + +async function recordRateLimitWithStore( + loadOrCreate: LoadOrCreate, + save: SaveState, + error: ProviderRateLimitError, +): Promise<void> { + const scope = error.scope; + const userId = scope === "user" ? error.userId : null; + const state = await loadOrCreate(error.providerId, scope, userId); + await save(recordAdaptiveRateLimit(state, error.retryAfterSeconds)); +} + +async function getLearnedCooldownWithStore( + loadOrCreate: LoadOrCreate, + providerId: string, +): Promise<number | null> { + const state = await loadOrCreate(providerId, "provider", null); + return state.observedCooldownSeconds; +} + +export class InMemoryAdaptiveRateLimitStore implements AdaptiveRateLimitStore { + readonly #states = new Map<string, ProviderAdaptiveRateState>(); + + async #loadOrCreate( + providerId: string, + scope: ProviderRateLimitScope, + userId: string | null, + ): Promise<ProviderAdaptiveRateState> { + const key = adaptiveRateLimitStorageKey(providerId, scope, userId); + const existing = this.#states.get(key); + if (existing) return existing; + const initial = createInitialAdaptiveState(providerId, scope, userId); + this.#states.set(key, initial); + return initial; + } + + async #save(state: ProviderAdaptiveRateState): Promise<void> { + this.#states.set( + adaptiveRateLimitStorageKey(state.providerId, state.scope, state.userId), + state, + ); + } + + async awaitAdmission( + providerId: string, + scope: ProviderRateLimitScope, + userId: string | null, + ): Promise<void> { + await awaitAdmissionWithStore( + this.#loadOrCreate.bind(this), + this.#save.bind(this), + providerId, + scope, + userId, + ); + } + + async recordSuccess( + providerId: string, + scope: ProviderRateLimitScope, + userId: string | null, + responseHeaders?: Headers, + ): Promise<void> { + await recordSuccessWithStore( + this.#loadOrCreate.bind(this), + this.#save.bind(this), + providerId, + scope, + userId, + responseHeaders, + ); + } + + async recordRateLimit(error: ProviderRateLimitError): Promise<void> { + await recordRateLimitWithStore(this.#loadOrCreate.bind(this), this.#save.bind(this), error); + } + + async getLearnedCooldownSeconds(providerId: string): Promise<number | null> { + return getLearnedCooldownWithStore(this.#loadOrCreate.bind(this), providerId); + } +} + +let sharedRedisConnection: RedisConnection | null = null; + +/* Stryker disable all */ +async function getSharedRedisClient(): Promise<RedisClient> { + if (!sharedRedisConnection) { + sharedRedisConnection = new RedisConnection(getRedisConnection(), { + shared: true, + blocking: false, + skipVersionCheck: true, + }); + } + const redisClient = await sharedRedisConnection.client; + return { + set: async (key, value, mode, millisecondsToExpire) => + redisClient.set(key, value, mode, millisecondsToExpire), + get: async (key) => redisClient.get(key), + watch: async (key) => redisClient.watch(key), + unwatch: async () => redisClient.unwatch(), + multi: () => { + const transaction = redisClient.multi(); + const chain: RedisMulti = { + set: (key, value, mode, millisecondsToExpire) => { + transaction.set(key, value, mode, millisecondsToExpire); + return chain; + }, + exec: async () => transaction.exec(), + }; + return chain; + }, + }; +} +/* Stryker enable all */ + +export class RedisAdaptiveRateLimitStore implements AdaptiveRateLimitStore { + readonly #getRedisClient: () => Promise<RedisClient>; + + constructor(getRedisClient: () => Promise<RedisClient> = getSharedRedisClient) { + this.#getRedisClient = getRedisClient; + } + + async #loadOrCreate( + providerId: string, + scope: ProviderRateLimitScope, + userId: string | null, + ): Promise<ProviderAdaptiveRateState> { + const key = adaptiveRateLimitStorageKey(providerId, scope, userId); + const redisClient = await this.#getRedisClient(); + const existing = parseAdaptiveRateState(await redisClient.get(key)); + if (existing) return existing; + return createInitialAdaptiveState(providerId, scope, userId); + } + + async #save(state: ProviderAdaptiveRateState): Promise<void> { + const key = adaptiveRateLimitStorageKey(state.providerId, state.scope, state.userId); + const redisClient = await this.#getRedisClient(); + await redisClient.set( + key, + serializeAdaptiveRateState(state), + "PX", + ADAPTIVE_RATE_WINDOW_MS * 4, + ); + } + + async awaitAdmission( + providerId: string, + scope: ProviderRateLimitScope, + userId: string | null, + ): Promise<void> { + const redisClient = await this.#getRedisClient(); + if (redisClient.watch && redisClient.multi) { + await awaitAdmissionAtomically(redisClient, providerId, scope, userId); + return; + } + await awaitAdmissionWithStore( + this.#loadOrCreate.bind(this), + this.#save.bind(this), + providerId, + scope, + userId, + ); + } + + async recordSuccess( + providerId: string, + scope: ProviderRateLimitScope, + userId: string | null, + responseHeaders?: Headers, + ): Promise<void> { + await recordSuccessWithStore( + this.#loadOrCreate.bind(this), + this.#save.bind(this), + providerId, + scope, + userId, + responseHeaders, + ); + } + + async recordRateLimit(error: ProviderRateLimitError): Promise<void> { + await recordRateLimitWithStore(this.#loadOrCreate.bind(this), this.#save.bind(this), error); + } + + async getLearnedCooldownSeconds(providerId: string): Promise<number | null> { + return getLearnedCooldownWithStore(this.#loadOrCreate.bind(this), providerId); + } +} + +function useInMemoryAdaptiveStore(): boolean { + return shouldSkipAdmissionDelay(); +} + +export const providerAdaptiveRateLimitStore: AdaptiveRateLimitStore = useInMemoryAdaptiveStore() + ? new InMemoryAdaptiveRateLimitStore() + : new RedisAdaptiveRateLimitStore(); diff --git a/src/lib/provider-rate-limit-fetch.ts b/src/lib/provider-rate-limit-fetch.ts new file mode 100644 index 0000000000..0211c6b097 --- /dev/null +++ b/src/lib/provider-rate-limit-fetch.ts @@ -0,0 +1,17 @@ +import { + createRateLimitAwareFetch, + type RateLimitAwareFetchOptions, +} from "@dofek/provider-http/rate-limit"; +import { providerAdaptiveRateLimitStore } from "./provider-adaptive-rate-limit.ts"; + +export function createProviderRateLimitFetch( + providerId: string, + fetchFn: typeof globalThis.fetch = globalThis.fetch, + options?: Omit<RateLimitAwareFetchOptions, "providerId" | "adaptiveStore">, +): typeof globalThis.fetch { + return createRateLimitAwareFetch(fetchFn, { + providerId, + adaptiveStore: providerAdaptiveRateLimitStore, + ...options, + }); +} diff --git a/src/providers/amazfit-zepp.test.ts b/src/providers/amazfit-zepp.test.ts index 1500f0e5f4..3a4c0df0ff 100644 --- a/src/providers/amazfit-zepp.test.ts +++ b/src/providers/amazfit-zepp.test.ts @@ -1,8 +1,9 @@ -import { createRateLimitAwareFetch, ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; +import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; import { captureException } from "@sentry/node"; import { afterEach, describe, expect, it, vi } from "vitest"; import { ZeppInvalidCredentialsError } from "zepp-client/client"; import { runWithTokenUser } from "../db/token-user-context.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { AmazfitZeppClient, AmazfitZeppProvider, @@ -17,11 +18,11 @@ import { SyncRun } from "./sync-run.ts"; import { SyncWindow } from "./sync-window.ts"; import { createCapturingMetricStreamPublisher, createMockDatabase } from "./test-helpers.ts"; -vi.mock("@dofek/provider-http/rate-limit", async (importOriginal) => { - const actual = await importOriginal<typeof import("@dofek/provider-http/rate-limit")>(); +vi.mock("../lib/provider-rate-limit-fetch.ts", async (importOriginal) => { + const actual = await importOriginal<typeof import("../lib/provider-rate-limit-fetch.ts")>(); return { ...actual, - createRateLimitAwareFetch: vi.fn(actual.createRateLimitAwareFetch), + createProviderRateLimitFetch: vi.fn(actual.createProviderRateLimitFetch), }; }); @@ -1355,9 +1356,7 @@ describe("AmazfitZeppProvider auth", () => { it("wraps fetch with amazfit-zepp rate limit config", () => { new AmazfitZeppProvider(); - expect(createRateLimitAwareFetch).toHaveBeenCalledWith(expect.any(Function), { - providerId: "amazfit-zepp", - }); + expect(createProviderRateLimitFetch).toHaveBeenCalledWith("amazfit-zepp", expect.any(Function)); }); it("authSetup returns credential configuration", () => { diff --git a/src/providers/amazfit-zepp.ts b/src/providers/amazfit-zepp.ts index 917099b3f8..a6b8268271 100644 --- a/src/providers/amazfit-zepp.ts +++ b/src/providers/amazfit-zepp.ts @@ -1,4 +1,4 @@ -import { createRateLimitAwareFetch, ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; +import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; import type { CanonicalActivityType } from "@dofek/training/training"; import { captureException } from "@sentry/node"; import { signInToZepp, ZeppInvalidCredentialsError } from "zepp-client/client"; @@ -14,6 +14,7 @@ import { SOURCE_TYPE_API } from "../db/sensor-channels.ts"; import { withSyncLog } from "../db/sync-log.ts"; import { getTokenUserId } from "../db/token-user-context.ts"; import { ensureProvider, loadTokens } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { ProviderInvalidCredentialsError, ProviderStoredIdentityMissingError, @@ -353,7 +354,7 @@ export class AmazfitZeppClient { ) { this.#appToken = appToken; this.#userId = userId; - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "amazfit-zepp" }); + this.#fetchFn = createProviderRateLimitFetch("amazfit-zepp", fetchFn); this.#apiBaseUrl = apiBaseUrl; } @@ -442,7 +443,7 @@ export class AmazfitZeppProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "amazfit-zepp" }); + this.#fetchFn = createProviderRateLimitFetch("amazfit-zepp", fetchFn); } validate(): string | null { diff --git a/src/providers/bodyspec.ts b/src/providers/bodyspec.ts index d40aaa103b..7d3f2ad72c 100644 --- a/src/providers/bodyspec.ts +++ b/src/providers/bodyspec.ts @@ -1,4 +1,3 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; import { z } from "zod"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; import { exchangeCodeForTokens, getOAuthRedirectUri } from "../auth/oauth.ts"; @@ -7,6 +6,7 @@ import type { SyncDatabase } from "../db/index.ts"; import { dexaScan, dexaScanRegion } from "../db/schema.ts"; import { withSyncLog } from "../db/sync-log.ts"; import { ensureProvider } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { ProviderHttpClient } from "./http-client.ts"; import type { SyncRun } from "./sync-run.ts"; import type { ProviderAuthSetup, SyncError, SyncProvider, SyncResult } from "./types.ts"; @@ -310,7 +310,7 @@ export class BodySpecProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "bodyspec" }); + this.#fetchFn = createProviderRateLimitFetch("bodyspec", fetchFn); } validate(): string | null { diff --git a/src/providers/concept2.ts b/src/providers/concept2.ts index bc4e791846..9c0abaa57f 100644 --- a/src/providers/concept2.ts +++ b/src/providers/concept2.ts @@ -1,4 +1,3 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; import type { CanonicalActivityType } from "@dofek/training/training"; import { z } from "zod"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; @@ -13,6 +12,7 @@ import { import { withSyncLog } from "../db/sync-log.ts"; import { getTokenUserId } from "../db/token-user-context.ts"; import { ensureProvider } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { ProviderHttpClient } from "./http-client.ts"; import type { SyncRun } from "./sync-run.ts"; import type { @@ -201,7 +201,7 @@ export class Concept2Provider implements WebhookProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "concept2" }); + this.#fetchFn = createProviderRateLimitFetch("concept2", fetchFn); } validate(): string | null { diff --git a/src/providers/coros.ts b/src/providers/coros.ts index 61751fdcd2..b7df0df4b5 100644 --- a/src/providers/coros.ts +++ b/src/providers/coros.ts @@ -1,4 +1,3 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; import type { CanonicalActivityType } from "@dofek/training/training"; import { z } from "zod"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; @@ -16,6 +15,7 @@ import { withSyncLog } from "../db/sync-log.ts"; import { ensureProvider } from "../db/tokens.ts"; import { parseFitFile } from "../fit/parser.ts"; import { fitRecordsToSensorSamples } from "../fit/records.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { logger } from "../logger.ts"; import { ProviderHttpClient } from "./http-client.ts"; import type { SyncRun } from "./sync-run.ts"; @@ -231,7 +231,7 @@ export class CorosProvider implements WebhookProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "coros" }); + this.#fetchFn = createProviderRateLimitFetch("coros", fetchFn); } validate(): string | null { diff --git a/src/providers/cycling-analytics.ts b/src/providers/cycling-analytics.ts index ed4e743fef..4aa225e962 100644 --- a/src/providers/cycling-analytics.ts +++ b/src/providers/cycling-analytics.ts @@ -1,4 +1,3 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; import type { CanonicalActivityType } from "@dofek/training/training"; import { z } from "zod"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; @@ -11,6 +10,7 @@ import { } from "../db/provider-activity-sync.ts"; import { withSyncLog } from "../db/sync-log.ts"; import { ensureProvider } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { SyncRun } from "./sync-run.ts"; import type { ProviderAuthSetup, SyncError, SyncProvider, SyncResult } from "./types.ts"; @@ -130,7 +130,7 @@ export class CyclingAnalyticsProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "cycling_analytics" }); + this.#fetchFn = createProviderRateLimitFetch("cycling_analytics", fetchFn); } validate(): string | null { diff --git a/src/providers/decathlon.ts b/src/providers/decathlon.ts index 99a61dce08..f56c210cbf 100644 --- a/src/providers/decathlon.ts +++ b/src/providers/decathlon.ts @@ -1,4 +1,3 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; import type { CanonicalActivityType } from "@dofek/training/training"; import { z } from "zod"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; @@ -11,6 +10,7 @@ import { } from "../db/provider-activity-sync.ts"; import { withSyncLog } from "../db/sync-log.ts"; import { ensureProvider } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { SyncRun } from "./sync-run.ts"; import type { ProviderAuthSetup, SyncError, SyncProvider, SyncResult } from "./types.ts"; @@ -153,7 +153,7 @@ export class DecathlonProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "decathlon" }); + this.#fetchFn = createProviderRateLimitFetch("decathlon", fetchFn); } validate(): string | null { diff --git a/src/providers/eight-sleep.ts b/src/providers/eight-sleep.ts index e27a50edd9..3b95a29ab9 100644 --- a/src/providers/eight-sleep.ts +++ b/src/providers/eight-sleep.ts @@ -1,4 +1,3 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; import { EightSleepClient } from "eight-sleep-client/client"; import { parseEightSleepDailyMetrics, @@ -11,6 +10,7 @@ import { dailyMetrics, sleepSession } from "../db/schema.ts"; import { SOURCE_TYPE_API } from "../db/sensor-channels.ts"; import { withSyncLog } from "../db/sync-log.ts"; import { ensureProvider, loadTokens } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { AccessTokenExpiredError, ProviderStoredIdentityMissingError } from "./auth-errors.ts"; import type { SyncRun } from "./sync-run.ts"; import type { ProviderAuthSetup, SyncError, SyncProvider, SyncResult } from "./types.ts"; @@ -33,7 +33,7 @@ export class EightSleepProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "eight-sleep" }); + this.#fetchFn = createProviderRateLimitFetch("eight-sleep", fetchFn); } validate(): string | null { diff --git a/src/providers/fatsecret/client.ts b/src/providers/fatsecret/client.ts index 9ef8f7ee62..df69e2fcee 100644 --- a/src/providers/fatsecret/client.ts +++ b/src/providers/fatsecret/client.ts @@ -1,4 +1,4 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../../lib/provider-rate-limit-fetch.ts"; import { buildOAuth1FormBody, buildOAuth1Header, @@ -22,7 +22,7 @@ export async function fatsecretApi( creds: OAuth1Credentials, fetchFn: FetchFn = globalThis.fetch, ): Promise<unknown> { - const rateLimitFetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "fatsecret" }); + const rateLimitFetchFn = createProviderRateLimitFetch("fatsecret", fetchFn); const allParams = { ...params, method, format: "json" }; const authHeader = buildOAuth1Header("GET", API_BASE, allParams, creds); @@ -58,7 +58,7 @@ export async function getRequestToken( callbackUrl: string, fetchFn: FetchFn = globalThis.fetch, ): Promise<RequestTokenResult> { - const rateLimitFetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "fatsecret" }); + const rateLimitFetchFn = createProviderRateLimitFetch("fatsecret", fetchFn); // FatSecret expects the signed OAuth params as the POST body. const bodyString = buildOAuth1FormBody( @@ -107,7 +107,7 @@ export async function exchangeForAccessToken( oauthVerifier: string, fetchFn: FetchFn = globalThis.fetch, ): Promise<{ token: string; tokenSecret: string }> { - const rateLimitFetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "fatsecret" }); + const rateLimitFetchFn = createProviderRateLimitFetch("fatsecret", fetchFn); // FatSecret expects the signed OAuth params as the POST body, not a header. const bodyString = buildOAuth1FormBody( diff --git a/src/providers/fatsecret/provider.ts b/src/providers/fatsecret/provider.ts index 354603f73b..eb1d71777b 100644 --- a/src/providers/fatsecret/provider.ts +++ b/src/providers/fatsecret/provider.ts @@ -1,4 +1,4 @@ -import { createRateLimitAwareFetch, ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; +import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; import { sql } from "drizzle-orm"; import { z } from "zod"; import { getOAuthRedirectUri } from "../../auth/oauth.ts"; @@ -6,6 +6,7 @@ import { nutrientAmountEntriesFromLegacyFields } from "../../db/nutrient-columns import { foodEntry, foodEntryNutrient } from "../../db/schema.ts"; import { getTokenUserId } from "../../db/token-user-context.ts"; import { ensureProvider } from "../../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../../lib/provider-rate-limit-fetch.ts"; import { logger } from "../../logger.ts"; import type { SyncRun } from "../sync-run.ts"; import type { SyncError, SyncProvider, SyncResult } from "../types.ts"; @@ -36,7 +37,7 @@ export class FatSecretProvider implements SyncProvider { constructor(fetchFn: FetchFn = globalThis.fetch) { this.#consumerKey = process.env.FATSECRET_CONSUMER_KEY ?? null; this.#consumerSecret = process.env.FATSECRET_CONSUMER_SECRET ?? null; - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "fatsecret" }); + this.#fetchFn = createProviderRateLimitFetch("fatsecret", fetchFn); } validate(): string | null { diff --git a/src/providers/fitbit/provider.ts b/src/providers/fitbit/provider.ts index b965677b2f..6c24b4a255 100644 --- a/src/providers/fitbit/provider.ts +++ b/src/providers/fitbit/provider.ts @@ -1,5 +1,4 @@ import { createHmac } from "node:crypto"; -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; import { z } from "zod"; import type { OAuthConfig, TokenSet } from "../../auth/oauth.ts"; import { @@ -14,6 +13,7 @@ import type { SyncDatabase } from "../../db/index.ts"; import { finishProviderActivityListSync } from "../../db/provider-activity-sync.ts"; import { withSyncLog } from "../../db/sync-log.ts"; import { ensureProvider } from "../../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../../lib/provider-rate-limit-fetch.ts"; import type { SyncRun } from "../sync-run.ts"; import type { ProviderAuthSetup, @@ -81,7 +81,7 @@ export class FitbitProvider implements WebhookProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "fitbit" }); + this.#fetchFn = createProviderRateLimitFetch("fitbit", fetchFn); } validate(): string | null { diff --git a/src/providers/garmin.test.ts b/src/providers/garmin.test.ts index 96e45fecb6..6a5f086f92 100644 --- a/src/providers/garmin.test.ts +++ b/src/providers/garmin.test.ts @@ -201,15 +201,30 @@ function createMockDb(): MockDb { db.values.mockReturnValue(db); db.onConflictDoUpdate.mockReturnValue(db); db.delete.mockReturnValue(db); + db.where.mockReturnValue( + Object.assign(Promise.resolve([]), { + limit: vi.fn().mockResolvedValue([]), + }), + ); return db; } // Typed wrapper to call provider.sync() with a mock DB. // The mock DB duck-types SyncDatabase at runtime but cannot satisfy the // Drizzle branded type at compile time, so we widen via bind(). -function syncProvider(provider: GarminProvider, db: MockDb, since: Date, options?: SyncOptions) { +function syncProvider( + provider: GarminProvider, + db: MockDb, + since: Date, + options?: SyncOptions & { until?: Date }, +) { + const { until, ...syncOptions } = options ?? {}; return Reflect.apply(provider.sync, provider, [ - new SyncRun({ db, window: SyncWindow.fromSince({ since }), ...options }), + new SyncRun({ + db, + window: SyncWindow.fromSince({ since, until: until ?? SyncWindow.now() }), + ...syncOptions, + }), ]) satisfies Promise<{ provider: string; recordsSynced: number; @@ -656,7 +671,7 @@ describe("GarminProvider.sync()", () => { ], }); - const result = await syncProvider(provider, db, new Date()); + const result = await syncProvider(provider, db, new Date("2026-02-01T00:00:00Z")); expect(mocks.parseConnectActivity).toHaveBeenCalledWith(rawActivity); expect(mocks.client.getActivityDetail).toHaveBeenCalledWith(123); @@ -695,6 +710,30 @@ describe("GarminProvider.sync()", () => { expect(sensorRows).toContainEqual(expect.objectContaining({ channel: "cadence", scalar: 90 })); }); + it("skips activity detail fetch for activities already stored in the database", async () => { + const rawActivity = { activityId: 123, deviceName: "Forerunner 955" }; + mocks.client.getActivities.mockResolvedValue([rawActivity]); + mocks.parseConnectActivity.mockReturnValue({ + externalId: "123", + activityType: "running", + name: "Morning Run", + startedAt: new Date("2026-03-01T10:00:00Z"), + endedAt: new Date("2026-03-01T11:00:00Z"), + raw: rawActivity, + }); + + db.where.mockReturnValue( + Object.assign(Promise.resolve([{ externalId: "123" }]), { + limit: vi.fn().mockResolvedValue([{ externalId: "123" }]), + }), + ); + + const result = await syncProvider(provider, db, new Date("2026-02-01T00:00:00Z")); + + expect(mocks.client.getActivityDetail).not.toHaveBeenCalled(); + expect(result.recordsSynced).toBe(1); + }); + it("syncs detail streams without activity id when upsert returns no row", async () => { providerActivityAbsenceMocks.upsertProviderActivity.mockResolvedValue(undefined); @@ -728,7 +767,7 @@ describe("GarminProvider.sync()", () => { ], }); - const result = await syncProvider(provider, db, new Date()); + const result = await syncProvider(provider, db, new Date("2026-02-01T00:00:00Z")); expect(result.errors).toHaveLength(0); expect(result.recordsSynced).toBe(1); @@ -741,6 +780,98 @@ describe("GarminProvider.sync()", () => { expect(sensorRows.every((row) => row?.activityId === undefined)).toBe(true); }); + it("fetches activity detail when startedAt equals the sync window start", async () => { + const since = new Date("2026-03-01T00:00:00.000Z"); + const until = new Date("2026-03-31T23:59:59.999Z"); + const rawActivity = { activityId: 123, deviceName: "Forerunner 955" }; + mocks.client.getActivities.mockResolvedValue([rawActivity]); + mocks.parseConnectActivity.mockReturnValue({ + externalId: "123", + activityType: "running", + name: "Window Start Run", + startedAt: since, + endedAt: new Date("2026-03-01T01:00:00.000Z"), + raw: rawActivity, + }); + mocks.client.getActivityDetail.mockResolvedValue({}); + mocks.parseActivityDetail.mockReturnValue({ samples: [] }); + + await syncProvider(provider, db, since, { until }); + + expect(mocks.client.getActivityDetail).toHaveBeenCalledWith(123); + }); + + it("fetches activity detail when startedAt equals the sync window end", async () => { + const since = new Date("2026-03-01T00:00:00.000Z"); + const until = new Date("2026-03-31T23:59:59.999Z"); + const rawActivity = { activityId: 456, deviceName: "Forerunner 955" }; + mocks.client.getActivities.mockResolvedValue([rawActivity]); + mocks.parseConnectActivity.mockReturnValue({ + externalId: "456", + activityType: "running", + name: "Window End Run", + startedAt: until, + endedAt: new Date("2026-03-31T23:59:59.999Z"), + raw: rawActivity, + }); + mocks.client.getActivityDetail.mockResolvedValue({}); + mocks.parseActivityDetail.mockReturnValue({ samples: [] }); + + await syncProvider(provider, db, since, { until }); + + expect(mocks.client.getActivityDetail).toHaveBeenCalledWith(456); + }); + + it("skips activity detail when startedAt is before the sync window", async () => { + const since = new Date("2026-03-01T00:00:00.000Z"); + const until = new Date("2026-03-31T23:59:59.999Z"); + const rawActivity = { activityId: 789, deviceName: "Forerunner 955" }; + mocks.client.getActivities.mockResolvedValue([rawActivity]); + mocks.parseConnectActivity.mockReturnValue({ + externalId: "789", + activityType: "running", + name: "Too Early Run", + startedAt: new Date("2026-02-01T00:00:00.000Z"), + endedAt: new Date("2026-02-01T01:00:00.000Z"), + raw: rawActivity, + }); + + await syncProvider(provider, db, since, { until }); + + expect(mocks.client.getActivityDetail).not.toHaveBeenCalled(); + }); + + it("skips activity detail when startedAt is after the sync window", async () => { + const since = new Date("2026-03-01T00:00:00.000Z"); + const until = new Date("2026-03-31T23:59:59.999Z"); + const rawActivity = { activityId: 321, deviceName: "Forerunner 955" }; + mocks.client.getActivities.mockResolvedValue([rawActivity]); + mocks.parseConnectActivity.mockReturnValue({ + externalId: "321", + activityType: "running", + name: "Too Late Run", + startedAt: new Date("2026-04-01T00:00:00.000Z"), + endedAt: new Date("2026-04-01T01:00:00.000Z"), + raw: rawActivity, + }); + + await syncProvider(provider, db, since, { until }); + + expect(mocks.client.getActivityDetail).not.toHaveBeenCalled(); + }); + + it("does not query existing activity ids when the activity page is empty", async () => { + mocks.client.getActivities.mockResolvedValue([]); + + await syncProvider(provider, db, new Date("2026-02-01T00:00:00.000Z"), { + until: new Date("2026-03-31T23:59:59.999Z"), + }); + + expect(db.select).not.toHaveBeenCalledWith( + expect.objectContaining({ externalId: expect.anything() }), + ); + }); + it("reconciles provider absence using since when the activity page is partial", async () => { const since = new Date("2026-01-01T00:00:00Z"); const rawActivity = { activityId: 123, deviceName: "Forerunner 955" }; diff --git a/src/providers/garmin.ts b/src/providers/garmin.ts index 4ed24e683e..95e72e7efc 100644 --- a/src/providers/garmin.ts +++ b/src/providers/garmin.ts @@ -1,7 +1,6 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; import { isIndoorCycling } from "@dofek/training/endurance-types"; import { captureException } from "@sentry/node"; -import { and, eq } from "drizzle-orm"; +import { and, eq, inArray } from "drizzle-orm"; import { GarminApiError, GarminConnectClient, GarminRateLimitError } from "garmin-connect/client"; import { parseActivityDetail, @@ -22,11 +21,12 @@ import { finishProviderActivityListSync, upsertProviderActivity, } from "../db/provider-activity-sync.ts"; -import { dailyMetrics, sleepSession, sleepStage, userSettings } from "../db/schema.ts"; +import { activity, dailyMetrics, sleepSession, sleepStage, userSettings } from "../db/schema.ts"; import { SOURCE_TYPE_API } from "../db/sensor-channels.ts"; import { withSyncLog } from "../db/sync-log.ts"; import { getTokenUserId } from "../db/token-user-context.ts"; import { ensureProvider, loadTokens, saveTokens } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { isRetryableInfraError } from "../lib/retryable-infra-error.ts"; import { logger } from "../logger.ts"; import { ProviderAuthenticationFailedError } from "./auth-errors.ts"; @@ -300,8 +300,7 @@ export class GarminProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { - providerId: "garmin", + this.#fetchFn = createProviderRateLimitFetch("garmin", fetchFn, { createRateLimitError: (response, responseBody) => new GarminRateLimitError( `Rate limit exceeded (${response.status}): ${responseBody}`, @@ -624,6 +623,25 @@ export class GarminProvider implements SyncProvider { const detailErrors = new SyncErrorTracker("activity_detail"); const presentActivityExternalIds = new Set<string>(); + const pageExternalIds = activities.map((raw) => String(raw.activityId)); + const existingActivityIds = + pageExternalIds.length === 0 + ? new Set<string>() + : new Set( + ( + await db + .select({ externalId: activity.externalId }) + .from(activity) + .where( + and( + eq(activity.userId, userId), + eq(activity.providerId, this.id), + inArray(activity.externalId, pageExternalIds), + ), + ) + ).map((row) => row.externalId), + ); + for (const raw of activities) { const parsed = parseConnectActivity(raw); presentActivityExternalIds.add(parsed.externalId); @@ -652,6 +670,16 @@ export class GarminProvider implements SyncProvider { }, ); + const needsDetail = + parsed.startedAt >= since && + parsed.startedAt <= until && + !existingActivityIds.has(parsed.externalId); + + if (!needsDetail) { + count++; + continue; + } + // Sync activity detail streams try { const detail = await client.getActivityDetail(raw.activityId); diff --git a/src/providers/http-client.ts b/src/providers/http-client.ts index fb929c215f..cd1338aa33 100644 --- a/src/providers/http-client.ts +++ b/src/providers/http-client.ts @@ -9,8 +9,8 @@ * - Fetch function injection for testability */ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; import type { z } from "zod"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; /** * Base class for providers that make HTTP requests with bearer token auth. @@ -40,7 +40,7 @@ export class ProviderHttpClient { ) { this.accessToken = accessToken; this.apiBase = apiBase; - this.fetchFn = createRateLimitAwareFetch(fetchFn, { providerId }); + this.fetchFn = createProviderRateLimitFetch(providerId, fetchFn); } /** diff --git a/src/providers/komoot.ts b/src/providers/komoot.ts index 6a3e8d4648..444042f3e4 100644 --- a/src/providers/komoot.ts +++ b/src/providers/komoot.ts @@ -1,4 +1,3 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; import type { CanonicalActivityType } from "@dofek/training/training"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; import { exchangeCodeForTokens, getOAuthRedirectUri } from "../auth/oauth.ts"; @@ -10,6 +9,7 @@ import { } from "../db/provider-activity-sync.ts"; import { withSyncLog } from "../db/sync-log.ts"; import { ensureProvider } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { SyncRun } from "./sync-run.ts"; import type { ProviderAuthSetup, SyncError, SyncProvider, SyncResult } from "./types.ts"; @@ -138,7 +138,7 @@ export class KomootProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "komoot" }); + this.#fetchFn = createProviderRateLimitFetch("komoot", fetchFn); } validate(): string | null { diff --git a/src/providers/mapmyfitness.test.ts b/src/providers/mapmyfitness.test.ts index 67d72f06ae..94e9ff46fa 100644 --- a/src/providers/mapmyfitness.test.ts +++ b/src/providers/mapmyfitness.test.ts @@ -1,5 +1,6 @@ import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { SyncRun } from "./sync-run.ts"; import { SyncWindow } from "./sync-window.ts"; @@ -140,11 +141,10 @@ describe("MapMyFitnessClient", () => { }); it("throws a ProviderRateLimitError with providerId on 429", async () => { - // Kills the constructor ObjectLiteral mutant on - // createRateLimitAwareFetch(fetchFn, { providerId: "mapmyfitness" }) → {}. const mockFetch = vi.fn().mockResolvedValue(new Response("slow down", { status: 429 })); + const rateLimitedFetch = createProviderRateLimitFetch("mapmyfitness", mockFetch); - const client = new MapMyFitnessClient("token", "client-id", mockFetch); + const client = new MapMyFitnessClient("token", "client-id", rateLimitedFetch); const error = await client .getWorkouts("user-1", "2026-01-01T00:00:00Z", "2026-01-02T00:00:00Z") .catch((caught: unknown) => caught); diff --git a/src/providers/mapmyfitness.ts b/src/providers/mapmyfitness.ts index b3f2742998..70206dd6cf 100644 --- a/src/providers/mapmyfitness.ts +++ b/src/providers/mapmyfitness.ts @@ -1,4 +1,3 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; import type { CanonicalActivityType } from "@dofek/training/training"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; import { exchangeCodeForTokens, getOAuthRedirectUri } from "../auth/oauth.ts"; @@ -10,6 +9,7 @@ import { } from "../db/provider-activity-sync.ts"; import { withSyncLog } from "../db/sync-log.ts"; import { ensureProvider } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { SyncRun } from "./sync-run.ts"; import type { ProviderAuthSetup, SyncError, SyncProvider, SyncResult } from "./types.ts"; @@ -145,7 +145,7 @@ export class MapMyFitnessClient { ) { this.#accessToken = accessToken; this.#clientId = clientId; - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "mapmyfitness" }); + this.#fetchFn = fetchFn; } async #get<T>(path: string): Promise<T> { @@ -202,7 +202,7 @@ export class MapMyFitnessProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "mapmyfitness" }); + this.#fetchFn = createProviderRateLimitFetch("mapmyfitness", fetchFn); } validate(): string | null { diff --git a/src/providers/oauth-providers.test.ts b/src/providers/oauth-providers.test.ts index 4d311a6f4c..dacc0911ce 100644 --- a/src/providers/oauth-providers.test.ts +++ b/src/providers/oauth-providers.test.ts @@ -1,5 +1,6 @@ import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; import { afterEach, describe, expect, it } from "vitest"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { MapMyFitnessClient, MapMyFitnessProvider, @@ -679,7 +680,11 @@ describe("MapMyFitnessClient — error handling", () => { return new Response("Rate limit exceeded", { status: 429 }); }; - const client = new MapMyFitnessClient("token", "client-id", mockFetch); + const client = new MapMyFitnessClient( + "token", + "client-id", + createProviderRateLimitFetch("mapmyfitness", mockFetch), + ); const error = await client .getWorkouts("-", "2026-03-01T00:00:00Z", "2026-03-02T00:00:00Z") .catch((caughtError: unknown) => caughtError); diff --git a/src/providers/oura/provider.ts b/src/providers/oura/provider.ts index ebe5345522..6aacd10220 100644 --- a/src/providers/oura/provider.ts +++ b/src/providers/oura/provider.ts @@ -1,4 +1,3 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; import { z } from "zod"; import { exchangeCodeForTokens } from "../../auth/oauth.ts"; import { resolveOAuthTokens } from "../../auth/resolve-tokens.ts"; @@ -8,6 +7,7 @@ import { hasProviderActivityListSyncErrors, } from "../../db/provider-activity-sync.ts"; import { ensureProvider } from "../../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../../lib/provider-rate-limit-fetch.ts"; import type { SyncRun } from "../sync-run.ts"; import type { ProviderAuthSetup, @@ -44,7 +44,7 @@ export class OuraProvider implements WebhookProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "oura" }); + this.#fetchFn = createProviderRateLimitFetch("oura", fetchFn); } validate(): string | null { diff --git a/src/providers/peloton-sync-ext.integration.test.ts b/src/providers/peloton-sync-ext.integration.test.ts index af22ff01c4..f727c974d4 100644 --- a/src/providers/peloton-sync-ext.integration.test.ts +++ b/src/providers/peloton-sync-ext.integration.test.ts @@ -600,6 +600,7 @@ describe("pelotonAutomatedLogin", () => { `; let step = 0; + let tokenRequestBody: string | null = null; loginServer.use( http.get("https://auth.onepeloton.com/authorize", () => { @@ -639,8 +640,9 @@ describe("pelotonAutomatedLogin", () => { }, }); }), - http.post("https://auth.onepeloton.com/oauth/token", () => { + http.post("https://auth.onepeloton.com/oauth/token", async ({ request }) => { step = 6; + tokenRequestBody = await request.text(); return HttpResponse.json({ access_token: "new-access-token", refresh_token: "new-refresh-token", @@ -655,6 +657,10 @@ describe("pelotonAutomatedLogin", () => { expect(step).toBe(6); expect(tokens.accessToken).toBe("new-access-token"); expect(tokens.refreshToken).toBe("new-refresh-token"); + expect(tokenRequestBody).toContain("code_verifier="); + const verifierMatch = /(?:^|&)code_verifier=([^&]+)/.exec(tokenRequestBody ?? ""); + expect(verifierMatch?.[1]).toBeDefined(); + expect(verifierMatch?.[1]).not.toBe("undefined"); }); it("throws when injectedConfig is not found in login page", async () => { diff --git a/src/providers/peloton.test.ts b/src/providers/peloton.test.ts index 6a6b51fd82..12680d00ca 100644 --- a/src/providers/peloton.test.ts +++ b/src/providers/peloton.test.ts @@ -1,5 +1,6 @@ import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; import { beforeEach, describe, expect, it, vi } from "vitest"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { mapFitnessDiscipline, PelotonClient, @@ -1420,7 +1421,10 @@ describe("Peloton — rate-limit aware fetch wiring", () => { new Response("rate limited", { status: 429, headers: { "Retry-After": "60" } }); it("PelotonClient surfaces a 429 as a ProviderRateLimitError tagged 'peloton'", async () => { - const client = new PelotonClient("token", rateLimited429); + const client = new PelotonClient( + "token", + createProviderRateLimitFetch("peloton", rateLimited429), + ); const err = await client.getUserId().catch((caught: unknown) => caught); expect(err).toBeInstanceOf(ProviderRateLimitError); if (err instanceof ProviderRateLimitError) { @@ -1440,9 +1444,11 @@ describe("Peloton — rate-limit aware fetch wiring", () => { }); it("pelotonAutomatedLogin surfaces a 429 tagged 'peloton'", async () => { - const err = await pelotonAutomatedLogin("user@test.com", "pass", rateLimited429).catch( - (caught: unknown) => caught, - ); + const err = await pelotonAutomatedLogin( + "user@test.com", + "pass", + createProviderRateLimitFetch("peloton", rateLimited429), + ).catch((caught: unknown) => caught); expect(err).toBeInstanceOf(ProviderRateLimitError); if (err instanceof ProviderRateLimitError) { expect(err.providerId).toBe("peloton"); diff --git a/src/providers/peloton.ts b/src/providers/peloton.ts index 7d0515794c..ea3bbba883 100644 --- a/src/providers/peloton.ts +++ b/src/providers/peloton.ts @@ -1,4 +1,3 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; import type { CanonicalActivityType } from "@dofek/training/training"; import { z } from "zod"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; @@ -18,6 +17,7 @@ import { import { SOURCE_TYPE_API } from "../db/sensor-channels.ts"; import { withSyncLog } from "../db/sync-log.ts"; import { ensureProvider } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { logger } from "../logger.ts"; import type { SyncRun } from "./sync-run.ts"; import type { ProviderAuthSetup, SyncError, SyncProvider, SyncResult } from "./types.ts"; @@ -214,7 +214,7 @@ export class PelotonClient { constructor(accessToken: string, fetchFn: typeof globalThis.fetch = globalThis.fetch) { this.#accessToken = accessToken; - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "peloton" }); + this.#fetchFn = fetchFn; } async #get<T>(path: string, params?: Record<string, string>): Promise<T> { @@ -389,7 +389,6 @@ export async function pelotonAutomatedLogin( password: string, fetchFn: typeof globalThis.fetch = globalThis.fetch, ): Promise<TokenSet> { - const rateLimitFetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "peloton" }); const config = pelotonOAuthConfig(); const codeVerifier = generateCodeVerifier(); const codeChallenge = generateCodeChallenge(codeVerifier); @@ -410,14 +409,10 @@ export async function pelotonAutomatedLogin( authorizeUrl.searchParams.set("nonce", nonce); logger.info("[peloton] Initiating Auth0 login flow..."); - let { response, location } = await followRedirects( - authorizeUrl.toString(), - jar, - rateLimitFetchFn, - ); + let { response, location } = await followRedirects(authorizeUrl.toString(), jar, fetchFn); while (location) { - ({ response, location } = await followRedirects(location, jar, rateLimitFetchFn)); + ({ response, location } = await followRedirects(location, jar, fetchFn)); } // Parse injectedConfig from login page (contains state, csrf, nonce) @@ -446,7 +441,7 @@ export async function pelotonAutomatedLogin( // Step 2: POST credentials to Auth0 login endpoint logger.info("[peloton] Submitting credentials..."); const loginUrl = `${PELOTON_AUTH_DOMAIN}/usernamepassword/login`; - const { response: loginResp } = await followRedirects(loginUrl, jar, rateLimitFetchFn, { + const { response: loginResp } = await followRedirects(loginUrl, jar, fetchFn, { method: "POST", headers: { "Content-Type": "application/json", @@ -492,7 +487,7 @@ export async function pelotonAutomatedLogin( // Step 4: Submit form, then follow redirects until we find ?code= in a Location header logger.info("[peloton] Following Auth0 redirect chain..."); - let { location: redirectUrl } = await followRedirects(formAction, jar, rateLimitFetchFn, { + let { location: redirectUrl } = await followRedirects(formAction, jar, fetchFn, { method: "POST", headers: { "Content-Type": "application/x-www-form-urlencoded" }, body: new URLSearchParams(fields).toString(), @@ -502,7 +497,7 @@ export async function pelotonAutomatedLogin( while (redirectUrl && maxRedirects > 0) { // Stop before fetching the callback URL — just read the code from it if (redirectUrl.includes("code=") || redirectUrl.includes("error=")) break; - ({ location: redirectUrl } = await followRedirects(redirectUrl, jar, rateLimitFetchFn)); + ({ location: redirectUrl } = await followRedirects(redirectUrl, jar, fetchFn)); maxRedirects--; } @@ -524,7 +519,7 @@ export async function pelotonAutomatedLogin( // Step 5: Exchange code for tokens logger.info("[peloton] Exchanging authorization code for tokens..."); - return exchangeCodeForTokens(config, authCode, rateLimitFetchFn, { codeVerifier }); + return exchangeCodeForTokens(config, authCode, fetchFn, { codeVerifier }); } export class PelotonProvider implements SyncProvider { @@ -533,7 +528,7 @@ export class PelotonProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "peloton" }); + this.#fetchFn = createProviderRateLimitFetch("peloton", fetchFn); } validate(): string | null { diff --git a/src/providers/polar/client.ts b/src/providers/polar/client.ts index 987459ab7b..3cb20cbd77 100644 --- a/src/providers/polar/client.ts +++ b/src/providers/polar/client.ts @@ -1,4 +1,4 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../../lib/provider-rate-limit-fetch.ts"; import { POLAR_API_BASE } from "./oauth.ts"; import type { PolarDailyActivity, @@ -35,7 +35,7 @@ export class PolarClient { constructor(accessToken: string, fetchFn: typeof globalThis.fetch = globalThis.fetch) { this.#accessToken = accessToken; - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "polar" }); + this.#fetchFn = createProviderRateLimitFetch("polar", fetchFn); } async #get<TResponse>( diff --git a/src/providers/polar/provider.ts b/src/providers/polar/provider.ts index b99cd2fb26..713285df5f 100644 --- a/src/providers/polar/provider.ts +++ b/src/providers/polar/provider.ts @@ -1,4 +1,3 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; import type { TokenSet } from "../../auth/oauth.ts"; import { logger } from "../../logger.ts"; import type { SyncRun } from "../sync-run.ts"; @@ -20,7 +19,7 @@ export class PolarProvider implements WebhookProvider { readonly #webhookService: PolarWebhookService; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "polar" }); + this.#fetchFn = fetchFn; this.#webhookService = new PolarWebhookService(this.#fetchFn); } diff --git a/src/providers/polar/webhook-service.ts b/src/providers/polar/webhook-service.ts index a0a32dadbf..97f3184463 100644 --- a/src/providers/polar/webhook-service.ts +++ b/src/providers/polar/webhook-service.ts @@ -1,6 +1,6 @@ import { createHmac, timingSafeEqual } from "node:crypto"; -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; import { z } from "zod"; +import { createProviderRateLimitFetch } from "../../lib/provider-rate-limit-fetch.ts"; import type { WebhookEvent } from "../types.ts"; const POLAR_WEBHOOK_URL = "https://www.polaraccesslink.com/v3/webhooks"; @@ -22,7 +22,7 @@ export class PolarWebhookService { readonly #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "polar" }); + this.#fetchFn = createProviderRateLimitFetch("polar", fetchFn); } async registerWebhook( diff --git a/src/providers/ride-with-gps-ext.test.ts b/src/providers/ride-with-gps-ext.test.ts index 1f26349431..7ff8129339 100644 --- a/src/providers/ride-with-gps-ext.test.ts +++ b/src/providers/ride-with-gps-ext.test.ts @@ -453,7 +453,7 @@ describe("RideWithGpsClient — Content-Type header", () => { }); describe("RideWithGpsProvider — constructor stores fetchFn", () => { - it("uses the injected fetch function for API calls", () => { + it("uses the injected fetch function for API calls", async () => { const customFetch = vi.fn().mockResolvedValue(Response.json({})); const provider = new RideWithGpsProvider(customFetch); process.env.RWGPS_CLIENT_ID = "test-id"; @@ -461,7 +461,7 @@ describe("RideWithGpsProvider — constructor stores fetchFn", () => { const setup = provider.authSetup(); const getUserIdentity = setup.getUserIdentity; expect(getUserIdentity).toBeDefined(); - getUserIdentity?.("tok").catch((_error: unknown) => {}); + await getUserIdentity?.("tok").catch((_error: unknown) => {}); expect(customFetch).toHaveBeenCalled(); }); }); diff --git a/src/providers/ride-with-gps.test.ts b/src/providers/ride-with-gps.test.ts index b84c0d4510..a4dfb46455 100644 --- a/src/providers/ride-with-gps.test.ts +++ b/src/providers/ride-with-gps.test.ts @@ -1,5 +1,6 @@ import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; import { afterEach, describe, expect, it } from "vitest"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { mapActivityType, parseTrackPoints, @@ -222,7 +223,10 @@ describe("RideWithGps — rate-limit aware fetch wiring", () => { }); it("RideWithGpsClient surfaces a 429 as a ProviderRateLimitError tagged 'ride-with-gps'", async () => { - const client = new RideWithGpsClient("access-token", rateLimitedFetch); + const client = new RideWithGpsClient( + "access-token", + createProviderRateLimitFetch("ride-with-gps", rateLimitedFetch), + ); const err = await client.sync("2024-01-01T00:00:00Z").catch((caught: unknown) => caught); expect(err).toBeInstanceOf(ProviderRateLimitError); diff --git a/src/providers/ride-with-gps.ts b/src/providers/ride-with-gps.ts index 55d9b93e1d..82052849c9 100644 --- a/src/providers/ride-with-gps.ts +++ b/src/providers/ride-with-gps.ts @@ -1,4 +1,3 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; import { isIndoorCycling } from "@dofek/training/endurance-types"; import { type CanonicalActivityType, @@ -21,6 +20,7 @@ import { userSettings } from "../db/schema.ts"; import { SOURCE_TYPE_API } from "../db/sensor-channels.ts"; import { getTokenUserId } from "../db/token-user-context.ts"; import { ensureProvider } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { SyncRun } from "./sync-run.ts"; import type { ProviderAuthSetup, @@ -245,7 +245,7 @@ export class RideWithGpsClient { constructor(accessToken: string, fetchFn: typeof globalThis.fetch = globalThis.fetch) { this.#accessToken = accessToken; - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "ride-with-gps" }); + this.#fetchFn = fetchFn; } async #get<T>(path: string, params?: Record<string, string>): Promise<T> { @@ -334,7 +334,7 @@ export class RideWithGpsProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "ride-with-gps" }); + this.#fetchFn = createProviderRateLimitFetch("ride-with-gps", fetchFn); } validate(): string | null { diff --git a/src/providers/strava-sync.integration.test.ts b/src/providers/strava-sync.integration.test.ts index db3cd80f19..51925e2eed 100644 --- a/src/providers/strava-sync.integration.test.ts +++ b/src/providers/strava-sync.integration.test.ts @@ -179,7 +179,7 @@ describe("StravaProvider.sync() (integration)", () => { server.use(...stravaHandlers(activities)); - const provider = new StravaProvider(globalThis.fetch, 0); + const provider = new StravaProvider(globalThis.fetch); const since = new Date("2026-02-01T00:00:00Z"); const result = await provider.sync( new SyncRun({ @@ -230,7 +230,7 @@ describe("StravaProvider.sync() (integration)", () => { server.use(...stravaHandlers(activities)); - const provider = new StravaProvider(globalThis.fetch, 0); + const provider = new StravaProvider(globalThis.fetch); await provider.sync( new SyncRun({ db: ctx.db, @@ -261,7 +261,7 @@ describe("StravaProvider.sync() (integration)", () => { server.use(...stravaHandlers([])); - const provider = new StravaProvider(globalThis.fetch, 0); + const provider = new StravaProvider(globalThis.fetch); await provider.sync( new SyncRun({ db: ctx.db, @@ -287,7 +287,7 @@ describe("StravaProvider.sync() (integration)", () => { server.use(...stravaHandlers(activities, { streamsError: true })); - const provider = new StravaProvider(globalThis.fetch, 0); + const provider = new StravaProvider(globalThis.fetch); const result = await provider.sync( new SyncRun({ db: ctx.db, @@ -321,7 +321,7 @@ describe("StravaProvider.sync() (integration)", () => { server.use(...stravaHandlers(activities, { rateLimited: true })); - const provider = new StravaProvider(globalThis.fetch, 0); + const provider = new StravaProvider(globalThis.fetch); const result = await provider.sync( new SyncRun({ db: ctx.db, @@ -340,7 +340,7 @@ describe("StravaProvider.sync() (integration)", () => { const { oauthToken } = await import("../db/schema.ts"); await ctx.db.delete(oauthToken).where(eq(oauthToken.providerId, "strava")); - const provider = new StravaProvider(globalThis.fetch, 0); + const provider = new StravaProvider(globalThis.fetch); const result = await provider.sync( new SyncRun({ db: ctx.db, diff --git a/src/providers/strava.test.ts b/src/providers/strava.test.ts index cdbe93a4c5..2ba810d00f 100644 --- a/src/providers/strava.test.ts +++ b/src/providers/strava.test.ts @@ -576,7 +576,7 @@ describe("StravaClient.getActivity", () => { }); }; - const client = new StravaClient("token", mockFetch, 0); + const client = new StravaClient("token", mockFetch); const result = await client.getActivity(12345678); expect(result.device_name).toBe("Garmin Edge 530"); expect(result.id).toBe(12345678); @@ -587,7 +587,7 @@ describe("StravaClient.getActivity", () => { return Response.json(sampleActivity); }; - const client = new StravaClient("token", mockFetch, 0); + const client = new StravaClient("token", mockFetch); const result = await client.getActivity(12345678); expect(result.device_name).toBeUndefined(); }); @@ -599,7 +599,7 @@ describe("StravaClient — error handling", () => { return new Response("Rate Limit Exceeded", { status: 429 }); }; - const client = new StravaClient("token", mockFetch, 0); + const client = new StravaClient("token", mockFetch); const err = await client.getActivities(0).catch((e: unknown) => e); expect(err).toBeInstanceOf(StravaRateLimitError); expect(err).toBeInstanceOf(ProviderRateLimitError); @@ -607,7 +607,7 @@ describe("StravaClient — error handling", () => { }); it("parses an HTTP-date Retry-After header into seconds", async () => { - vi.useFakeTimers(); + vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(new Date("2024-01-15T00:00:00Z")); const retryAt = new Date("2024-01-15T00:01:00Z").toUTCString(); const mockFetch: typeof globalThis.fetch = async (): Promise<Response> => { @@ -617,7 +617,7 @@ describe("StravaClient — error handling", () => { }); }; - const client = new StravaClient("token", mockFetch, 0); + const client = new StravaClient("token", mockFetch); const err = await client.getActivities(0).catch((caughtError: unknown) => caughtError); expect(err).toBeInstanceOf(StravaRateLimitError); expect(err).toHaveProperty("retryAfterSeconds", 60); @@ -629,7 +629,7 @@ describe("StravaClient — error handling", () => { return new Response("Server Error", { status: 500 }); }; - const client = new StravaClient("token", mockFetch, 0); + const client = new StravaClient("token", mockFetch); await expect(client.getActivities(0)).rejects.toThrow("Strava API error (500): Server Error"); }); @@ -641,7 +641,7 @@ describe("StravaClient — error handling", () => { }); }; - const client = new StravaClient("token", mockFetch, 0); + const client = new StravaClient("token", mockFetch); const err = await client.getActivities(0).catch((e: unknown) => e); expect(err).toBeInstanceOf(StravaNotFoundError); expect(err).toHaveProperty("message", expect.stringContaining("/athlete/activities")); @@ -655,7 +655,7 @@ describe("StravaClient — error handling", () => { }); }; - const client = new StravaClient("token", mockFetch, 0); + const client = new StravaClient("token", mockFetch); const err = await client.getActivities(0).catch((e: unknown) => e); expect(err).toBeInstanceOf(StravaNotFoundError); expect(err).toHaveProperty("message", expect.stringContaining("/athlete/activities")); @@ -669,7 +669,7 @@ describe("StravaClient — error handling", () => { }); }; - const client = new StravaClient("token", mockFetch, 0); + const client = new StravaClient("token", mockFetch); const err = await client.getActivities(0).catch((e: unknown) => e); expect(err).toBeInstanceOf(StravaUnauthorizedError); expect(err).toHaveProperty("message", expect.stringContaining("unauthorized (401)")); @@ -680,7 +680,7 @@ describe("StravaClient — error handling", () => { return new Response("Forbidden", { status: 403 }); }; - const client = new StravaClient("token", mockFetch, 0); + const client = new StravaClient("token", mockFetch); const err = await client.getActivities(0).catch((e: unknown) => e); expect(err).toBeInstanceOf(StravaUnauthorizedError); expect(err).toHaveProperty("message", expect.stringContaining("unauthorized (403)")); @@ -694,7 +694,7 @@ describe("StravaClient — error handling", () => { }); }; - const client = new StravaClient("token", mockFetch, 0); + const client = new StravaClient("token", mockFetch); await expect(client.getActivities(0)).rejects.toThrow( 'Strava API error (500): {"message":"bad request"}', ); @@ -708,7 +708,7 @@ describe("StravaClient — error handling", () => { }); }; - const client = new StravaClient("token", mockFetch, 0); + const client = new StravaClient("token", mockFetch); await expect(client.getActivities(0)).rejects.toThrow( "Strava API error (500): (HTML error page)", ); @@ -720,7 +720,7 @@ describe("StravaClient — error handling", () => { return new Response(longText, { status: 500 }); }; - const client = new StravaClient("token", mockFetch, 0); + const client = new StravaClient("token", mockFetch); await expect(client.getActivities(0)).rejects.toThrow( `Strava API error (500): ${"x".repeat(200)}…`, ); @@ -728,41 +728,8 @@ describe("StravaClient — error handling", () => { }); describe("StravaClient — request throttling", () => { - beforeEach(() => { - vi.useFakeTimers(); - }); - - afterEach(() => { - vi.useRealTimers(); - }); - - it("enforces minimum delay between consecutive API requests", async () => { - const callTimestamps: number[] = []; - const mockFetch: typeof globalThis.fetch = async (): Promise<Response> => { - callTimestamps.push(Date.now()); - return Response.json([]); - }; - - const client = new StravaClient("token", mockFetch); - - // First request — should go immediately - const p1 = client.getActivities(0); - await vi.advanceTimersByTimeAsync(0); - await p1; - - // Second request — should be delayed by the throttle interval - const p2 = client.getActivities(0); - // Advance past the throttle delay - await vi.advanceTimersByTimeAsync(10_000); - await p2; - - expect(callTimestamps).toHaveLength(2); - const first = callTimestamps[0] ?? 0; - const second = callTimestamps[1] ?? 0; - expect(second - first).toBeGreaterThanOrEqual(10_000); - }); - it("does not delay the first request", async () => { + vi.useFakeTimers(); const callTimestamps: number[] = []; const mockFetch: typeof globalThis.fetch = async (): Promise<Response> => { callTimestamps.push(Date.now()); @@ -774,8 +741,8 @@ describe("StravaClient — request throttling", () => { await vi.advanceTimersByTimeAsync(0); await pendingRequest; - // First call should happen at time 0 (no throttle delay) expect(callTimestamps).toHaveLength(1); + vi.useRealTimers(); }); }); @@ -939,7 +906,7 @@ describe("StravaProvider.syncWebhookEvent", () => { }); it("returns immediately for non-activity objectType", async () => { - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const mockDb = { select: vi.fn(), insert: vi.fn(), @@ -965,7 +932,7 @@ describe("StravaProvider.syncWebhookEvent", () => { }); it("returns immediately when objectId is missing", async () => { - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const mockDb = { select: vi.fn(), insert: vi.fn(), @@ -988,7 +955,7 @@ describe("StravaProvider.syncWebhookEvent", () => { }); it("handles delete events by marking activity provider-absent", async () => { - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const mockDb = { select: vi.fn(), @@ -1020,7 +987,7 @@ describe("StravaProvider.syncWebhookEvent", () => { }); it("handles delete event when activity not found", async () => { - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const mockDb = { select: vi.fn(), @@ -1050,7 +1017,7 @@ describe("StravaProvider.syncWebhookEvent", () => { }); it("returns error when token resolution fails", async () => { - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const mockDb = { select: makeStravaSelectMock(null), insert: vi.fn(), @@ -1115,7 +1082,7 @@ describe("StravaProvider.syncWebhookEvent", () => { execute: vi.fn(), }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.syncWebhookEvent( mockDb, { @@ -1187,7 +1154,7 @@ describe("StravaProvider.syncWebhookEvent", () => { execute: vi.fn(), }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.syncWebhookEvent( mockDb, { @@ -1230,7 +1197,7 @@ describe("StravaProvider.syncWebhookEvent", () => { execute: vi.fn(), }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.syncWebhookEvent( mockDb, { @@ -1269,7 +1236,7 @@ describe("StravaProvider.syncWebhookEvent", () => { execute: vi.fn(), }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.syncWebhookEvent( mockDb, { @@ -1301,7 +1268,7 @@ describe("StravaProvider.registerWebhook", () => { it("throws when STRAVA_CLIENT_ID is missing", async () => { delete process.env.STRAVA_CLIENT_ID; delete process.env.STRAVA_CLIENT_SECRET; - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); await expect( provider.registerWebhook("https://example.com/webhook", "verify-token"), ).rejects.toThrow("STRAVA_CLIENT_ID"); @@ -1313,7 +1280,7 @@ describe("StravaProvider.registerWebhook", () => { const mockFetch: typeof globalThis.fetch = async (): Promise<Response> => { return new Response("Conflict", { status: 409 }); }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); await expect( provider.registerWebhook("https://example.com/webhook", "verify-token"), ).rejects.toThrow("Strava webhook registration failed (409)"); @@ -1325,7 +1292,7 @@ describe("StravaProvider.registerWebhook", () => { const mockFetch: typeof globalThis.fetch = async (): Promise<Response> => { return Response.json({ id: 42 }); }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.registerWebhook("https://example.com/webhook", "verify-token"); expect(result.subscriptionId).toBe("42"); }); @@ -1342,7 +1309,7 @@ describe("StravaProvider.unregisterWebhook", () => { delete process.env.STRAVA_CLIENT_ID; delete process.env.STRAVA_CLIENT_SECRET; const mockFetch = vi.fn(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); await provider.unregisterWebhook("42"); expect(mockFetch).not.toHaveBeenCalled(); }); @@ -1353,7 +1320,7 @@ describe("StravaProvider.unregisterWebhook", () => { const mockFetch: typeof globalThis.fetch = async (): Promise<Response> => { return new Response(null, { status: 200 }); }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); // Should not throw await provider.unregisterWebhook("42"); }); @@ -1364,7 +1331,7 @@ describe("StravaProvider.unregisterWebhook", () => { const mockFetch: typeof globalThis.fetch = async (): Promise<Response> => { return new Response("Not Found", { status: 404 }); }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); // Should not throw await provider.unregisterWebhook("42"); }); @@ -1375,7 +1342,7 @@ describe("StravaProvider.unregisterWebhook", () => { const mockFetch: typeof globalThis.fetch = async (): Promise<Response> => { return new Response("Server Error", { status: 500 }); }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); // Should not throw, just logs warning await provider.unregisterWebhook("42"); }); @@ -1387,7 +1354,7 @@ describe("StravaProvider.unregisterWebhook", () => { describe("StravaProvider — precise webhook string/object assertions", () => { it("parseWebhookPayload maps all three Strava aspect_types correctly", async () => { - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); for (const [aspect, expected] of [ ["create", "create"], @@ -1405,7 +1372,7 @@ describe("StravaProvider — precise webhook string/object assertions", () => { }); it("parseWebhookPayload converts owner_id number to string", async () => { - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const events = provider.parseWebhookPayload({ aspect_type: "create", object_type: "activity", @@ -1416,7 +1383,7 @@ describe("StravaProvider — precise webhook string/object assertions", () => { }); it("parseWebhookPayload converts object_id number to string", async () => { - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const events = provider.parseWebhookPayload({ aspect_type: "create", object_type: "activity", @@ -1427,7 +1394,7 @@ describe("StravaProvider — precise webhook string/object assertions", () => { }); it("parseWebhookPayload treats object_id=0 as falsy (undefined)", async () => { - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const events = provider.parseWebhookPayload({ aspect_type: "create", object_type: "activity", @@ -1438,7 +1405,7 @@ describe("StravaProvider — precise webhook string/object assertions", () => { }); it("handleValidationChallenge echoes back the exact challenge string", async () => { - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const result = provider.handleValidationChallenge( { "hub.mode": "subscribe", @@ -1451,7 +1418,7 @@ describe("StravaProvider — precise webhook string/object assertions", () => { }); it("handleValidationChallenge compares token exactly (not substring)", async () => { - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); // Partial match should fail const result = provider.handleValidationChallenge( { "hub.mode": "subscribe", "hub.challenge": "abc", "hub.verify_token": "tok" }, @@ -1471,7 +1438,7 @@ describe("StravaProvider — precise webhook string/object assertions", () => { return Response.json({ id: 1 }); }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); await provider.registerWebhook("https://example.com/callback", "my-verify-token"); expect(capturedBody?.get("client_id")).toBe("my-client-id"); @@ -1493,7 +1460,7 @@ describe("StravaProvider — precise webhook string/object assertions", () => { return Response.json({ id: 1 }); }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); await provider.registerWebhook("https://example.com/cb", "tok"); expect(capturedUrl).toBe("https://www.strava.com/api/v3/push_subscriptions"); @@ -1511,7 +1478,7 @@ describe("StravaProvider — precise webhook string/object assertions", () => { return Response.json({ id: 1 }); }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); await provider.registerWebhook("https://example.com/cb", "tok"); expect(capturedHeaders).toEqual( expect.objectContaining({ "Content-Type": "application/x-www-form-urlencoded" }), @@ -1531,7 +1498,7 @@ describe("StravaProvider — precise webhook string/object assertions", () => { return new Response(null, { status: 200 }); }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); await provider.unregisterWebhook("sub-42"); const parsed = new URL(capturedUrl); @@ -1543,7 +1510,7 @@ describe("StravaProvider — precise webhook string/object assertions", () => { }); it("syncWebhookEvent returns provider as 'strava' for all paths", async () => { - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const mockDb = { select: vi.fn(), insert: vi.fn(), @@ -1570,7 +1537,7 @@ describe("StravaProvider — precise webhook string/object assertions", () => { execute: vi.fn(), }; - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const result = await provider.syncWebhookEvent( mockDb, { @@ -1594,7 +1561,7 @@ describe("StravaProvider — precise webhook string/object assertions", () => { }); it("syncWebhookEvent falls back to token user context when options.userId is missing", async () => { - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const mockDb = { select: vi.fn(), insert: vi.fn(), delete: vi.fn(), execute: vi.fn() }; const result = await provider.syncWebhookEvent(mockDb, { @@ -1694,7 +1661,7 @@ describe("StravaClient.getActivityStreams", () => { return Response.json(apiResponse); }; - const client = new StravaClient("token", mockFetch, 0); + const client = new StravaClient("token", mockFetch); const streams = await client.getActivityStreams(12345); // Verify all 10 STREAM_KEYS are present in the result @@ -1739,7 +1706,7 @@ describe("StravaClient.getActivityStreams", () => { return Response.json(apiResponse); }; - const client = new StravaClient("token", mockFetch, 0); + const client = new StravaClient("token", mockFetch); const streams = await client.getActivityStreams(1); expect(streams.time).toBeDefined(); @@ -1756,7 +1723,7 @@ describe("StravaClient.getActivityStreams", () => { return Response.json([]); }; - const client = new StravaClient("token", mockFetch, 0); + const client = new StravaClient("token", mockFetch); await client.getActivityStreams(99999); // Verify base URL is the Strava API @@ -1795,7 +1762,7 @@ describe("StravaClient — API base URL", () => { return Response.json([]); }; - const client = new StravaClient("token", mockFetch, 0); + const client = new StravaClient("token", mockFetch); await client.getActivities(0); expect(capturedUrl).toMatch(/^https:\/\/www\.strava\.com\/api\/v3\//); @@ -1810,7 +1777,7 @@ describe("StravaClient — API base URL", () => { return Response.json(sampleActivity); }; - const client = new StravaClient("token", mockFetch, 0); + const client = new StravaClient("token", mockFetch); await client.getActivity(42); expect(capturedUrl).toBe("https://www.strava.com/api/v3/activities/42"); }); @@ -1825,7 +1792,7 @@ describe("StravaClient — API base URL", () => { return Response.json([]); }; - const client = new StravaClient("my-secret-token", mockFetch, 0); + const client = new StravaClient("my-secret-token", mockFetch); await client.getActivities(0); expect(capturedHeaders).toEqual({ Authorization: "Bearer my-secret-token" }); }); @@ -1862,7 +1829,7 @@ describe("StravaClient", () => { ]), ); - const client = new StravaClient("test-token", mockFetch, 0); + const client = new StravaClient("test-token", mockFetch); const result = await client.getActivities(1000, 2, 50); expect(mockFetch).toHaveBeenCalledOnce(); @@ -1912,7 +1879,7 @@ describe("StravaClient", () => { ]), ); - const client = new StravaClient("test-token", mockFetch, 0); + const client = new StravaClient("test-token", mockFetch); const streams = await client.getActivityStreams(12345); const calledUrl = String(mockFetch.mock.calls[0]?.[0]); @@ -1940,7 +1907,7 @@ describe("StravaProvider.sync", () => { process.env.STRAVA_CLIENT_SECRET = "secret"; const mockFetch = vi.fn(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); // Mock db with loadTokens returning null const mockDb = { @@ -1983,7 +1950,7 @@ describe("StravaProvider.sync", () => { return Response.json([]); }); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); // Provide tokens const futureDate = new Date("2099-01-01"); @@ -2149,7 +2116,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), @@ -2203,7 +2170,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), ); @@ -2247,7 +2214,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: sinceDate }) }), @@ -2276,7 +2243,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: sinceDate }) }), @@ -2312,7 +2279,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), @@ -2367,7 +2334,7 @@ describe("StravaProvider.sync — additional coverage", () => { execute: vi.fn().mockResolvedValue([]), }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), ); @@ -2397,7 +2364,7 @@ describe("StravaProvider.sync — additional coverage", () => { const mockDb = createMockDb(); const onProgress = vi.fn(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); await provider.sync( new SyncRun({ @@ -2425,7 +2392,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), @@ -2455,7 +2422,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), @@ -2477,7 +2444,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const before = Date.now(); const result = await provider.sync( @@ -2518,7 +2485,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb([EXPIRED_TOKEN]); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), @@ -2542,7 +2509,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb([VALID_TOKEN]); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), @@ -2597,7 +2564,7 @@ describe("StravaProvider.sync — additional coverage", () => { execute: vi.fn().mockResolvedValue([]), }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), ); @@ -2624,7 +2591,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), ); @@ -2652,7 +2619,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), ); @@ -2679,7 +2646,7 @@ describe("StravaProvider.sync — additional coverage", () => { providerActivityAbsenceMocks.upsertProviderActivity.mockResolvedValueOnce(undefined); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), ); @@ -2712,7 +2679,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), @@ -2769,7 +2736,7 @@ describe("StravaProvider.sync — additional coverage", () => { execute: vi.fn().mockResolvedValue([]), }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), ); @@ -2844,7 +2811,7 @@ describe("StravaProvider.sync — additional coverage", () => { execute: vi.fn().mockResolvedValue([]), }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), ); @@ -2872,7 +2839,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), ); @@ -2900,7 +2867,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), ); @@ -2960,7 +2927,7 @@ describe("StravaProvider.sync — additional coverage", () => { execute: vi.fn().mockResolvedValue([]), }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), ); diff --git a/src/providers/strava.ts b/src/providers/strava.ts index c8ea875f95..3b0218b400 100644 --- a/src/providers/strava.ts +++ b/src/providers/strava.ts @@ -1,8 +1,4 @@ -import { - createRateLimitAwareFetch, - ProviderRateLimitError, - parseRetryAfterHeader, -} from "@dofek/provider-http/rate-limit"; +import { ProviderRateLimitError, parseRetryAfterHeader } from "@dofek/provider-http/rate-limit"; import { isIndoorCycling } from "@dofek/training/endurance-types"; import { type CanonicalActivityType, @@ -28,6 +24,7 @@ import { import { activity } from "../db/schema.ts"; import { SOURCE_TYPE_API } from "../db/sensor-channels.ts"; import { getTokenUserId } from "../db/token-user-context.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { logger } from "../logger.ts"; import { ProviderAuthorizationFailedError } from "./auth-errors.ts"; import type { SyncRun } from "./sync-run.ts"; @@ -260,34 +257,15 @@ export const STRAVA_THROTTLE_MS = 10_000; export class StravaClient { #accessToken: string; #fetchFn: typeof globalThis.fetch; - #lastRequestTime = 0; - #throttleMs: number; - - constructor( - accessToken: string, - fetchFn: typeof globalThis.fetch = globalThis.fetch, - throttleMs = STRAVA_THROTTLE_MS, - ) { + + constructor(accessToken: string, fetchFn: typeof globalThis.fetch = globalThis.fetch) { this.#accessToken = accessToken; - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { - providerId: "strava", + this.#fetchFn = createProviderRateLimitFetch("strava", fetchFn, { createRateLimitError: createStravaRateLimitError, }); - this.#throttleMs = throttleMs; - } - - async #throttle(): Promise<void> { - if (this.#throttleMs <= 0) return; - const now = Date.now(); - const elapsed = now - this.#lastRequestTime; - if (this.#lastRequestTime > 0 && elapsed < this.#throttleMs) { - await new Promise((resolve) => setTimeout(resolve, this.#throttleMs - elapsed)); - } - this.#lastRequestTime = Date.now(); } async #get<T>(path: string, params?: Record<string, string>): Promise<T> { - await this.#throttle(); const url = new URL(path, STRAVA_API_BASE); if (params) { for (const [key, value] of Object.entries(params)) { @@ -439,17 +417,11 @@ export class StravaProvider implements WebhookProvider { readonly name = "Strava"; readonly webhookScope = "app" as const; #fetchFn: typeof globalThis.fetch; - #throttleMs: number; - constructor( - fetchFn: typeof globalThis.fetch = globalThis.fetch, - throttleMs = STRAVA_THROTTLE_MS, - ) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { - providerId: "strava", + constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { + this.#fetchFn = createProviderRateLimitFetch("strava", fetchFn, { createRateLimitError: createStravaRateLimitError, }); - this.#throttleMs = throttleMs; } validate(): string | null { @@ -654,7 +626,7 @@ export class StravaProvider implements WebhookProvider { return { provider: this.id, recordsSynced, errors, duration: Date.now() - start }; } - const client = new StravaClient(tokens.accessToken, this.#fetchFn, this.#throttleMs); + const client = new StravaClient(tokens.accessToken, this.#fetchFn); // Fetch the single activity detail (1 API call) const detail = await client.getActivity(activityExternalId); @@ -744,7 +716,7 @@ export class StravaProvider implements WebhookProvider { return { provider: this.id, recordsSynced, errors, duration: Date.now() - start }; } - const client = new StravaClient(tokens.accessToken, this.#fetchFn, this.#throttleMs); + const client = new StravaClient(tokens.accessToken, this.#fetchFn); const since = window.since; const syncWindowEnd = window.until; diff --git a/src/providers/suunto.ts b/src/providers/suunto.ts index 8ff6d19f51..b3d319cd24 100644 --- a/src/providers/suunto.ts +++ b/src/providers/suunto.ts @@ -1,5 +1,4 @@ import { createHmac, timingSafeEqual } from "node:crypto"; -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; import type { CanonicalActivityType } from "@dofek/training/training"; import { z } from "zod"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; @@ -16,6 +15,7 @@ import { withSyncLog } from "../db/sync-log.ts"; import { ensureProvider } from "../db/tokens.ts"; import { parseFitFile } from "../fit/parser.ts"; import { fitRecordsToSensorSamples } from "../fit/records.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { logger } from "../logger.ts"; import type { SyncRun } from "./sync-run.ts"; import type { @@ -175,7 +175,7 @@ export class SuuntoProvider implements WebhookProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "suunto" }); + this.#fetchFn = createProviderRateLimitFetch("suunto", fetchFn); } validate(): string | null { diff --git a/src/providers/trainerroad.ts b/src/providers/trainerroad.ts index 27cefa5cb7..0c4e5c01b3 100644 --- a/src/providers/trainerroad.ts +++ b/src/providers/trainerroad.ts @@ -1,4 +1,3 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; import { TrainerRoadClient } from "trainerroad-client/client"; import { parseTrainerRoadActivity } from "trainerroad-client/parsing"; import { @@ -7,6 +6,7 @@ import { } from "../db/provider-activity-sync.ts"; import { withSyncLog } from "../db/sync-log.ts"; import { ensureProvider, loadTokens } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { ProviderSessionExpiredError, ProviderStoredIdentityMissingError } from "./auth-errors.ts"; import type { SyncRun } from "./sync-run.ts"; import type { ProviderAuthSetup, SyncError, SyncProvider, SyncResult } from "./types.ts"; @@ -31,7 +31,7 @@ export class TrainerRoadProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "trainerroad" }); + this.#fetchFn = createProviderRateLimitFetch("trainerroad", fetchFn); } validate(): string | null { diff --git a/src/providers/ultrahuman.ts b/src/providers/ultrahuman.ts index 8f1292de87..467bce3b8b 100644 --- a/src/providers/ultrahuman.ts +++ b/src/providers/ultrahuman.ts @@ -1,7 +1,8 @@ -import { createRateLimitAwareFetch, ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; +import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; import { dailyMetrics, sleepSession } from "../db/schema.ts"; import { withSyncLog } from "../db/sync-log.ts"; import { ensureProvider, loadTokens } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { SyncRun } from "./sync-run.ts"; import type { SyncError, SyncProvider, SyncResult } from "./types.ts"; @@ -114,7 +115,7 @@ export class UltrahumanClient { constructor(token: string, email: string, fetchFn: typeof globalThis.fetch = globalThis.fetch) { this.#token = token; this.#email = email; - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "ultrahuman" }); + this.#fetchFn = createProviderRateLimitFetch("ultrahuman", fetchFn); } async getDailyMetrics(date: string): Promise<UltrahumanDailyMetricsResponse> { @@ -153,7 +154,7 @@ export class UltrahumanProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "ultrahuman" }); + this.#fetchFn = createProviderRateLimitFetch("ultrahuman", fetchFn); } validate(): string | null { diff --git a/src/providers/velohero.ts b/src/providers/velohero.ts index 5ad14740e3..9080f43ea4 100644 --- a/src/providers/velohero.ts +++ b/src/providers/velohero.ts @@ -1,4 +1,3 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; import { VeloHeroClient } from "velohero-client/client"; import { parseVeloHeroWorkout } from "velohero-client/parsing"; import { @@ -7,6 +6,7 @@ import { } from "../db/provider-activity-sync.ts"; import { withSyncLog } from "../db/sync-log.ts"; import { ensureProvider, loadTokens } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { logger } from "../logger.ts"; import { ProviderSessionExpiredError } from "./auth-errors.ts"; import type { SyncRun } from "./sync-run.ts"; @@ -32,7 +32,7 @@ export class VeloHeroProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "velohero" }); + this.#fetchFn = createProviderRateLimitFetch("velohero", fetchFn); } validate(): string | null { diff --git a/src/providers/wahoo/provider.ts b/src/providers/wahoo/provider.ts index 43da55a7ec..47ffe334d3 100644 --- a/src/providers/wahoo/provider.ts +++ b/src/providers/wahoo/provider.ts @@ -1,4 +1,3 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; import type { OAuthConfig, TokenSet } from "../../auth/oauth.ts"; import { exchangeCodeForTokens, @@ -8,6 +7,7 @@ import { import { resolveOAuthTokens } from "../../auth/resolve-tokens.ts"; import type { SyncDatabase } from "../../db/index.ts"; import { finishProviderActivityListSync } from "../../db/provider-activity-sync.ts"; +import { createProviderRateLimitFetch } from "../../lib/provider-rate-limit-fetch.ts"; import { logger } from "../../logger.ts"; import { AccessTokenExpiredError } from "../auth-errors.ts"; import type { SyncRun } from "../sync-run.ts"; @@ -51,7 +51,7 @@ export class WahooProvider implements WebhookProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "wahoo" }); + this.#fetchFn = createProviderRateLimitFetch("wahoo", fetchFn); } validate(): string | null { diff --git a/src/providers/webhook.test.ts b/src/providers/webhook.test.ts index b5a50bef89..9d3ef19779 100644 --- a/src/providers/webhook.test.ts +++ b/src/providers/webhook.test.ts @@ -5,14 +5,14 @@ import { isWebhookProvider } from "./types.ts"; describe("StravaProvider webhook methods", () => { it("is detected as a WebhookProvider", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); expect(isWebhookProvider(provider)).toBe(true); expect(provider.webhookScope).toBe("app"); }); it("parseWebhookPayload extracts activity create event", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const events = provider.parseWebhookPayload({ aspect_type: "create", @@ -34,7 +34,7 @@ describe("StravaProvider webhook methods", () => { it("parseWebhookPayload handles delete event", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const events = provider.parseWebhookPayload({ aspect_type: "delete", @@ -49,7 +49,7 @@ describe("StravaProvider webhook methods", () => { it("parseWebhookPayload returns empty for invalid payload", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); expect(provider.parseWebhookPayload({})).toHaveLength(0); expect(provider.parseWebhookPayload("bad")).toHaveLength(0); @@ -58,7 +58,7 @@ describe("StravaProvider webhook methods", () => { it("handleValidationChallenge responds to valid challenge", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const result = provider.handleValidationChallenge( { "hub.mode": "subscribe", "hub.challenge": "abc123", "hub.verify_token": "mytoken" }, @@ -69,7 +69,7 @@ describe("StravaProvider webhook methods", () => { it("handleValidationChallenge rejects wrong token", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const result = provider.handleValidationChallenge( { "hub.mode": "subscribe", "hub.challenge": "abc123", "hub.verify_token": "wrong" }, @@ -80,7 +80,7 @@ describe("StravaProvider webhook methods", () => { it("verifyWebhookSignature always returns true (Strava trusts registered URLs)", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); expect(provider.verifyWebhookSignature(Buffer.from(""), {}, "")).toBe(true); }); }); @@ -525,7 +525,7 @@ describe("StravaProvider register/unregister", () => { async () => new Response(JSON.stringify({ id: 12345 }), { status: 200 }), ); const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const original = { ...process.env }; process.env.STRAVA_CLIENT_ID = "test-client-id"; @@ -544,7 +544,7 @@ describe("StravaProvider register/unregister", () => { it("registerWebhook throws without client credentials", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const original = { ...process.env }; delete process.env.STRAVA_CLIENT_ID; @@ -561,7 +561,7 @@ describe("StravaProvider register/unregister", () => { it("registerWebhook throws on non-ok response", async () => { const mockFetch = vi.fn(async () => new Response("error", { status: 403 })); const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const original = { ...process.env }; process.env.STRAVA_CLIENT_ID = "id"; @@ -578,7 +578,7 @@ describe("StravaProvider register/unregister", () => { it("unregisterWebhook sends DELETE to Strava API", async () => { const mockFetch = vi.fn(async () => new Response(null, { status: 200 })); const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const original = { ...process.env }; process.env.STRAVA_CLIENT_ID = "id"; @@ -597,7 +597,7 @@ describe("StravaProvider register/unregister", () => { it("unregisterWebhook is a no-op without credentials", async () => { const mockFetch = vi.fn(async () => new Response()); const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const original = { ...process.env }; delete process.env.STRAVA_CLIENT_ID; @@ -612,7 +612,7 @@ describe("StravaProvider register/unregister", () => { it("parseWebhookPayload handles update event", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const events = provider.parseWebhookPayload({ aspect_type: "update", @@ -864,7 +864,7 @@ describe("CorosProvider register/unregister", () => { describe("StravaProvider webhook — precise assertions", () => { it("parseWebhookPayload returns exact event structure for create", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const events = provider.parseWebhookPayload({ aspect_type: "create", @@ -887,7 +887,7 @@ describe("StravaProvider webhook — precise assertions", () => { it("parseWebhookPayload defaults to update when aspect_type is unrecognized", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const events = provider.parseWebhookPayload({ aspect_type: "unknown_aspect", @@ -901,7 +901,7 @@ describe("StravaProvider webhook — precise assertions", () => { it("parseWebhookPayload defaults to update when aspect_type is missing", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const events = provider.parseWebhookPayload({ object_type: "activity", @@ -914,7 +914,7 @@ describe("StravaProvider webhook — precise assertions", () => { it("parseWebhookPayload returns objectId as undefined when object_id is missing", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const events = provider.parseWebhookPayload({ aspect_type: "create", @@ -928,7 +928,7 @@ describe("StravaProvider webhook — precise assertions", () => { it("parseWebhookPayload passes through object_type as-is", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const events = provider.parseWebhookPayload({ aspect_type: "create", @@ -942,7 +942,7 @@ describe("StravaProvider webhook — precise assertions", () => { it("handleValidationChallenge returns null when hub.mode is not subscribe", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const result = provider.handleValidationChallenge( { "hub.mode": "unsubscribe", "hub.challenge": "abc", "hub.verify_token": "mytoken" }, @@ -953,7 +953,7 @@ describe("StravaProvider webhook — precise assertions", () => { it("handleValidationChallenge returns null when hub.challenge is missing", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const result = provider.handleValidationChallenge( { "hub.mode": "subscribe", "hub.verify_token": "mytoken" }, @@ -964,19 +964,19 @@ describe("StravaProvider webhook — precise assertions", () => { it("id is exactly 'strava'", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); expect(provider.id).toBe("strava"); }); it("name is exactly 'Strava'", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); expect(provider.name).toBe("Strava"); }); it("webhookScope is exactly 'app'", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); expect(provider.webhookScope).toBe("app"); }); }); diff --git a/src/providers/wger.ts b/src/providers/wger.ts index 3723138518..c08da505c7 100644 --- a/src/providers/wger.ts +++ b/src/providers/wger.ts @@ -1,4 +1,3 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; import type { CanonicalActivityType } from "@dofek/training/training"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; import { exchangeCodeForTokens, getOAuthRedirectUri } from "../auth/oauth.ts"; @@ -12,6 +11,7 @@ import { import { SOURCE_TYPE_API } from "../db/sensor-channels.ts"; import { withSyncLog } from "../db/sync-log.ts"; import { ensureProvider } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { SyncRun } from "./sync-run.ts"; import type { ProviderAuthSetup, SyncError, SyncProvider, SyncResult } from "./types.ts"; @@ -118,7 +118,7 @@ export class WgerProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "wger" }); + this.#fetchFn = createProviderRateLimitFetch("wger", fetchFn); } validate(): string | null { diff --git a/src/providers/whoop.test.ts b/src/providers/whoop.test.ts index fe79b46de7..cdb229174d 100644 --- a/src/providers/whoop.test.ts +++ b/src/providers/whoop.test.ts @@ -20,6 +20,7 @@ import { parseWorkout, } from "./whoop/parsing.ts"; import { WhoopProvider } from "./whoop/provider.ts"; +import * as syncDailyActivityModule from "./whoop/sync-daily-activity.ts"; const { publishedMetricStreamBatches } = vi.hoisted<{ publishedMetricStreamBatches: Record<string, unknown>[][]; @@ -135,6 +136,8 @@ function makeSyncMockFetch(options: { hrError?: boolean; journalError?: boolean; cyclesError?: boolean; + cyclesRateLimit?: boolean; + strainRateLimit?: boolean; }) { const mockFetch: typeof globalThis.fetch = (input: RequestInfo | URL, _init?: RequestInit) => { const url = input.toString(); @@ -155,12 +158,23 @@ function makeSyncMockFetch(options: { // Cycles (core-details-bff/v0/cycles/details) if (url.includes("cycles/details")) { + if (options.cyclesRateLimit) { + return Promise.resolve(new Response("rate limited", { status: 429 })); + } if (options.cyclesError) { return Promise.resolve(new Response("Server error", { status: 500 })); } return Promise.resolve(Response.json(options.cycles ?? [])); } + // Strain deep dive (daily steps) + if (url.includes("deep-dive/strain")) { + if (options.strainRateLimit) { + return Promise.resolve(new Response("rate limited", { status: 429 })); + } + return Promise.resolve(Response.json({ sections: [] })); + } + // Sleep by ID (sleep-service/v1/sleep-events?activityId=...) if (url.includes("sleep-service")) { if (options.sleepError) { @@ -866,6 +880,73 @@ describe("WhoopProvider.sync() — cycles error", () => { expect(result.errors.length).toBeGreaterThan(0); expect(result.errors[0]?.message).toContain("getCycles"); }); + + it("rethrows rate limit errors from getCycles", async () => { + const { loadTokens } = await import("../db/tokens.ts"); + vi.mocked(loadTokens).mockResolvedValueOnce({ + accessToken: "tok", + refreshToken: "ref", + expiresAt: futureExpiry, + scopes: "userId:42", + }); + + const mockFetch = makeSyncMockFetch({ cyclesRateLimit: true }); + const provider = new WhoopProvider(mockFetch); + const db = makeChainableMock(); + + await expect( + provider.sync( + new SyncRun({ + db: db, + window: SyncWindow.fromSince({ since: new Date("2026-03-01") }), + userId: "00000000-0000-0000-0000-000000000001", + }), + ), + ).rejects.toMatchObject({ + providerId: "whoop", + statusCode: 429, + }); + }); + + it("rethrows collected rate limit errors from sub-syncs", async () => { + const { loadTokens } = await import("../db/tokens.ts"); + vi.mocked(loadTokens).mockResolvedValueOnce({ + accessToken: "tok", + refreshToken: "ref", + expiresAt: futureExpiry, + scopes: "userId:42", + }); + + const rateLimitError = new ProviderRateLimitError({ + message: "WHOOP API rate limit exceeded (429):", + providerId: "whoop", + statusCode: 429, + responseBody: "", + }); + vi.spyOn(syncDailyActivityModule, "syncWhoopDailyActivity").mockImplementationOnce( + async (context) => { + context.errors.push({ + message: `daily_activity: ${rateLimitError.message}`, + cause: rateLimitError, + }); + return { count: 0, rateLimited: true }; + }, + ); + + const mockFetch = makeSyncMockFetch({ cycles: [] }); + const provider = new WhoopProvider(mockFetch); + const db = makeChainableMock(); + + await expect( + provider.sync( + new SyncRun({ + db: db, + window: SyncWindow.fromSince({ since: new Date("2026-03-01") }), + userId: "00000000-0000-0000-0000-000000000001", + }), + ), + ).rejects.toBe(rateLimitError); + }); }); describe("WhoopProvider.sync() — recovery sync", () => { @@ -2890,7 +2971,12 @@ describe("WhoopProvider.sync() — strength sync", () => { ); expect(db.select).toHaveBeenCalledWith(expect.objectContaining({ id: expect.anything() })); - expect(db.select).toHaveBeenCalledTimes(1); + const exerciseSelectCalls = vi + .mocked(db.select) + .mock.calls.filter( + (call: Parameters<typeof db.select>) => isRecord(call[0]) && "id" in call[0], + ); + expect(exerciseSelectCalls).toHaveLength(1); const valuesCallArgs = getValuesCallArgs(db); const exerciseInserts = valuesCallArgs.filter( diff --git a/src/providers/whoop/provider.ts b/src/providers/whoop/provider.ts index b8e546c207..74100dcdfe 100644 --- a/src/providers/whoop/provider.ts +++ b/src/providers/whoop/provider.ts @@ -1,10 +1,10 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; import { WhoopClient } from "whoop-whoop/client"; import type { WhoopCycle } from "whoop-whoop/types"; import { z } from "zod"; import type { OAuthConfig } from "../../auth/oauth.ts"; import { exchangeCodeForTokens, getOAuthRedirectUri } from "../../auth/oauth.ts"; import { ensureProvider, loadTokens, saveTokens } from "../../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../../lib/provider-rate-limit-fetch.ts"; import { logger } from "../../logger.ts"; import { ProviderStoredIdentityMissingError } from "../auth-errors.ts"; import type { SyncRun } from "../sync-run.ts"; @@ -15,6 +15,7 @@ import type { SyncProvider, SyncResult, } from "../types.ts"; +import { findWhoopRateLimitError, isWhoopRateLimitError } from "./rate-limit.ts"; import { syncWhoopDailyActivity } from "./sync-daily-activity.ts"; import { syncWhoopJournal } from "./sync-journal.ts"; import { syncWhoopRecovery } from "./sync-recovery.ts"; @@ -34,7 +35,7 @@ export class WhoopProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "whoop" }); + this.#fetchFn = createProviderRateLimitFetch("whoop", fetchFn); } validate(): string | null { @@ -169,6 +170,9 @@ export class WhoopProvider implements SyncProvider { } logger.info(`[whoop] Fetched ${cycles.length} total cycles`); } catch (err) { + if (isWhoopRateLimitError(err)) { + throw err; + } errors.push({ message: `getCycles: ${err instanceof Error ? err.message : String(err)}`, cause: err, @@ -211,6 +215,11 @@ export class WhoopProvider implements SyncProvider { recordsSynced += await syncWhoopJournal(context); } + const rateLimitError = findWhoopRateLimitError(errors); + if (rateLimitError) { + throw rateLimitError; + } + return { provider: this.id, recordsSynced, diff --git a/src/providers/whoop/rate-limit.test.ts b/src/providers/whoop/rate-limit.test.ts new file mode 100644 index 0000000000..0ced390d5f --- /dev/null +++ b/src/providers/whoop/rate-limit.test.ts @@ -0,0 +1,42 @@ +import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; +import { describe, expect, it } from "vitest"; +import { WhoopRateLimitError } from "whoop-whoop/client"; +import { findWhoopRateLimitError, isWhoopRateLimitError } from "./rate-limit.ts"; + +describe("isWhoopRateLimitError", () => { + it("recognizes WhoopRateLimitError", () => { + expect(isWhoopRateLimitError(new WhoopRateLimitError("limited"))).toBe(true); + }); + + it("recognizes ProviderRateLimitError tagged with whoop", () => { + const error = new ProviderRateLimitError({ + message: "whoop API rate limit exceeded (429):", + providerId: "whoop", + statusCode: 429, + responseBody: "", + }); + expect(isWhoopRateLimitError(error)).toBe(true); + }); + + it("rejects other provider rate-limit errors", () => { + const error = new ProviderRateLimitError({ + message: "garmin API rate limit exceeded (429):", + providerId: "garmin", + statusCode: 429, + responseBody: "", + }); + expect(isWhoopRateLimitError(error)).toBe(false); + }); +}); + +describe("findWhoopRateLimitError", () => { + it("returns the first whoop rate-limit error from sync errors", () => { + const error = new ProviderRateLimitError({ + message: "whoop API rate limit exceeded (429):", + providerId: "whoop", + statusCode: 429, + responseBody: "", + }); + expect(findWhoopRateLimitError([{ cause: error }])).toBe(error); + }); +}); diff --git a/src/providers/whoop/rate-limit.ts b/src/providers/whoop/rate-limit.ts new file mode 100644 index 0000000000..706f291055 --- /dev/null +++ b/src/providers/whoop/rate-limit.ts @@ -0,0 +1,17 @@ +import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; + +export function isWhoopRateLimitError(err: unknown): err is ProviderRateLimitError { + return err instanceof ProviderRateLimitError && err.providerId === "whoop"; +} + +export function findWhoopRateLimitError( + errors: Array<{ cause?: unknown }>, +): ProviderRateLimitError | null { + for (const syncError of errors) { + const cause = syncError.cause; + if (isWhoopRateLimitError(cause)) { + return cause; + } + } + return null; +} diff --git a/src/providers/whoop/sync-daily-activity.ts b/src/providers/whoop/sync-daily-activity.ts index 17d100daf4..b433882783 100644 --- a/src/providers/whoop/sync-daily-activity.ts +++ b/src/providers/whoop/sync-daily-activity.ts @@ -1,7 +1,9 @@ -import { WhoopRateLimitError } from "whoop-whoop/client"; +import { and, eq, isNotNull } from "drizzle-orm"; import { dailyMetrics } from "../../db/schema.ts"; import { withSyncLog } from "../../db/sync-log.ts"; +import { getTokenUserId } from "../../db/token-user-context.ts"; import { parseStrainDeepDiveSteps } from "./parsing.ts"; +import { isWhoopRateLimitError } from "./rate-limit.ts"; import type { WhoopSyncContext } from "./sync-types.ts"; export type WhoopDailyActivityResult = { @@ -25,7 +27,7 @@ function* iterateUtcDates(start: Date, endMs: number): Generator<string> { export async function syncWhoopDailyActivity( context: WhoopSyncContext, ): Promise<WhoopDailyActivityResult> { - const { db, client, providerId, since, options } = context; + const { db, client, providerId, since, windowEnd, options } = context; try { const count = await withSyncLog( @@ -33,10 +35,28 @@ export async function syncWhoopDailyActivity( providerId, "daily_activity", async () => { - const nowMs = Date.now(); const stepsByDate = new Map<string, number>(); + const userId = options?.userId ?? getTokenUserId(); + const syncedStepDates = + userId == null + ? new Set<string>() + : new Set( + ( + await db + .select({ date: dailyMetrics.date }) + .from(dailyMetrics) + .where( + and( + eq(dailyMetrics.userId, userId), + eq(dailyMetrics.providerId, providerId), + isNotNull(dailyMetrics.steps), + ), + ) + ).map((row) => row.date), + ); - for (const date of iterateUtcDates(since, nowMs)) { + for (const date of iterateUtcDates(since, windowEnd.getTime())) { + if (syncedStepDates.has(date)) continue; const raw = await client.getStrainDeepDive(date); const steps = parseStrainDeepDiveSteps(raw); if (steps != null) { @@ -65,7 +85,11 @@ export async function syncWhoopDailyActivity( ); return { count, rateLimited: false }; } catch (err) { - if (err instanceof WhoopRateLimitError) { + if (isWhoopRateLimitError(err)) { + context.errors.push({ + message: `daily_activity: ${err instanceof Error ? err.message : String(err)}`, + cause: err, + }); return { count: 0, rateLimited: true }; } context.errors.push({ diff --git a/src/providers/whoop/sync-helpers.test.ts b/src/providers/whoop/sync-helpers.test.ts index f226b875f1..1c29897776 100644 --- a/src/providers/whoop/sync-helpers.test.ts +++ b/src/providers/whoop/sync-helpers.test.ts @@ -1,15 +1,21 @@ +import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { WhoopClient } from "whoop-whoop/client"; import type { WhoopCycle, WhoopWorkoutRecord } from "whoop-whoop/types"; import type { SyncDatabase } from "../../db/index.ts"; import { writeMetricStreamBatch } from "../../db/metric-stream-writer.ts"; import { SOURCE_TYPE_API } from "../../db/sensor-channels.ts"; +import { withSyncLog } from "../../db/sync-log.ts"; import { SyncWindow } from "../sync-window.ts"; import { syncWhoopDailyActivity } from "./sync-daily-activity.ts"; import { syncWhoopSleepSessions, syncWhoopSleepStages } from "./sync-sleep.ts"; import { syncWhoopHeartRateStream } from "./sync-streams.ts"; import type { WhoopSyncContext } from "./sync-types.ts"; -import { syncWhoopWorkouts } from "./sync-workouts.ts"; +import { syncWhoopStrength, syncWhoopWorkouts } from "./sync-workouts.ts"; + +const tokenUserContextMocks = vi.hoisted(() => ({ + getTokenUserId: vi.fn((): string | undefined => "00000000-0000-0000-0000-000000000001"), +})); const providerActivityAbsenceMocks = vi.hoisted(() => ({ finishProviderActivityListSync: vi.fn().mockResolvedValue(undefined), @@ -39,6 +45,10 @@ vi.mock("../../db/metric-stream-writer.ts", () => ({ writeMetricStreamBatch: vi.fn().mockResolvedValue(undefined), })); +vi.mock("../../db/token-user-context.ts", () => ({ + getTokenUserId: tokenUserContextMocks.getTokenUserId, +})); + function makeDb(selectedRows: unknown[] = []) { const chain = { values: vi.fn(), @@ -51,7 +61,11 @@ function makeDb(selectedRows: unknown[] = []) { chain.values.mockReturnValue(chain); chain.onConflictDoUpdate.mockResolvedValue(undefined); chain.from.mockReturnValue(chain); - chain.where.mockReturnValue(chain); + chain.where.mockReturnValue( + Object.assign(Promise.resolve(selectedRows), { + limit: vi.fn().mockResolvedValue(selectedRows), + }), + ); chain.limit.mockResolvedValue(selectedRows); const db: SyncDatabase = { @@ -87,6 +101,15 @@ function makeContext(overrides: Partial<WhoopSyncContext> = {}): WhoopSyncContex }; } +function makeWhoopRateLimitError(message = "whoop API rate limit exceeded (429):") { + return new ProviderRateLimitError({ + message, + providerId: "whoop", + statusCode: 429, + responseBody: "", + }); +} + beforeEach(() => { vi.useFakeTimers(); vi.setSystemTime(new Date("2026-05-09T00:00:00.000Z")); @@ -94,6 +117,7 @@ beforeEach(() => { providerActivityAbsenceMocks.finishProviderActivityListSync.mockClear(); providerActivityAbsenceMocks.upsertProviderActivity.mockClear(); providerActivityAbsenceMocks.upsertProviderActivity.mockResolvedValue(undefined); + tokenUserContextMocks.getTokenUserId.mockReturnValue("00000000-0000-0000-0000-000000000001"); }); function makeWorkoutRecord( @@ -166,7 +190,7 @@ describe("WHOOP sync helpers", () => { expect(result).toEqual({ count: 2, rateLimited: false }); expect(getStrainDeepDive).toHaveBeenCalledWith("2026-05-01"); - expect(getStrainDeepDive).toHaveBeenCalledWith("2026-05-09"); + expect(getStrainDeepDive).toHaveBeenCalledWith("2026-05-02"); expect(db.chain.values).toHaveBeenCalledWith({ date: "2026-05-01", providerId: "whoop", @@ -203,6 +227,139 @@ describe("WHOOP sync helpers", () => { expect(context.errors[0]?.message).toBe("daily_activity: network down"); }); + it("marks daily activity sync as rate limited when the API returns 429", async () => { + const rateLimitError = makeWhoopRateLimitError(); + const client = makeClient(); + vi.spyOn(client, "getStrainDeepDive").mockRejectedValue(rateLimitError); + const context = makeContext({ client }); + + await expect(syncWhoopDailyActivity(context)).resolves.toEqual({ + count: 0, + rateLimited: true, + }); + expect(context.errors).toEqual([ + { + message: `daily_activity: ${rateLimitError.message}`, + cause: rateLimitError, + }, + ]); + }); + + it("skips dates that already have synced step counts for the resolved user", async () => { + const db = makeDb([{ date: "2026-05-01" }]); + const client = makeClient(); + const getStrainDeepDive = vi.spyOn(client, "getStrainDeepDive").mockResolvedValue({ + sections: [ + { + items: [ + { + type: "CONTRIBUTORS_TILE", + content: { + id: "STRAIN_CONTRIBUTORS_TILE", + metrics: [{ id: "CONTRIBUTORS_TILE_STEPS", status: "1,000" }], + }, + }, + ], + }, + ], + }); + const context = makeContext({ db: db.db, client, options: { userId: "user-1" } }); + + await expect(syncWhoopDailyActivity(context)).resolves.toEqual({ + count: 1, + rateLimited: false, + }); + expect(db.select).toHaveBeenCalled(); + expect(getStrainDeepDive).toHaveBeenCalledTimes(1); + expect(getStrainDeepDive).toHaveBeenCalledWith("2026-05-02"); + expect(db.chain.values).toHaveBeenCalledWith({ + date: "2026-05-02", + providerId: "whoop", + steps: 1000, + }); + }); + + it("loads synced step dates from options.userId when provided", async () => { + const db = makeDb([]); + const client = makeClient(); + vi.spyOn(client, "getStrainDeepDive").mockResolvedValue({ sections: [] }); + const context = makeContext({ db: db.db, client, options: { userId: "explicit-user" } }); + + await syncWhoopDailyActivity(context); + + expect(db.select).toHaveBeenCalled(); + }); + + it("loads synced step dates from the token user id when options is omitted", async () => { + const db = makeDb([]); + const client = makeClient(); + vi.spyOn(client, "getStrainDeepDive").mockResolvedValue({ sections: [] }); + tokenUserContextMocks.getTokenUserId.mockReturnValue("token-user"); + const context = makeContext({ db: db.db, client, options: undefined }); + + await syncWhoopDailyActivity(context); + + expect(db.select).toHaveBeenCalled(); + }); + + it("does not query synced step dates when no user id can be resolved", async () => { + const db = makeDb([]); + const client = makeClient(); + const getStrainDeepDive = vi.spyOn(client, "getStrainDeepDive").mockResolvedValue({ + sections: [ + { + items: [ + { + type: "CONTRIBUTORS_TILE", + content: { + id: "STRAIN_CONTRIBUTORS_TILE", + metrics: [{ id: "CONTRIBUTORS_TILE_STEPS", status: "500" }], + }, + }, + ], + }, + ], + }); + tokenUserContextMocks.getTokenUserId.mockReturnValue(undefined); + const context = makeContext({ db: db.db, client, options: undefined }); + + await expect(syncWhoopDailyActivity(context)).resolves.toEqual({ + count: 2, + rateLimited: false, + }); + expect(db.select).not.toHaveBeenCalled(); + expect(getStrainDeepDive).toHaveBeenCalledWith("2026-05-01"); + expect(getStrainDeepDive).toHaveBeenCalledWith("2026-05-02"); + }); + + it("records strength sync rate limits without failing the whole provider sync", async () => { + const rateLimitError = makeWhoopRateLimitError("strength limited"); + vi.mocked(withSyncLog).mockRejectedValueOnce(rateLimitError); + const context = makeContext({ cycles: [{ workouts: [makeWorkoutRecord()] }] }); + + await expect(syncWhoopStrength(context)).resolves.toEqual({ + count: 0, + rateLimited: true, + }); + expect(context.errors).toEqual([ + { + message: "strength: strength limited", + cause: rateLimitError, + }, + ]); + }); + + it("records strength sync errors without marking them rate limited", async () => { + vi.mocked(withSyncLog).mockRejectedValueOnce(new Error("database unavailable")); + const context = makeContext({ cycles: [{ workouts: [makeWorkoutRecord()] }] }); + + await expect(syncWhoopStrength(context)).resolves.toEqual({ + count: 0, + rateLimited: false, + }); + expect(context.errors[0]?.message).toBe("strength: database unavailable"); + }); + it("writes parsed heart-rate stream rows in weekly windows", async () => { const client = makeClient(); vi.spyOn(client, "getHeartRate").mockResolvedValue([ diff --git a/src/providers/whoop/sync-streams.ts b/src/providers/whoop/sync-streams.ts index 9ecd34fb04..9a7d662e4d 100644 --- a/src/providers/whoop/sync-streams.ts +++ b/src/providers/whoop/sync-streams.ts @@ -1,8 +1,8 @@ -import { WhoopRateLimitError } from "whoop-whoop/client"; import { writeMetricStreamBatch } from "../../db/metric-stream-writer.ts"; import { SOURCE_TYPE_API } from "../../db/sensor-channels.ts"; import { withSyncLog } from "../../db/sync-log.ts"; import { parseHeartRateValues } from "./parsing.ts"; +import { isWhoopRateLimitError } from "./rate-limit.ts"; import type { WhoopSyncContext } from "./sync-types.ts"; export type WhoopStreamSyncResult = { @@ -57,9 +57,9 @@ export async function syncWhoopHeartRateStream( ); return { count, rateLimited: false }; } catch (err) { - if (err instanceof WhoopRateLimitError) { + if (isWhoopRateLimitError(err)) { context.errors.push({ - message: `hr_stream: ${err.message}`, + message: `hr_stream: ${err instanceof Error ? err.message : String(err)}`, cause: err, }); return { count: 0, rateLimited: true }; diff --git a/src/providers/whoop/sync-workouts.ts b/src/providers/whoop/sync-workouts.ts index 0e9cafade7..5deac7ed56 100644 --- a/src/providers/whoop/sync-workouts.ts +++ b/src/providers/whoop/sync-workouts.ts @@ -1,5 +1,4 @@ import { and, eq, sql } from "drizzle-orm"; -import { WhoopRateLimitError } from "whoop-whoop/client"; import type { WhoopWorkoutRecord } from "whoop-whoop/types"; import { parseDuringRange } from "whoop-whoop/utils"; import { @@ -15,6 +14,7 @@ import { parseWorkout, resolveWhoopWorkoutExternalId, } from "./parsing.ts"; +import { isWhoopRateLimitError } from "./rate-limit.ts"; import type { WhoopSyncContext } from "./sync-types.ts"; export type WhoopWorkoutSyncResult = { @@ -296,9 +296,9 @@ export async function syncWhoopStrength( ); return { count, rateLimited: false }; } catch (err) { - if (err instanceof WhoopRateLimitError) { + if (isWhoopRateLimitError(err)) { context.errors.push({ - message: `strength: ${err.message}`, + message: `strength: ${err instanceof Error ? err.message : String(err)}`, cause: err, }); return { count: 0, rateLimited: true }; diff --git a/src/providers/withings.test.ts b/src/providers/withings.test.ts index 87b074ea10..9d5f86ee26 100644 --- a/src/providers/withings.test.ts +++ b/src/providers/withings.test.ts @@ -8,6 +8,7 @@ vi.mock("../db/token-user-context.ts", () => ({ runWithTokenUser: async (_userId: string, callback: () => Promise<unknown>) => callback(), })); +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { createMockDatabase } from "./test-helpers.ts"; import { exchangeWithingsCode, @@ -1065,9 +1066,11 @@ describe("Withings — rate-limit aware fetch wiring", () => { }; it("token exchange surfaces a 429 as a ProviderRateLimitError tagged 'withings'", async () => { - const err = await exchangeWithingsCode(oauthConfig, "code", rateLimited429).catch( - (caught: unknown) => caught, - ); + const err = await exchangeWithingsCode( + oauthConfig, + "code", + createProviderRateLimitFetch("withings", rateLimited429), + ).catch((caught: unknown) => caught); expect(err).toBeInstanceOf(ProviderRateLimitError); if (err instanceof ProviderRateLimitError) { expect(err.providerId).toBe("withings"); @@ -1076,7 +1079,10 @@ describe("Withings — rate-limit aware fetch wiring", () => { }); it("WithingsClient surfaces a 429 as a ProviderRateLimitError tagged 'withings'", async () => { - const client = new WithingsClient("access-token", rateLimited429); + const client = new WithingsClient( + "access-token", + createProviderRateLimitFetch("withings", rateLimited429), + ); const err = await client.getMeas(0, 1).catch((caught: unknown) => caught); expect(err).toBeInstanceOf(ProviderRateLimitError); if (err instanceof ProviderRateLimitError) { diff --git a/src/providers/withings.ts b/src/providers/withings.ts index 02f7d9d5d8..fe4ebae429 100644 --- a/src/providers/withings.ts +++ b/src/providers/withings.ts @@ -1,4 +1,3 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; import { z } from "zod"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; import { getOAuthRedirectUri } from "../auth/oauth.ts"; @@ -7,6 +6,7 @@ import { writeMetricStreamBatch } from "../db/metric-stream-writer.ts"; import { SOURCE_TYPE_API } from "../db/sensor-channels.ts"; import { withSyncLog } from "../db/sync-log.ts"; import { deleteTokens, ensureProvider, loadTokens, saveTokens } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { logger } from "../logger.ts"; import { ProviderAuthenticationFailedError, RefreshTokenRevokedError } from "./auth-errors.ts"; import type { SyncRun } from "./sync-run.ts"; @@ -182,7 +182,6 @@ async function withingsTokenExchange( params: Record<string, string>, fetchFn: typeof globalThis.fetch = globalThis.fetch, ): Promise<TokenSet> { - const rateLimitFetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "withings" }); const bodyParams: Record<string, string> = { action: "requesttoken", client_id: config.clientId, @@ -191,7 +190,7 @@ async function withingsTokenExchange( if (config.clientSecret) bodyParams.client_secret = config.clientSecret; const body = new URLSearchParams(bodyParams); - const response = await rateLimitFetchFn(config.tokenUrl, { + const response = await fetchFn(config.tokenUrl, { method: "POST", headers: { "Content-Type": "application/x-www-form-urlencoded" }, body: body.toString(), @@ -263,7 +262,7 @@ export class WithingsClient { constructor(accessToken: string, fetchFn: typeof globalThis.fetch = globalThis.fetch) { this.#accessToken = accessToken; - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "withings" }); + this.#fetchFn = fetchFn; } async #post<T>(path: string, params: Record<string, string>): Promise<T> { @@ -330,7 +329,7 @@ export class WithingsProvider implements WebhookProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "withings" }); + this.#fetchFn = createProviderRateLimitFetch("withings", fetchFn); } validate(): string | null { @@ -403,7 +402,7 @@ export class WithingsProvider implements WebhookProvider { if (!config) throw new Error("WITHINGS_CLIENT_ID and WITHINGS_CLIENT_SECRET are required"); return { oauthConfig: config, - exchangeCode: (code) => exchangeWithingsCode(config, code), + exchangeCode: (code) => exchangeWithingsCode(config, code, this.#fetchFn), apiBaseUrl: WITHINGS_API_BASE, }; } diff --git a/src/providers/xert.ts b/src/providers/xert.ts index 269b21fd5b..53ce086d7b 100644 --- a/src/providers/xert.ts +++ b/src/providers/xert.ts @@ -1,4 +1,3 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; import type { CanonicalActivityType } from "@dofek/training/training"; import { z } from "zod"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; @@ -11,6 +10,7 @@ import { } from "../db/provider-activity-sync.ts"; import { withSyncLog } from "../db/sync-log.ts"; import { ensureProvider } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { SyncRun } from "./sync-run.ts"; import type { ProviderAuthSetup, SyncError, SyncProvider, SyncResult } from "./types.ts"; @@ -133,7 +133,7 @@ export async function signInToXert( password: string, fetchFn: typeof globalThis.fetch = globalThis.fetch, ): Promise<TokenSet> { - const rateLimitFetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "xert" }); + const rateLimitFetchFn = createProviderRateLimitFetch("xert", fetchFn); const clientId = process.env.XERT_CLIENT_ID ?? "xert_public"; const clientSecret = process.env.XERT_CLIENT_SECRET ?? "xert_public"; @@ -197,7 +197,7 @@ export class XertProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "xert" }); + this.#fetchFn = createProviderRateLimitFetch("xert", fetchFn); } validate(): string | null { diff --git a/src/providers/zwift.ts b/src/providers/zwift.ts index a14027f395..2447ee1de1 100644 --- a/src/providers/zwift.ts +++ b/src/providers/zwift.ts @@ -1,4 +1,3 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; import { z } from "zod"; import { ZWIFT_API_BASE, ZwiftClient } from "zwift-client/client"; import { parseZwiftActivity, parseZwiftFitnessData } from "zwift-client/parsing"; @@ -11,6 +10,7 @@ import { import { SOURCE_TYPE_API } from "../db/sensor-channels.ts"; import { withSyncLog } from "../db/sync-log.ts"; import { ensureProvider, loadTokens, saveTokens } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { logger } from "../logger.ts"; import { ProviderAuthenticationFailedError, @@ -30,7 +30,7 @@ export class ZwiftProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "zwift" }); + this.#fetchFn = createProviderRateLimitFetch("zwift", fetchFn); } validate(): string | null {