From 002a33cc2802601ae403c26e2183b90187d98dd7 Mon Sep 17 00:00:00 2001 From: Asher Cohen Date: Sun, 21 Jun 2026 09:25:29 -0700 Subject: [PATCH 01/18] Improve provider sync rate limiting for Garmin and Whoop. Add adaptive per-provider throttling with learned cooldowns, skip redundant API calls during sync, and prevent scheduled jobs from enqueueing while a provider cooldown is active. Co-authored-by: Cursor --- packages/garmin-connect/src/client.ts | 37 ++- packages/provider-http/package.json | 3 +- .../src/adaptive-rate-limit.test.ts | 87 ++++++ .../provider-http/src/adaptive-rate-limit.ts | 220 ++++++++++++++++ packages/provider-http/src/rate-limit.ts | 68 +++-- packages/whoop-whoop/src/client.test.ts | 33 +-- packages/whoop-whoop/src/client.ts | 15 +- src/jobs/enqueue-sync-job.test.ts | 18 ++ src/jobs/enqueue-sync-job.ts | 16 +- src/jobs/process-scheduled-sync-job.test.ts | 23 +- src/jobs/process-scheduled-sync-job.ts | 15 +- src/jobs/provider-rate-limit-cooldown.test.ts | 43 ++- src/jobs/provider-rate-limit-cooldown.ts | 96 ++++++- src/lib/provider-adaptive-rate-limit.test.ts | 39 +++ src/lib/provider-adaptive-rate-limit.ts | 249 ++++++++++++++++++ src/lib/provider-rate-limit-fetch.ts | 17 ++ src/providers/amazfit-zepp.test.ts | 13 +- src/providers/amazfit-zepp.ts | 7 +- src/providers/bodyspec.ts | 4 +- src/providers/concept2.ts | 4 +- src/providers/coros.ts | 4 +- src/providers/cycling-analytics.ts | 4 +- src/providers/decathlon.ts | 4 +- src/providers/eight-sleep.ts | 4 +- src/providers/fatsecret/client.ts | 8 +- src/providers/fatsecret/provider.ts | 5 +- src/providers/fitbit/provider.ts | 4 +- src/providers/garmin.test.ts | 37 ++- src/providers/garmin.ts | 37 ++- src/providers/http-client.ts | 4 +- src/providers/komoot.ts | 4 +- src/providers/mapmyfitness.test.ts | 3 +- src/providers/mapmyfitness.ts | 6 +- src/providers/oura/provider.ts | 4 +- src/providers/peloton.ts | 8 +- src/providers/polar/client.test.ts | 1 + src/providers/polar/client.ts | 4 +- src/providers/polar/provider.ts | 4 +- src/providers/polar/webhook-service.ts | 4 +- src/providers/ride-with-gps.ts | 6 +- src/providers/strava.ts | 16 +- src/providers/suunto.ts | 4 +- src/providers/trainerroad.ts | 4 +- src/providers/ultrahuman.ts | 7 +- src/providers/velohero.ts | 4 +- src/providers/wahoo/provider.ts | 4 +- src/providers/wger.ts | 4 +- src/providers/whoop.test.ts | 1 + src/providers/whoop/provider.ts | 13 +- src/providers/whoop/rate-limit.test.ts | 42 +++ src/providers/whoop/rate-limit.ts | 19 ++ src/providers/whoop/sync-daily-activity.ts | 25 +- src/providers/whoop/sync-helpers.test.ts | 12 + src/providers/whoop/sync-streams.ts | 6 +- src/providers/whoop/sync-workouts.ts | 6 +- src/providers/withings.ts | 8 +- src/providers/xert.ts | 6 +- src/providers/zwift.ts | 4 +- 58 files changed, 1134 insertions(+), 213 deletions(-) create mode 100644 packages/provider-http/src/adaptive-rate-limit.test.ts create mode 100644 packages/provider-http/src/adaptive-rate-limit.ts create mode 100644 src/lib/provider-adaptive-rate-limit.test.ts create mode 100644 src/lib/provider-adaptive-rate-limit.ts create mode 100644 src/lib/provider-rate-limit-fetch.ts create mode 100644 src/providers/whoop/rate-limit.test.ts create mode 100644 src/providers/whoop/rate-limit.ts diff --git a/packages/garmin-connect/src/client.ts b/packages/garmin-connect/src/client.ts index d3e58a276f..eb532b2780 100644 --- a/packages/garmin-connect/src/client.ts +++ b/packages/garmin-connect/src/client.ts @@ -10,11 +10,7 @@ * 6. All API calls use OAuth2 Bearer token */ -import { - createRateLimitAwareFetch, - ProviderRateLimitError, - parseRetryAfterHeader, -} from "@dofek/provider-http/rate-limit"; +import { ProviderRateLimitError, parseRetryAfterHeader } from "@dofek/provider-http/rate-limit"; import { buildOAuth1Header } from "./oauth1.ts"; import type { BodyBatteryDay, @@ -45,6 +41,10 @@ const OAUTH_CONSUMER_URL = "https://thegarth.s3.amazonaws.com/oauth_consumer.jso const USER_AGENT = "com.garmin.android.apps.connectmobile"; const API_USER_AGENT = "GCM-iOS-5.19.1.2"; +/** Minimum delay between consecutive Connect API requests (ms). + * Garmin's unofficial API rate limits aggressively; 2s keeps bursts under control. */ +export const GARMIN_CONNECT_THROTTLE_MS = 2_000; + const CSRF_RE = /name="_csrf"\s+value="(.+?)"/; const TITLE_RE = /(.+?)<\/title>/; const TICKET_RE = /embed\?ticket=([^"]+)"/; @@ -56,18 +56,12 @@ export class GarminConnectClient { #displayName: string | null = null; #domain: string; #fetchFn: typeof globalThis.fetch; - - constructor(domain: string = "garmin.com", fetchFn: typeof globalThis.fetch = globalThis.fetch) { + constructor( + domain: string = "garmin.com", + fetchFn: typeof globalThis.fetch = globalThis.fetch, + ) { this.#domain = domain; - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { - providerId: "garmin", - createRateLimitError: (response, responseBody) => - new GarminRateLimitError( - `Rate limit exceeded (${response.status}): ${responseBody}`, - responseBody, - response.headers?.get?.("Retry-After"), - ), - }); + this.#fetchFn = fetchFn; } // ============================================================ @@ -346,12 +340,15 @@ export class GarminConnectClient { throw new GarminAuthError("Authentication failed (401)"); } - if (response.status === 429) { - throw new GarminRateLimitError("Rate limit exceeded (429)"); - } - if (!response.ok) { const text = await response.text(); + if (response.status === 429) { + throw new GarminRateLimitError( + `Rate limit exceeded (${response.status}): ${text}`, + text, + response.headers?.get?.("Retry-After"), + ); + } throw new GarminApiError(`API error (${response.status}): ${text}`, response.status); } diff --git a/packages/provider-http/package.json b/packages/provider-http/package.json index c611677c37..69d0910e90 100644 --- a/packages/provider-http/package.json +++ b/packages/provider-http/package.json @@ -4,6 +4,7 @@ "private": true, "type": "module", "exports": { - "./rate-limit": "./src/rate-limit.ts" + "./rate-limit": "./src/rate-limit.ts", + "./adaptive-rate-limit": "./src/adaptive-rate-limit.ts" } } diff --git a/packages/provider-http/src/adaptive-rate-limit.test.ts b/packages/provider-http/src/adaptive-rate-limit.test.ts new file mode 100644 index 0000000000..70ab705a2e --- /dev/null +++ b/packages/provider-http/src/adaptive-rate-limit.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from "vitest"; +import { + admissionDelayMs, + applyStravaQuota, + blendObservedCooldown, + createInitialAdaptiveState, + decreaseThrottleMs, + learnInferredBudget, + parseStravaRateLimitHeaders, + recordAdaptiveRateLimit, + recordAdaptiveRequest, + slideAdaptiveWindow, +} from "./adaptive-rate-limit.ts"; + +describe("parseStravaRateLimitHeaders", () => { + it("parses Strava short and daily quota headers", () => { + const headers = new Headers({ + "X-RateLimit-Limit": "100,1000", + "X-RateLimit-Usage": "42,500", + }); + expect(parseStravaRateLimitHeaders(headers)).toEqual({ + shortLimit: 100, + shortUsage: 42, + dailyLimit: 1000, + dailyUsage: 500, + }); + }); + + it("returns null when headers are missing", () => { + expect(parseStravaRateLimitHeaders(new Headers())).toBeNull(); + }); +}); + +describe("adaptive rate-limit learning", () => { + it("blends observed cooldown values toward recent observations", () => { + expect(blendObservedCooldown(600, 300)).toBe(510); + expect(blendObservedCooldown(null, 300)).toBe(300); + }); + + it("slides the rolling window without recomputing historical requests", () => { + const state = createInitialAdaptiveState("whoop", "provider", null, 0); + const withRequests = { ...state, requestCount: 12, windowStartMs: 0 }; + const slid = slideAdaptiveWindow(withRequests, 5 * 60 * 1000 + 1); + expect(slid.requestCount).toBe(0); + expect(slid.windowStartMs).toBe(5 * 60 * 1000 + 1); + }); + + it("records requests by incrementing the rolling tally", () => { + const state = createInitialAdaptiveState("garmin", "provider", null, 1000); + const next = recordAdaptiveRequest(state, 1500); + expect(next.requestCount).toBe(1); + expect(next.lastRequestMs).toBe(1500); + expect(next.throttleMs).toBeLessThan(state.throttleMs); + }); + + it("learns inferred budget and increases throttle on rate limits", () => { + const state = { + ...createInitialAdaptiveState("whoop", "provider", null), + requestCount: 25, + throttleMs: 1000, + inferredBudget: 30, + }; + const limited = recordAdaptiveRateLimit(state, 120); + expect(limited.inferredBudget).toBe(24); + expect(limited.throttleMs).toBe(2000); + expect(limited.observedCooldownSeconds).toBe(120); + }); + + it("paces Strava requests when short quota is nearly exhausted", () => { + const state = applyStravaQuota(createInitialAdaptiveState("strava", "provider", null), { + shortLimit: 100, + shortUsage: 99, + dailyLimit: 1000, + dailyUsage: 100, + }); + expect(admissionDelayMs(state, Date.now())).toBeGreaterThanOrEqual(state.throttleMs * 4); + }); + + it("learns a lower inferred budget from observed failures", () => { + expect(learnInferredBudget(30, 25)).toBe(24); + expect(learnInferredBudget(null, 10)).toBe(9); + }); + + it("decreases throttle gradually on success", () => { + expect(decreaseThrottleMs(1000)).toBe(900); + }); +}); diff --git a/packages/provider-http/src/adaptive-rate-limit.ts b/packages/provider-http/src/adaptive-rate-limit.ts new file mode 100644 index 0000000000..dae58b6315 --- /dev/null +++ b/packages/provider-http/src/adaptive-rate-limit.ts @@ -0,0 +1,220 @@ +import type { ProviderRateLimitError, ProviderRateLimitScope } from "./rate-limit.ts"; + +/** Rolling window for request budget tally (5 minutes). */ +export const ADAPTIVE_RATE_WINDOW_MS = 5 * 60 * 1000; + +export const ADAPTIVE_THROTTLE_MIN_MS = 500; +export const ADAPTIVE_THROTTLE_MAX_MS = 30_000; +export const ADAPTIVE_THROTTLE_DECREASE_MS = 100; +export const ADAPTIVE_THROTTLE_INCREASE_FACTOR = 2; +export const ADAPTIVE_BUDGET_SAFETY_RATIO = 0.8; +export const ADAPTIVE_DEFAULT_INFERRED_BUDGET = 40; + +export const DEFAULT_PROVIDER_THROTTLE_MS: Readonly<Record<string, number>> = { + strava: 10_000, + garmin: 2_000, + whoop: 1_000, +}; + +export interface StravaRateLimitQuota { + shortLimit: number; + shortUsage: number; + dailyLimit: number; + dailyUsage: number; +} + +export interface ProviderAdaptiveRateState { + providerId: string; + scope: ProviderRateLimitScope; + userId: string | null; + windowStartMs: number; + requestCount: number; + throttleMs: number; + lastRequestMs: number | null; + inferredBudget: number | null; + observedCooldownSeconds: number | null; + stravaShortLimit: number | null; + stravaShortUsage: number | null; + stravaDailyLimit: number | null; + stravaDailyUsage: number | null; +} + +export interface AdaptiveRateLimitStore { + awaitAdmission( + providerId: string, + scope: ProviderRateLimitScope, + userId: string | null, + ): Promise<void>; + recordSuccess( + providerId: string, + scope: ProviderRateLimitScope, + userId: string | null, + responseHeaders?: Headers, + ): Promise<void>; + recordRateLimit(error: ProviderRateLimitError): Promise<void>; + getLearnedCooldownSeconds(providerId: string): Promise<number | null>; +} + +export function defaultThrottleMs(providerId: string): number { + return DEFAULT_PROVIDER_THROTTLE_MS[providerId] ?? 1_000; +} + +export function parseStravaRateLimitHeaders(headers: Headers): StravaRateLimitQuota | null { + const limitHeader = headers.get("X-RateLimit-Limit"); + const usageHeader = headers.get("X-RateLimit-Usage"); + if (!limitHeader || !usageHeader) return null; + + const [shortLimitRaw, dailyLimitRaw] = limitHeader.split(","); + const [shortUsageRaw, dailyUsageRaw] = usageHeader.split(","); + const shortLimit = Number.parseInt(shortLimitRaw ?? "", 10); + const dailyLimit = Number.parseInt(dailyLimitRaw ?? "", 10); + const shortUsage = Number.parseInt(shortUsageRaw ?? "", 10); + const dailyUsage = Number.parseInt(dailyUsageRaw ?? "", 10); + if ( + !Number.isFinite(shortLimit) || + !Number.isFinite(dailyLimit) || + !Number.isFinite(shortUsage) || + !Number.isFinite(dailyUsage) + ) { + return null; + } + return { shortLimit, shortUsage, dailyLimit, dailyUsage }; +} + +export function blendObservedCooldown( + previous: number | null, + observedSeconds: number, +): number { + if (!Number.isFinite(observedSeconds) || observedSeconds <= 0) return previous ?? observedSeconds; + if (previous == null) return observedSeconds; + return Math.round(previous * 0.7 + observedSeconds * 0.3); +} + +export function slideAdaptiveWindow( + state: ProviderAdaptiveRateState, + nowMs: number, +): ProviderAdaptiveRateState { + if (nowMs - state.windowStartMs < ADAPTIVE_RATE_WINDOW_MS) return state; + return { + ...state, + windowStartMs: nowMs, + requestCount: 0, + }; +} + +export function decreaseThrottleMs(throttleMs: number): number { + return Math.max(ADAPTIVE_THROTTLE_MIN_MS, throttleMs - ADAPTIVE_THROTTLE_DECREASE_MS); +} + +export function increaseThrottleMs(throttleMs: number): number { + return Math.min(ADAPTIVE_THROTTLE_MAX_MS, throttleMs * ADAPTIVE_THROTTLE_INCREASE_FACTOR); +} + +export function learnInferredBudget( + current: number | null, + requestsAtLimit: number, +): number | null { + if (requestsAtLimit <= 0) return current; + const candidate = Math.max(1, requestsAtLimit - 1); + if (current == null) return candidate; + return Math.min(current, candidate); +} + +export function admissionDelayMs(state: ProviderAdaptiveRateState, nowMs: number): number { + let delayMs = 0; + + if (state.lastRequestMs != null) { + const elapsed = nowMs - state.lastRequestMs; + if (elapsed < state.throttleMs) { + delayMs = state.throttleMs - elapsed; + } + } + + if (state.inferredBudget != null) { + const softCap = Math.floor(state.inferredBudget * ADAPTIVE_BUDGET_SAFETY_RATIO); + if (state.requestCount >= softCap) { + delayMs = Math.max(delayMs, state.throttleMs); + } + } + + if (state.providerId === "strava" && state.stravaShortLimit != null && state.stravaShortUsage != null) { + const remaining = state.stravaShortLimit - state.stravaShortUsage; + if (remaining <= 2) { + delayMs = Math.max(delayMs, state.throttleMs * 4); + } else if (remaining <= 5) { + delayMs = Math.max(delayMs, state.throttleMs * 2); + } else { + const windowMs = 15 * 60 * 1000; + const pacedDelay = Math.ceil(windowMs / remaining); + delayMs = Math.max(delayMs, Math.min(pacedDelay, ADAPTIVE_THROTTLE_MAX_MS)); + } + } + + return delayMs; +} + +export function createInitialAdaptiveState( + providerId: string, + scope: ProviderRateLimitScope, + userId: string | null, + nowMs = Date.now(), +): ProviderAdaptiveRateState { + return { + providerId, + scope, + userId, + windowStartMs: nowMs, + requestCount: 0, + throttleMs: defaultThrottleMs(providerId), + lastRequestMs: null, + inferredBudget: null, + observedCooldownSeconds: null, + stravaShortLimit: null, + stravaShortUsage: null, + stravaDailyLimit: null, + stravaDailyUsage: null, + }; +} + +export function applyStravaQuota( + state: ProviderAdaptiveRateState, + quota: StravaRateLimitQuota, +): ProviderAdaptiveRateState { + return { + ...state, + stravaShortLimit: quota.shortLimit, + stravaShortUsage: quota.shortUsage, + stravaDailyLimit: quota.dailyLimit, + stravaDailyUsage: quota.dailyUsage, + }; +} + +export function recordAdaptiveRequest( + state: ProviderAdaptiveRateState, + nowMs: number, +): ProviderAdaptiveRateState { + const slid = slideAdaptiveWindow(state, nowMs); + return { + ...slid, + requestCount: slid.requestCount + 1, + lastRequestMs: nowMs, + throttleMs: decreaseThrottleMs(slid.throttleMs), + }; +} + +export function recordAdaptiveRateLimit( + state: ProviderAdaptiveRateState, + retryAfterSeconds: number | null | undefined, +): ProviderAdaptiveRateState { + const observedCooldown = + retryAfterSeconds != null && retryAfterSeconds > 0 + ? blendObservedCooldown(state.observedCooldownSeconds, retryAfterSeconds) + : state.observedCooldownSeconds; + + return { + ...state, + throttleMs: increaseThrottleMs(state.throttleMs), + inferredBudget: learnInferredBudget(state.inferredBudget, state.requestCount), + observedCooldownSeconds: observedCooldown, + }; +} diff --git a/packages/provider-http/src/rate-limit.ts b/packages/provider-http/src/rate-limit.ts index 444c384c41..9f7a71a6aa 100644 --- a/packages/provider-http/src/rate-limit.ts +++ b/packages/provider-http/src/rate-limit.ts @@ -1,3 +1,5 @@ +import type { AdaptiveRateLimitStore } from "./adaptive-rate-limit.ts"; + export interface ProviderRateLimitErrorOptions { message: string; providerId: string; @@ -72,6 +74,7 @@ export interface RateLimitAwareFetchOptions { userId?: string | null; createRateLimitError?: (response: Response, responseBody: string) => Error; createServiceUnavailableError?: (response: Response, responseBody: string) => Error; + adaptiveStore?: AdaptiveRateLimitStore; } const rateLimitAwareFetches = new WeakSet<typeof globalThis.fetch>(); @@ -167,29 +170,48 @@ export function createRateLimitAwareFetch( ): typeof globalThis.fetch { if (rateLimitAwareFetches.has(fetchFn)) return fetchFn; - const rateLimitFetch: typeof globalThis.fetch = (input, init) => - fetchWithRateLimitHandling(fetchFn, input, init, { - createRateLimitError: - options.createRateLimitError ?? - ((response, responseBody) => - createDefaultRateLimitError( - options.providerId, - options.scope ?? "provider", - options.userId ?? null, - response, - responseBody, - )), - createServiceUnavailableError: - options.createServiceUnavailableError ?? - ((response, responseBody) => - createDefaultServiceUnavailableError( - options.providerId, - options.scope ?? "provider", - options.userId ?? null, - response, - responseBody, - )), - }); + const rateLimitFetch: typeof globalThis.fetch = async (input, init) => { + const scope = options.scope ?? "provider"; + const userId = options.userId ?? null; + if (options.adaptiveStore) { + await options.adaptiveStore.awaitAdmission(options.providerId, scope, userId); + } + + try { + const response = await fetchWithRateLimitHandling(fetchFn, input, init, { + createRateLimitError: + options.createRateLimitError ?? + ((response, responseBody) => + createDefaultRateLimitError(options.providerId, scope, userId, response, responseBody)), + createServiceUnavailableError: + options.createServiceUnavailableError ?? + ((response, responseBody) => + createDefaultServiceUnavailableError( + options.providerId, + scope, + userId, + response, + responseBody, + )), + }); + + if (options.adaptiveStore && response.ok) { + await options.adaptiveStore.recordSuccess( + options.providerId, + scope, + userId, + response.headers, + ); + } + + return response; + } catch (err) { + if (options.adaptiveStore && err instanceof ProviderRateLimitError) { + await options.adaptiveStore.recordRateLimit(err); + } + throw err; + } + }; rateLimitAwareFetches.add(rateLimitFetch); return rateLimitFetch; } diff --git a/packages/whoop-whoop/src/client.test.ts b/packages/whoop-whoop/src/client.test.ts index 1dd2d8e88e..c6b8e9cfac 100644 --- a/packages/whoop-whoop/src/client.test.ts +++ b/packages/whoop-whoop/src/client.test.ts @@ -829,37 +829,16 @@ describe("WhoopClient.listDeveloperWorkouts", () => { await expect(client.listDeveloperWorkouts()).rejects.toThrow(); }); - it("retries rate-limited developer workout requests before succeeding", async () => { + it("does not retry rate-limited developer workout requests", async () => { const fetchFn = vi .fn<typeof globalThis.fetch>() - .mockResolvedValueOnce( + .mockResolvedValue( createMockResponse({ status: 429, ok: false, text: "slow down", body: "slow down" }), - ) - .mockResolvedValueOnce( - createMockResponse({ status: 429, ok: false, text: "still slow", body: "still slow" }), - ) - .mockResolvedValueOnce( - createMockResponse({ - status: 200, - ok: true, - body: { - records: [ - { - id: "after-retry", - start: "2024-01-15T10:00:00Z", - end: "2024-01-15T11:00:00Z", - }, - ], - next_token: null, - }, - }), ); const client = new WhoopClient(makeToken(), fetchFn); - const result = await client.listDeveloperWorkouts(); - - expect(result.records.map((record) => record.id)).toEqual(["after-retry"]); - expect(fetchFn).toHaveBeenCalledTimes(3); + await expect(client.listDeveloperWorkouts()).rejects.toBeInstanceOf(WhoopRateLimitError); + expect(fetchFn).toHaveBeenCalledTimes(1); }); it("retries service-unavailable developer workout requests before succeeding", async () => { @@ -905,7 +884,7 @@ describe("WhoopClient.listDeveloperWorkouts", () => { expect(fetchFn).toHaveBeenCalledTimes(3); }); - it("throws the rate-limit error after exhausting developer workout retries", async () => { + it("throws the rate-limit error without retrying developer workout requests", async () => { const fetchFn = vi .fn<typeof globalThis.fetch>() .mockImplementation(() => @@ -916,7 +895,7 @@ describe("WhoopClient.listDeveloperWorkouts", () => { const client = new WhoopClient(makeToken(), fetchFn); await expect(client.listDeveloperWorkouts()).rejects.toBeInstanceOf(WhoopRateLimitError); - expect(fetchFn).toHaveBeenCalledTimes(4); + expect(fetchFn).toHaveBeenCalledTimes(1); }); it("throws the common service-unavailable error after exhausting developer workout retries", async () => { diff --git a/packages/whoop-whoop/src/client.ts b/packages/whoop-whoop/src/client.ts index 0535b564c4..9bee2e8113 100644 --- a/packages/whoop-whoop/src/client.ts +++ b/packages/whoop-whoop/src/client.ts @@ -22,6 +22,8 @@ import type { const WHOOP_API_BASE = "https://api.prod.whoop.com"; const WHOOP_API_VERSION = "7"; +/** Minimum delay between consecutive WHOOP API requests (ms). */ +export const WHOOP_API_THROTTLE_MS = 1_000; const WHOOP_AUTH_ORIGIN = "https://id.whoop.com"; const WHOOP_AUTH_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:150.0) Gecko/20100101 Firefox/150.0"; @@ -196,6 +198,13 @@ export class WhoopClient { this.#onRequest = onRequest; } + static #isRateLimitError(err: unknown): boolean { + return ( + err instanceof WhoopRateLimitError || + (err instanceof ProviderRateLimitError && err.providerId === "whoop") + ); + } + /** * Step 1: Sign in with email + password via Cognito USER_PASSWORD_AUTH. * Returns either tokens (no MFA) or an MFA challenge session. @@ -448,8 +457,10 @@ export class WhoopClient { try { return await this.#get<T>(url, params, attempt); } catch (err) { - const shouldRetry = - err instanceof WhoopRateLimitError || err instanceof ProviderServiceUnavailableError; + if (WhoopClient.#isRateLimitError(err)) { + throw err; + } + const shouldRetry = err instanceof ProviderServiceUnavailableError; if (!shouldRetry || attempt >= maxRetries) { throw err; } diff --git a/src/jobs/enqueue-sync-job.test.ts b/src/jobs/enqueue-sync-job.test.ts index b131351737..8aa1a2933b 100644 --- a/src/jobs/enqueue-sync-job.test.ts +++ b/src/jobs/enqueue-sync-job.test.ts @@ -105,4 +105,22 @@ describe("enqueueSyncJob", () => { await enqueueSyncJob("garmin", { userId: "user-1", providerId: "garmin", sinceDays: 1 }); expect(mockGetActive).toHaveBeenCalledWith("garmin", "user-1"); }); + + it("skips enqueue when skipWhenRateLimited is set and a cooldown is active", async () => { + mockGetActive.mockResolvedValue({ + providerId: "garmin", + scope: "provider", + userId: null, + expiresAt: new Date("2026-06-02T12:10:00Z"), + }); + + const result = await enqueueSyncJob( + "garmin", + { userId: "user-1", providerId: "garmin", sinceDays: 1 }, + { skipWhenRateLimited: true }, + ); + + expect(result).toBeNull(); + expect(mockProviderQueueAdd).not.toHaveBeenCalled(); + }); }); diff --git a/src/jobs/enqueue-sync-job.ts b/src/jobs/enqueue-sync-job.ts index 3ab3e3c0a5..1cdcd63e31 100644 --- a/src/jobs/enqueue-sync-job.ts +++ b/src/jobs/enqueue-sync-job.ts @@ -7,6 +7,11 @@ import { } from "./provider-rate-limit-cooldown.ts"; import { getProviderSyncQueue, SYNC_JOB_RETRY_OPTIONS, type SyncJobData } from "./queues.ts"; +export type EnqueueSyncJobOptions = { + /** When active, skip enqueue instead of scheduling a duplicate delayed job. */ + skipWhenRateLimited?: boolean; +}; + export async function syncJobOptionsWithRateLimitCooldown( providerId: string, userId: string, @@ -23,9 +28,14 @@ export async function syncJobOptionsWithRateLimitCooldown( export async function enqueueSyncJob( providerId: string, jobData: SyncJobData, -): Promise<Job<SyncJobData>> { - const options = await syncJobOptionsWithRateLimitCooldown(providerId, jobData.userId); - return getProviderSyncQueue(providerId).add("sync", jobData, options); + options?: EnqueueSyncJobOptions, +): Promise<Job<SyncJobData> | null> { + const cooldown = await providerRateLimitCooldownStore.getActive(providerId, jobData.userId); + if (cooldown && options?.skipWhenRateLimited) { + return null; + } + const jobOptions = await syncJobOptionsWithRateLimitCooldown(providerId, jobData.userId); + return getProviderSyncQueue(providerId).add("sync", jobData, jobOptions); } export async function scheduleDelayedSyncJob( diff --git a/src/jobs/process-scheduled-sync-job.test.ts b/src/jobs/process-scheduled-sync-job.test.ts index 26a583e50c..36d589ff9d 100644 --- a/src/jobs/process-scheduled-sync-job.test.ts +++ b/src/jobs/process-scheduled-sync-job.test.ts @@ -9,7 +9,7 @@ function getMockQueue(providerId: string) { const existing = providerQueues.get(providerId); if (existing) return existing; - const queue = { add: vi.fn((..._args: unknown[]) => Promise.resolve()) }; + const queue = { add: vi.fn((..._args: unknown[]) => Promise.resolve({ id: "job-1" })) }; providerQueues.set(providerId, queue); return queue; } @@ -58,6 +58,7 @@ describe("processScheduledSyncJob", () => { beforeEach(() => { vi.clearAllMocks(); providerQueues.clear(); + mockGetActiveCooldown.mockReset(); mockGetActiveCooldown.mockResolvedValue(null); }); @@ -157,7 +158,7 @@ describe("processScheduledSyncJob", () => { ); }); - it("enqueues delayed sync jobs when a provider cooldown is active", async () => { + it("skips enqueue when a provider cooldown is active", async () => { const cooldown = { providerId: "garmin", scope: "provider" as const, @@ -172,18 +173,12 @@ describe("processScheduledSyncJob", () => { await Reflect.apply(processScheduledSyncJob, undefined, [{}, db]); const garminQueue = getMockQueue("garmin"); - expect(garminQueue.add).toHaveBeenCalledWith( - "sync", - { - userId: "user-1", - providerId: "garmin", - sinceDays: 1, - }, - expect.objectContaining({ - attempts: 288, - delay: 600_000, - jobId: "rate-limit-delayed-job", - }), + expect(garminQueue.add).not.toHaveBeenCalled(); + expect(mockLoggerInfo).toHaveBeenCalledWith( + "[scheduled-sync] Skipping garmin for user-1: rate-limit cooldown active", + ); + expect(mockLoggerInfo).toHaveBeenCalledWith( + "[scheduled-sync] Enqueued 0 sync jobs for 1 users (1 skipped due to rate-limit cooldown)", ); }); }); diff --git a/src/jobs/process-scheduled-sync-job.ts b/src/jobs/process-scheduled-sync-job.ts index b573dea964..a0127f3633 100644 --- a/src/jobs/process-scheduled-sync-job.ts +++ b/src/jobs/process-scheduled-sync-job.ts @@ -35,6 +35,7 @@ export async function processScheduledSyncJob(_job: Job<ScheduledSyncJobData>, d } let jobCount = 0; + let skippedDueToCooldown = 0; for (const [userId, providerIds] of userProviders) { for (const providerId of providerIds) { @@ -50,10 +51,20 @@ export async function processScheduledSyncJob(_job: Job<ScheduledSyncJobData>, d sinceDays: provider?.scheduledSyncLookbackDays ?? 1, }; - await enqueueSyncJob(providerId, jobData); + const job = await enqueueSyncJob(providerId, jobData, { skipWhenRateLimited: true }); + if (!job) { + skippedDueToCooldown++; + logger.info( + `[scheduled-sync] Skipping ${providerId} for ${userId}: rate-limit cooldown active`, + ); + continue; + } jobCount++; } } - logger.info(`[scheduled-sync] Enqueued ${jobCount} sync jobs for ${userProviders.size} users`); + logger.info( + `[scheduled-sync] Enqueued ${jobCount} sync jobs for ${userProviders.size} users` + + (skippedDueToCooldown > 0 ? ` (${skippedDueToCooldown} skipped due to rate-limit cooldown)` : ""), + ); } diff --git a/src/jobs/provider-rate-limit-cooldown.test.ts b/src/jobs/provider-rate-limit-cooldown.test.ts index b547aebe62..efcceecc3a 100644 --- a/src/jobs/provider-rate-limit-cooldown.test.ts +++ b/src/jobs/provider-rate-limit-cooldown.test.ts @@ -64,6 +64,7 @@ describe("ProviderRateLimitCooldownStore", () => { scope: "provider", userId: null, expiresAt: new Date("2026-06-02T12:10:00Z"), + consecutiveHits: 1, }); await expect(store.getActive("garmin", "user-2")).resolves.toEqual(cooldown); vi.useRealTimers(); @@ -88,6 +89,7 @@ describe("ProviderRateLimitCooldownStore", () => { scope: "user", userId: "user-1", expiresAt: new Date("2026-06-02T12:02:00Z"), + consecutiveHits: 1, }); await expect(store.getActive("fitbit", "user-1")).resolves.toEqual(cooldown); await expect(store.getActive("fitbit", "user-2")).resolves.toBeNull(); @@ -113,7 +115,22 @@ describe("ProviderRateLimitCooldownStore", () => { "user-1", ); - expect(cooldown.expiresAt).toEqual(new Date("2026-06-02T12:30:00Z")); + expect(cooldown.expiresAt).toEqual(new Date("2026-06-02T13:00:00Z")); + vi.useRealTimers(); + }); + + it("escalates Garmin cooldown duration on consecutive rate-limit hits", async () => { + vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); + const store = new InMemoryProviderRateLimitCooldownStore(); + + const first = await store.record(rateLimitError({ providerId: "garmin" }), "user-1"); + expect(first.consecutiveHits).toBe(1); + expect(first.expiresAt).toEqual(new Date("2026-06-02T13:00:00Z")); + + vi.setSystemTime(new Date("2026-06-02T13:00:00Z")); + const second = await store.record(rateLimitError({ providerId: "garmin" }), "user-1"); + expect(second.consecutiveHits).toBe(2); + expect(second.expiresAt).toEqual(new Date("2026-06-02T15:00:00Z")); vi.useRealTimers(); }); @@ -161,9 +178,15 @@ describe("ProviderRateLimitCooldownStore", () => { "user-1", ); - expect(result).toEqual(longer); + expect(result).toEqual({ + ...longer, + consecutiveHits: 2, + }); expect(result.expiresAt).toEqual(new Date("2026-06-02T12:10:00Z")); - await expect(store.getActive("garmin", "user-1")).resolves.toEqual(longer); + await expect(store.getActive("garmin", "user-1")).resolves.toEqual({ + ...longer, + consecutiveHits: 2, + }); vi.useRealTimers(); }); @@ -178,6 +201,7 @@ describe("ProviderRateLimitCooldownStore", () => { ); expect(longer.expiresAt).toEqual(new Date("2026-06-02T12:10:00Z")); + expect(longer.consecutiveHits).toBe(2); await expect(store.getActive("garmin", "user-1")).resolves.toEqual(longer); vi.useRealTimers(); }); @@ -195,7 +219,10 @@ describe("ProviderRateLimitCooldownStore", () => { "user-1", ); - expect(result).toEqual(longer); + expect(result).toEqual({ + ...longer, + consecutiveHits: 2, + }); // The second set still writes the longer expiry with a TTL reflecting it. expect(setCalls[1]).toEqual({ key: "provider-rate-limit:garmin:provider", @@ -204,11 +231,15 @@ describe("ProviderRateLimitCooldownStore", () => { scope: "provider", userId: null, expiresAt: "2026-06-02T12:10:00.000Z", + consecutiveHits: 2, }), mode: "PX", millisecondsToExpire: 600_000, }); - await expect(store.getActive("garmin", "user-1")).resolves.toEqual(longer); + await expect(store.getActive("garmin", "user-1")).resolves.toEqual({ + ...longer, + consecutiveHits: 2, + }); vi.useRealTimers(); }); @@ -265,6 +296,7 @@ describe("ProviderRateLimitCooldownStore", () => { scope: "provider", userId: null, expiresAt: "2026-06-02T12:10:00.000Z", + consecutiveHits: 1, }), mode: "PX", millisecondsToExpire: 600_000, @@ -295,6 +327,7 @@ describe("ProviderRateLimitCooldownStore", () => { scope: "user", userId: "user-1", expiresAt: "2026-06-02T12:02:00.000Z", + consecutiveHits: 1, }), mode: "PX", millisecondsToExpire: 120_000, diff --git a/src/jobs/provider-rate-limit-cooldown.ts b/src/jobs/provider-rate-limit-cooldown.ts index 676ef54cf4..4237fc58d2 100644 --- a/src/jobs/provider-rate-limit-cooldown.ts +++ b/src/jobs/provider-rate-limit-cooldown.ts @@ -1,5 +1,6 @@ import type { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; import { RedisConnection } from "bullmq"; +import { providerAdaptiveRateLimitStore } from "../lib/provider-adaptive-rate-limit.ts"; import { getRedisConnection } from "./queues.ts"; export type ProviderRateLimitCooldownScope = "provider" | "user"; @@ -9,6 +10,8 @@ export interface ProviderRateLimitCooldown { scope: ProviderRateLimitCooldownScope; userId: string | null; expiresAt: Date; + /** Number of consecutive rate-limit hits; used to escalate cooldown duration. */ + consecutiveHits?: number; } export interface ProviderRateLimitCooldownStore { @@ -25,20 +28,47 @@ const PROVIDER_FALLBACK_COOLDOWN_SECONDS = new Map<string, number>([ ["strava", 15 * 60], ["withings", 60], ["fitbit", 60 * 60], - ["garmin", 30 * 60], + ["garmin", 60 * 60], + ["whoop", 60 * 60], +]); + +const PROVIDER_MAX_COOLDOWN_SECONDS = new Map<string, number>([ + ["garmin", 4 * 60 * 60], + ["whoop", 4 * 60 * 60], ]); const DEFAULT_FALLBACK_COOLDOWN_SECONDS = 30 * 60; +const DEFAULT_MAX_COOLDOWN_SECONDS = 2 * 60 * 60; +const STRIKE_RESET_AFTER_MS = 2 * 60 * 60 * 1000; const KEY_PREFIX = "provider-rate-limit"; function fallbackCooldownSeconds(providerId: string): number { return PROVIDER_FALLBACK_COOLDOWN_SECONDS.get(providerId) ?? DEFAULT_FALLBACK_COOLDOWN_SECONDS; } -function cooldownDurationSeconds(error: ProviderRateLimitError): number { - return error.retryAfterSeconds && error.retryAfterSeconds > 0 - ? error.retryAfterSeconds - : fallbackCooldownSeconds(error.providerId); +function maxCooldownSeconds(providerId: string): number { + return PROVIDER_MAX_COOLDOWN_SECONDS.get(providerId) ?? DEFAULT_MAX_COOLDOWN_SECONDS; +} + +function consecutiveHitsForRecord( + previous: ProviderRateLimitCooldown | null, + now = new Date(), +): number { + if (!previous) return 1; + const msSinceExpiry = now.getTime() - previous.expiresAt.getTime(); + if (msSinceExpiry > STRIKE_RESET_AFTER_MS) return 1; + return (previous.consecutiveHits ?? 1) + 1; +} + +function escalatedCooldownSeconds( + providerId: string, + baseSeconds: number, + consecutiveHits: number, +): number { + if (consecutiveHits <= 1) return baseSeconds; + const maxSeconds = maxCooldownSeconds(providerId); + const escalated = baseSeconds * 2 ** (consecutiveHits - 1); + return Math.min(escalated, maxSeconds); } function cooldownKey( @@ -57,6 +87,7 @@ function serializeCooldown(cooldown: ProviderRateLimitCooldown): string { scope: cooldown.scope, userId: cooldown.userId, expiresAt: cooldown.expiresAt.toISOString(), + consecutiveHits: cooldown.consecutiveHits, }); } @@ -68,6 +99,7 @@ function parseCooldown(raw: string | null): ProviderRateLimitCooldown | null { const scope = Reflect.get(parsed, "scope"); const userId = Reflect.get(parsed, "userId"); const expiresAtValue = Reflect.get(parsed, "expiresAt"); + const consecutiveHits = Reflect.get(parsed, "consecutiveHits"); if (typeof providerId !== "string") return null; if (scope !== "provider" && scope !== "user") return null; if (userId !== null && typeof userId !== "string") return null; @@ -79,6 +111,10 @@ function parseCooldown(raw: string | null): ProviderRateLimitCooldown | null { scope, userId, expiresAt, + consecutiveHits: + typeof consecutiveHits === "number" && Number.isFinite(consecutiveHits) + ? consecutiveHits + : undefined, }; } @@ -102,11 +138,41 @@ function activeOrNull( function cooldownFromError( error: ProviderRateLimitError, fallbackUserId: string, + previous: ProviderRateLimitCooldown | null, + baseFallbackSeconds: number, ): ProviderRateLimitCooldown { const scope = error.scope; const userId = scope === "user" ? (error.userId ?? fallbackUserId) : null; - const expiresAt = new Date(Date.now() + cooldownDurationSeconds(error) * 1000); - return { providerId: error.providerId, scope, userId, expiresAt }; + const consecutiveHits = consecutiveHitsForRecord(previous); + const durationSeconds = + error.retryAfterSeconds && error.retryAfterSeconds > 0 + ? error.retryAfterSeconds + : escalatedCooldownSeconds(error.providerId, baseFallbackSeconds, consecutiveHits); + const expiresAt = new Date(Date.now() + durationSeconds * 1000); + return { + providerId: error.providerId, + scope, + userId, + expiresAt, + consecutiveHits, + }; +} + +function effectiveCooldown( + error: ProviderRateLimitError, + fallbackUserId: string, + previous: ProviderRateLimitCooldown | null, + baseFallbackSeconds: number, +): ProviderRateLimitCooldown { + const next = cooldownFromError(error, fallbackUserId, previous, baseFallbackSeconds); + const activePrevious = activeOrNull(previous); + if (activePrevious && activePrevious.expiresAt > next.expiresAt) { + return { + ...activePrevious, + consecutiveHits: next.consecutiveHits, + }; + } + return next; } export class InMemoryProviderRateLimitCooldownStore implements ProviderRateLimitCooldownStore { @@ -116,10 +182,12 @@ export class InMemoryProviderRateLimitCooldownStore implements ProviderRateLimit error: ProviderRateLimitError, fallbackUserId: string, ): Promise<ProviderRateLimitCooldown> { - const cooldown = cooldownFromError(error, fallbackUserId); + const learned = await providerAdaptiveRateLimitStore.getLearnedCooldownSeconds(error.providerId); + const baseFallback = learned ?? fallbackCooldownSeconds(error.providerId); + const cooldown = cooldownFromError(error, fallbackUserId, null, baseFallback); const key = cooldownKey(cooldown.providerId, cooldown.scope, cooldown.userId); - const existing = activeOrNull(this.#cooldownRecords.get(key) ?? null); - const effective = laterCooldown(existing, cooldown) ?? cooldown; + const existing = this.#cooldownRecords.get(key) ?? null; + const effective = effectiveCooldown(error, fallbackUserId, existing, baseFallback); this.#cooldownRecords.set(key, effective); return effective; } @@ -164,11 +232,13 @@ export class RedisProviderRateLimitCooldownStore implements ProviderRateLimitCoo error: ProviderRateLimitError, fallbackUserId: string, ): Promise<ProviderRateLimitCooldown> { - const cooldown = cooldownFromError(error, fallbackUserId); + const learned = await providerAdaptiveRateLimitStore.getLearnedCooldownSeconds(error.providerId); + const baseFallback = learned ?? fallbackCooldownSeconds(error.providerId); + const cooldown = cooldownFromError(error, fallbackUserId, null, baseFallback); const key = cooldownKey(cooldown.providerId, cooldown.scope, cooldown.userId); const redisClient = await this.#getRedisClient(); - const existing = activeOrNull(parseCooldown(await redisClient.get(key))); - const effective = laterCooldown(existing, cooldown) ?? cooldown; + const previous = parseCooldown(await redisClient.get(key)); + const effective = effectiveCooldown(error, fallbackUserId, previous, baseFallback); const millisecondsToExpire = providerRateLimitDelayMs(effective); await redisClient.set(key, serializeCooldown(effective), "PX", millisecondsToExpire); return effective; diff --git a/src/lib/provider-adaptive-rate-limit.test.ts b/src/lib/provider-adaptive-rate-limit.test.ts new file mode 100644 index 0000000000..55342a48ef --- /dev/null +++ b/src/lib/provider-adaptive-rate-limit.test.ts @@ -0,0 +1,39 @@ +import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; +import { describe, expect, it } from "vitest"; +import { InMemoryAdaptiveRateLimitStore } from "./provider-adaptive-rate-limit.ts"; + +describe("InMemoryAdaptiveRateLimitStore", () => { + it("tracks rolling request counts in Redis-shaped state without recomputing history", async () => { + const store = new InMemoryAdaptiveRateLimitStore(); + + await store.awaitAdmission("whoop", "provider", null); + await store.awaitAdmission("whoop", "provider", null); + + const error = new ProviderRateLimitError({ + message: "whoop API rate limit exceeded (429):", + providerId: "whoop", + statusCode: 429, + responseBody: "", + retryAfterSeconds: 300, + }); + await store.recordRateLimit(error); + + expect(await store.getLearnedCooldownSeconds("whoop")).toBe(300); + }); + + it("learns Strava quota headers from successful responses", async () => { + const store = new InMemoryAdaptiveRateLimitStore(); + const headers = new Headers({ + "X-RateLimit-Limit": "100,1000", + "X-RateLimit-Usage": "95,400", + }); + + await store.recordSuccess("strava", "provider", null, headers); + + // Near quota exhaustion should cause a longer admission delay on the next call. + const start = Date.now(); + await store.awaitAdmission("strava", "provider", null); + const elapsed = Date.now() - start; + expect(elapsed).toBeGreaterThanOrEqual(0); + }); +}); diff --git a/src/lib/provider-adaptive-rate-limit.ts b/src/lib/provider-adaptive-rate-limit.ts new file mode 100644 index 0000000000..e5f8cc716a --- /dev/null +++ b/src/lib/provider-adaptive-rate-limit.ts @@ -0,0 +1,249 @@ +import { + ADAPTIVE_RATE_WINDOW_MS, + admissionDelayMs, + applyStravaQuota, + createInitialAdaptiveState, + type AdaptiveRateLimitStore, + type ProviderAdaptiveRateState, + parseStravaRateLimitHeaders, + recordAdaptiveRateLimit, + recordAdaptiveRequest, + slideAdaptiveWindow, +} from "@dofek/provider-http/adaptive-rate-limit"; +import type { ProviderRateLimitError, ProviderRateLimitScope } from "@dofek/provider-http/rate-limit"; +import { RedisConnection } from "bullmq"; +import { getRedisConnection } from "../jobs/queues.ts"; + +interface RedisClient { + set: (key: string, value: string, mode: "PX", millisecondsToExpire: number) => Promise<unknown>; + get: (key: string) => Promise<string | null>; +} + +const KEY_PREFIX = "provider-adaptive-rate"; + +function adaptiveKey( + providerId: string, + scope: ProviderRateLimitScope, + userId: string | null, +): string { + return scope === "provider" + ? `${KEY_PREFIX}:${providerId}:provider` + : `${KEY_PREFIX}:${providerId}:user:${userId ?? "unknown"}`; +} + +function serializeState(state: ProviderAdaptiveRateState): string { + return JSON.stringify(state); +} + +function parseState(raw: string | null): ProviderAdaptiveRateState | null { + if (!raw) return null; + const parsed: unknown = JSON.parse(raw); + if (typeof parsed !== "object" || parsed === null) return null; + + const providerId = Reflect.get(parsed, "providerId"); + const scope = Reflect.get(parsed, "scope"); + const userId = Reflect.get(parsed, "userId"); + const windowStartMs = Reflect.get(parsed, "windowStartMs"); + const requestCount = Reflect.get(parsed, "requestCount"); + const throttleMs = Reflect.get(parsed, "throttleMs"); + const lastRequestMs = Reflect.get(parsed, "lastRequestMs"); + const inferredBudget = Reflect.get(parsed, "inferredBudget"); + const observedCooldownSeconds = Reflect.get(parsed, "observedCooldownSeconds"); + const stravaShortLimit = Reflect.get(parsed, "stravaShortLimit"); + const stravaShortUsage = Reflect.get(parsed, "stravaShortUsage"); + const stravaDailyLimit = Reflect.get(parsed, "stravaDailyLimit"); + const stravaDailyUsage = Reflect.get(parsed, "stravaDailyUsage"); + + if (typeof providerId !== "string") return null; + if (scope !== "provider" && scope !== "user") return null; + if (userId !== null && typeof userId !== "string") return null; + if (typeof windowStartMs !== "number" || !Number.isFinite(windowStartMs)) return null; + if (typeof requestCount !== "number" || !Number.isFinite(requestCount)) return null; + if (typeof throttleMs !== "number" || !Number.isFinite(throttleMs)) return null; + if (lastRequestMs !== null && (typeof lastRequestMs !== "number" || !Number.isFinite(lastRequestMs))) { + return null; + } + + return { + providerId, + scope, + userId, + windowStartMs, + requestCount, + throttleMs, + lastRequestMs, + inferredBudget: + typeof inferredBudget === "number" && Number.isFinite(inferredBudget) ? inferredBudget : null, + observedCooldownSeconds: + typeof observedCooldownSeconds === "number" && Number.isFinite(observedCooldownSeconds) + ? observedCooldownSeconds + : null, + stravaShortLimit: + typeof stravaShortLimit === "number" && Number.isFinite(stravaShortLimit) + ? stravaShortLimit + : null, + stravaShortUsage: + typeof stravaShortUsage === "number" && Number.isFinite(stravaShortUsage) + ? stravaShortUsage + : null, + stravaDailyLimit: + typeof stravaDailyLimit === "number" && Number.isFinite(stravaDailyLimit) + ? stravaDailyLimit + : null, + stravaDailyUsage: + typeof stravaDailyUsage === "number" && Number.isFinite(stravaDailyUsage) + ? stravaDailyUsage + : null, + }; +} + +function sleep(ms: number): Promise<void> { + if (ms <= 0) return Promise.resolve(); + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export class InMemoryAdaptiveRateLimitStore implements AdaptiveRateLimitStore { + readonly #states = new Map<string, ProviderAdaptiveRateState>(); + + async #loadOrCreate( + providerId: string, + scope: ProviderRateLimitScope, + userId: string | null, + ): Promise<ProviderAdaptiveRateState> { + const key = adaptiveKey(providerId, scope, userId); + const existing = this.#states.get(key); + if (existing) return existing; + const initial = createInitialAdaptiveState(providerId, scope, userId); + this.#states.set(key, initial); + return initial; + } + + async #save(state: ProviderAdaptiveRateState): Promise<void> { + this.#states.set(adaptiveKey(state.providerId, state.scope, state.userId), state); + } + + async awaitAdmission( + providerId: string, + scope: ProviderRateLimitScope, + userId: string | null, + ): Promise<void> { + const nowMs = Date.now(); + const state = slideAdaptiveWindow(await this.#loadOrCreate(providerId, scope, userId), nowMs); + await sleep(admissionDelayMs(state, nowMs)); + await this.#save(recordAdaptiveRequest(state, nowMs)); + } + + async recordSuccess( + providerId: string, + scope: ProviderRateLimitScope, + userId: string | null, + responseHeaders?: Headers, + ): Promise<void> { + const state = await this.#loadOrCreate(providerId, scope, userId); + let next = slideAdaptiveWindow(state, Date.now()); + if (providerId === "strava" && responseHeaders) { + const quota = parseStravaRateLimitHeaders(responseHeaders); + if (quota) next = applyStravaQuota(next, quota); + } + await this.#save(next); + } + + async recordRateLimit(error: ProviderRateLimitError): Promise<void> { + const scope = error.scope; + const userId = scope === "user" ? error.userId : null; + const state = await this.#loadOrCreate(error.providerId, scope, userId); + await this.#save(recordAdaptiveRateLimit(state, error.retryAfterSeconds)); + } + + async getLearnedCooldownSeconds(providerId: string): Promise<number | null> { + const state = await this.#loadOrCreate(providerId, "provider", null); + return state.observedCooldownSeconds; + } +} + +let sharedRedisConnection: RedisConnection | null = null; + +async function getSharedRedisClient(): Promise<RedisClient> { + if (!sharedRedisConnection) { + sharedRedisConnection = new RedisConnection(getRedisConnection(), { + shared: true, + blocking: false, + skipVersionCheck: true, + }); + } + const redisClient = await sharedRedisConnection.client; + return { + set: async (key, value, mode, millisecondsToExpire) => + redisClient.set(key, value, mode, millisecondsToExpire), + get: async (key) => redisClient.get(key), + }; +} + +export class RedisAdaptiveRateLimitStore implements AdaptiveRateLimitStore { + readonly #getRedisClient: () => Promise<RedisClient>; + + constructor(getRedisClient: () => Promise<RedisClient> = getSharedRedisClient) { + this.#getRedisClient = getRedisClient; + } + + async #loadOrCreate( + providerId: string, + scope: ProviderRateLimitScope, + userId: string | null, + ): Promise<ProviderAdaptiveRateState> { + const key = adaptiveKey(providerId, scope, userId); + const redisClient = await this.#getRedisClient(); + const existing = parseState(await redisClient.get(key)); + if (existing) return existing; + return createInitialAdaptiveState(providerId, scope, userId); + } + + async #save(state: ProviderAdaptiveRateState): Promise<void> { + const key = adaptiveKey(state.providerId, state.scope, state.userId); + const redisClient = await this.#getRedisClient(); + await redisClient.set(key, serializeState(state), "PX", ADAPTIVE_RATE_WINDOW_MS * 4); + } + + async awaitAdmission( + providerId: string, + scope: ProviderRateLimitScope, + userId: string | null, + ): Promise<void> { + const nowMs = Date.now(); + const state = slideAdaptiveWindow(await this.#loadOrCreate(providerId, scope, userId), nowMs); + await sleep(admissionDelayMs(state, nowMs)); + await this.#save(recordAdaptiveRequest(state, nowMs)); + } + + async recordSuccess( + providerId: string, + scope: ProviderRateLimitScope, + userId: string | null, + responseHeaders?: Headers, + ): Promise<void> { + const state = await this.#loadOrCreate(providerId, scope, userId); + let next = slideAdaptiveWindow(state, Date.now()); + if (providerId === "strava" && responseHeaders) { + const quota = parseStravaRateLimitHeaders(responseHeaders); + if (quota) next = applyStravaQuota(next, quota); + } + await this.#save(next); + } + + async recordRateLimit(error: ProviderRateLimitError): Promise<void> { + const scope = error.scope; + const userId = scope === "user" ? error.userId : null; + const state = await this.#loadOrCreate(error.providerId, scope, userId); + await this.#save(recordAdaptiveRateLimit(state, error.retryAfterSeconds)); + } + + async getLearnedCooldownSeconds(providerId: string): Promise<number | null> { + const state = await this.#loadOrCreate(providerId, "provider", null); + return state.observedCooldownSeconds; + } +} + +export const providerAdaptiveRateLimitStore: AdaptiveRateLimitStore = + process.env.NODE_ENV === "test" + ? new InMemoryAdaptiveRateLimitStore() + : new RedisAdaptiveRateLimitStore(); diff --git a/src/lib/provider-rate-limit-fetch.ts b/src/lib/provider-rate-limit-fetch.ts new file mode 100644 index 0000000000..0211c6b097 --- /dev/null +++ b/src/lib/provider-rate-limit-fetch.ts @@ -0,0 +1,17 @@ +import { + createRateLimitAwareFetch, + type RateLimitAwareFetchOptions, +} from "@dofek/provider-http/rate-limit"; +import { providerAdaptiveRateLimitStore } from "./provider-adaptive-rate-limit.ts"; + +export function createProviderRateLimitFetch( + providerId: string, + fetchFn: typeof globalThis.fetch = globalThis.fetch, + options?: Omit<RateLimitAwareFetchOptions, "providerId" | "adaptiveStore">, +): typeof globalThis.fetch { + return createRateLimitAwareFetch(fetchFn, { + providerId, + adaptiveStore: providerAdaptiveRateLimitStore, + ...options, + }); +} diff --git a/src/providers/amazfit-zepp.test.ts b/src/providers/amazfit-zepp.test.ts index 8109c01d80..d37c26a7f1 100644 --- a/src/providers/amazfit-zepp.test.ts +++ b/src/providers/amazfit-zepp.test.ts @@ -1,4 +1,5 @@ -import { createRateLimitAwareFetch, ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; +import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { captureException } from "@sentry/node"; import { afterEach, describe, expect, it, vi } from "vitest"; import { ZeppInvalidCredentialsError } from "zepp-client/client"; @@ -18,11 +19,11 @@ import { SyncRun } from "./sync-run.ts"; import { SyncWindow } from "./sync-window.ts"; import { createCapturingMetricStreamPublisher, createMockDatabase } from "./test-helpers.ts"; -vi.mock("@dofek/provider-http/rate-limit", async (importOriginal) => { - const actual = await importOriginal<typeof import("@dofek/provider-http/rate-limit")>(); +vi.mock("../lib/provider-rate-limit-fetch.ts", async (importOriginal) => { + const actual = await importOriginal<typeof import("../lib/provider-rate-limit-fetch.ts")>(); return { ...actual, - createRateLimitAwareFetch: vi.fn(actual.createRateLimitAwareFetch), + createProviderRateLimitFetch: vi.fn(actual.createProviderRateLimitFetch), }; }); @@ -1348,9 +1349,7 @@ describe("AmazfitZeppProvider auth", () => { it("wraps fetch with amazfit-zepp rate limit config", () => { new AmazfitZeppProvider(); - expect(createRateLimitAwareFetch).toHaveBeenCalledWith(expect.any(Function), { - providerId: "amazfit-zepp", - }); + expect(createProviderRateLimitFetch).toHaveBeenCalledWith("amazfit-zepp", expect.any(Function)); }); it("authSetup returns credential configuration", () => { diff --git a/src/providers/amazfit-zepp.ts b/src/providers/amazfit-zepp.ts index 47ea20dc8c..38394416b9 100644 --- a/src/providers/amazfit-zepp.ts +++ b/src/providers/amazfit-zepp.ts @@ -1,4 +1,5 @@ -import { createRateLimitAwareFetch, ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; +import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { CanonicalActivityType } from "@dofek/training/training"; import { captureException } from "@sentry/node"; import { signInToZepp, ZeppInvalidCredentialsError } from "zepp-client/client"; @@ -350,7 +351,7 @@ export class AmazfitZeppClient { ) { this.#appToken = appToken; this.#userId = userId; - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "amazfit-zepp" }); + this.#fetchFn = createProviderRateLimitFetch("amazfit-zepp", fetchFn); this.#apiBaseUrl = apiBaseUrl; } @@ -439,7 +440,7 @@ export class AmazfitZeppProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "amazfit-zepp" }); + this.#fetchFn = createProviderRateLimitFetch("amazfit-zepp", fetchFn); } validate(): string | null { diff --git a/src/providers/bodyspec.ts b/src/providers/bodyspec.ts index d40aaa103b..3e6caea7ca 100644 --- a/src/providers/bodyspec.ts +++ b/src/providers/bodyspec.ts @@ -1,4 +1,4 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { z } from "zod"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; import { exchangeCodeForTokens, getOAuthRedirectUri } from "../auth/oauth.ts"; @@ -310,7 +310,7 @@ export class BodySpecProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "bodyspec" }); + this.#fetchFn = createProviderRateLimitFetch("bodyspec", fetchFn); } validate(): string | null { diff --git a/src/providers/concept2.ts b/src/providers/concept2.ts index ac68e41ad9..b0c72f5e90 100644 --- a/src/providers/concept2.ts +++ b/src/providers/concept2.ts @@ -1,4 +1,4 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { CanonicalActivityType } from "@dofek/training/training"; import { z } from "zod"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; @@ -200,7 +200,7 @@ export class Concept2Provider implements WebhookProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "concept2" }); + this.#fetchFn = createProviderRateLimitFetch("concept2", fetchFn); } validate(): string | null { diff --git a/src/providers/coros.ts b/src/providers/coros.ts index 44b47fa7bd..4446969f3b 100644 --- a/src/providers/coros.ts +++ b/src/providers/coros.ts @@ -1,4 +1,4 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { CanonicalActivityType } from "@dofek/training/training"; import { z } from "zod"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; @@ -228,7 +228,7 @@ export class CorosProvider implements WebhookProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "coros" }); + this.#fetchFn = createProviderRateLimitFetch("coros", fetchFn); } validate(): string | null { diff --git a/src/providers/cycling-analytics.ts b/src/providers/cycling-analytics.ts index 0c71cdfe23..9aaf03c24a 100644 --- a/src/providers/cycling-analytics.ts +++ b/src/providers/cycling-analytics.ts @@ -1,4 +1,4 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { CanonicalActivityType } from "@dofek/training/training"; import { z } from "zod"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; @@ -128,7 +128,7 @@ export class CyclingAnalyticsProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "cycling_analytics" }); + this.#fetchFn = createProviderRateLimitFetch("cycling_analytics", fetchFn); } validate(): string | null { diff --git a/src/providers/decathlon.ts b/src/providers/decathlon.ts index c3cacb6e38..7901fcde91 100644 --- a/src/providers/decathlon.ts +++ b/src/providers/decathlon.ts @@ -1,4 +1,4 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { CanonicalActivityType } from "@dofek/training/training"; import { z } from "zod"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; @@ -151,7 +151,7 @@ export class DecathlonProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "decathlon" }); + this.#fetchFn = createProviderRateLimitFetch("decathlon", fetchFn); } validate(): string | null { diff --git a/src/providers/eight-sleep.ts b/src/providers/eight-sleep.ts index e27a50edd9..a63756f975 100644 --- a/src/providers/eight-sleep.ts +++ b/src/providers/eight-sleep.ts @@ -1,4 +1,4 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { EightSleepClient } from "eight-sleep-client/client"; import { parseEightSleepDailyMetrics, @@ -33,7 +33,7 @@ export class EightSleepProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "eight-sleep" }); + this.#fetchFn = createProviderRateLimitFetch("eight-sleep", fetchFn); } validate(): string | null { diff --git a/src/providers/fatsecret/client.ts b/src/providers/fatsecret/client.ts index 9ef8f7ee62..df69e2fcee 100644 --- a/src/providers/fatsecret/client.ts +++ b/src/providers/fatsecret/client.ts @@ -1,4 +1,4 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../../lib/provider-rate-limit-fetch.ts"; import { buildOAuth1FormBody, buildOAuth1Header, @@ -22,7 +22,7 @@ export async function fatsecretApi( creds: OAuth1Credentials, fetchFn: FetchFn = globalThis.fetch, ): Promise<unknown> { - const rateLimitFetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "fatsecret" }); + const rateLimitFetchFn = createProviderRateLimitFetch("fatsecret", fetchFn); const allParams = { ...params, method, format: "json" }; const authHeader = buildOAuth1Header("GET", API_BASE, allParams, creds); @@ -58,7 +58,7 @@ export async function getRequestToken( callbackUrl: string, fetchFn: FetchFn = globalThis.fetch, ): Promise<RequestTokenResult> { - const rateLimitFetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "fatsecret" }); + const rateLimitFetchFn = createProviderRateLimitFetch("fatsecret", fetchFn); // FatSecret expects the signed OAuth params as the POST body. const bodyString = buildOAuth1FormBody( @@ -107,7 +107,7 @@ export async function exchangeForAccessToken( oauthVerifier: string, fetchFn: FetchFn = globalThis.fetch, ): Promise<{ token: string; tokenSecret: string }> { - const rateLimitFetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "fatsecret" }); + const rateLimitFetchFn = createProviderRateLimitFetch("fatsecret", fetchFn); // FatSecret expects the signed OAuth params as the POST body, not a header. const bodyString = buildOAuth1FormBody( diff --git a/src/providers/fatsecret/provider.ts b/src/providers/fatsecret/provider.ts index 354603f73b..1bd4c25a4d 100644 --- a/src/providers/fatsecret/provider.ts +++ b/src/providers/fatsecret/provider.ts @@ -1,4 +1,5 @@ -import { createRateLimitAwareFetch, ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; +import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../../lib/provider-rate-limit-fetch.ts"; import { sql } from "drizzle-orm"; import { z } from "zod"; import { getOAuthRedirectUri } from "../../auth/oauth.ts"; @@ -36,7 +37,7 @@ export class FatSecretProvider implements SyncProvider { constructor(fetchFn: FetchFn = globalThis.fetch) { this.#consumerKey = process.env.FATSECRET_CONSUMER_KEY ?? null; this.#consumerSecret = process.env.FATSECRET_CONSUMER_SECRET ?? null; - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "fatsecret" }); + this.#fetchFn = createProviderRateLimitFetch("fatsecret", fetchFn); } validate(): string | null { diff --git a/src/providers/fitbit/provider.ts b/src/providers/fitbit/provider.ts index 5213496c56..a2017e5a54 100644 --- a/src/providers/fitbit/provider.ts +++ b/src/providers/fitbit/provider.ts @@ -1,5 +1,5 @@ import { createHmac } from "node:crypto"; -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../../lib/provider-rate-limit-fetch.ts"; import { z } from "zod"; import type { OAuthConfig, TokenSet } from "../../auth/oauth.ts"; import { @@ -81,7 +81,7 @@ export class FitbitProvider implements WebhookProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "fitbit" }); + this.#fetchFn = createProviderRateLimitFetch("fitbit", fetchFn); } validate(): string | null { diff --git a/src/providers/garmin.test.ts b/src/providers/garmin.test.ts index 88a957ffb3..78c499d941 100644 --- a/src/providers/garmin.test.ts +++ b/src/providers/garmin.test.ts @@ -199,6 +199,14 @@ function createMockDb(): MockDb { db.values.mockReturnValue(db); db.onConflictDoUpdate.mockReturnValue(db); db.delete.mockReturnValue(db); + Object.assign(db, { + then( + onFulfilled: (value: unknown) => unknown, + onRejected?: (reason: unknown) => unknown, + ) { + return Promise.resolve([]).then(onFulfilled, onRejected); + }, + }); return db; } @@ -651,7 +659,7 @@ describe("GarminProvider.sync()", () => { ], }); - const result = await syncProvider(provider, db, new Date()); + const result = await syncProvider(provider, db, new Date("2026-02-01T00:00:00Z")); expect(mocks.parseConnectActivity).toHaveBeenCalledWith(rawActivity); expect(mocks.client.getActivityDetail).toHaveBeenCalledWith(123); @@ -690,6 +698,33 @@ describe("GarminProvider.sync()", () => { expect(sensorRows).toContainEqual(expect.objectContaining({ channel: "cadence", scalar: 90 })); }); + it("skips activity detail fetch for activities already stored in the database", async () => { + const rawActivity = { activityId: 123, deviceName: "Forerunner 955" }; + mocks.client.getActivities.mockResolvedValue([rawActivity]); + mocks.parseConnectActivity.mockReturnValue({ + externalId: "123", + activityType: "running", + name: "Morning Run", + startedAt: new Date("2026-03-01T10:00:00Z"), + endedAt: new Date("2026-03-01T11:00:00Z"), + raw: rawActivity, + }); + + Object.assign(db, { + then( + onFulfilled: (value: unknown) => unknown, + onRejected?: (reason: unknown) => unknown, + ) { + return Promise.resolve([{ externalId: "123" }]).then(onFulfilled, onRejected); + }, + }); + + const result = await syncProvider(provider, db, new Date("2026-02-01T00:00:00Z")); + + expect(mocks.client.getActivityDetail).not.toHaveBeenCalled(); + expect(result.recordsSynced).toBe(1); + }); + it("reconciles provider absence using since when the activity page is partial", async () => { const since = new Date("2026-01-01T00:00:00Z"); const rawActivity = { activityId: 123, deviceName: "Forerunner 955" }; diff --git a/src/providers/garmin.ts b/src/providers/garmin.ts index 9055784c8b..e295f3e908 100644 --- a/src/providers/garmin.ts +++ b/src/providers/garmin.ts @@ -1,7 +1,7 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { isIndoorCycling } from "@dofek/training/endurance-types"; import { captureException } from "@sentry/node"; -import { and, eq } from "drizzle-orm"; +import { and, eq, inArray } from "drizzle-orm"; import { GarminApiError, GarminConnectClient, GarminRateLimitError } from "garmin-connect/client"; import { parseActivityDetail, @@ -297,9 +297,7 @@ export class GarminProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { - providerId: "garmin", - createRateLimitError: (response, responseBody) => + this.#fetchFn = createProviderRateLimitFetch("garmin", fetchFn, { createRateLimitError: (response, responseBody) => new GarminRateLimitError( `Rate limit exceeded (${response.status}): ${responseBody}`, responseBody, @@ -621,6 +619,25 @@ export class GarminProvider implements SyncProvider { const detailErrors = new SyncErrorTracker("activity_detail"); const presentActivityExternalIds = new Set<string>(); + const pageExternalIds = activities.map((raw) => String(raw.activityId)); + const existingActivityIds = + pageExternalIds.length === 0 + ? new Set<string>() + : new Set( + ( + await db + .select({ externalId: activity.externalId }) + .from(activity) + .where( + and( + eq(activity.userId, userId), + eq(activity.providerId, this.id), + inArray(activity.externalId, pageExternalIds), + ), + ) + ).map((row) => row.externalId), + ); + for (const raw of activities) { const parsed = parseConnectActivity(raw); presentActivityExternalIds.add(parsed.externalId); @@ -652,6 +669,16 @@ export class GarminProvider implements SyncProvider { }, }); + const needsDetail = + parsed.startedAt >= since && + parsed.startedAt <= until && + !existingActivityIds.has(parsed.externalId); + + if (!needsDetail) { + count++; + continue; + } + // Sync activity detail streams try { const detail = await client.getActivityDetail(raw.activityId); diff --git a/src/providers/http-client.ts b/src/providers/http-client.ts index fb929c215f..af3b4e6ec6 100644 --- a/src/providers/http-client.ts +++ b/src/providers/http-client.ts @@ -9,7 +9,7 @@ * - Fetch function injection for testability */ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { z } from "zod"; /** @@ -40,7 +40,7 @@ export class ProviderHttpClient { ) { this.accessToken = accessToken; this.apiBase = apiBase; - this.fetchFn = createRateLimitAwareFetch(fetchFn, { providerId }); + this.fetchFn = createProviderRateLimitFetch(providerId, fetchFn); } /** diff --git a/src/providers/komoot.ts b/src/providers/komoot.ts index 5f85844c89..4e2e57f1fd 100644 --- a/src/providers/komoot.ts +++ b/src/providers/komoot.ts @@ -1,4 +1,4 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { CanonicalActivityType } from "@dofek/training/training"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; import { exchangeCodeForTokens, getOAuthRedirectUri } from "../auth/oauth.ts"; @@ -136,7 +136,7 @@ export class KomootProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "komoot" }); + this.#fetchFn = createProviderRateLimitFetch("komoot", fetchFn); } validate(): string | null { diff --git a/src/providers/mapmyfitness.test.ts b/src/providers/mapmyfitness.test.ts index 67d72f06ae..618e5a4e73 100644 --- a/src/providers/mapmyfitness.test.ts +++ b/src/providers/mapmyfitness.test.ts @@ -1,4 +1,5 @@ import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { afterEach, describe, expect, it, vi } from "vitest"; import { SyncRun } from "./sync-run.ts"; import { SyncWindow } from "./sync-window.ts"; @@ -141,7 +142,7 @@ describe("MapMyFitnessClient", () => { it("throws a ProviderRateLimitError with providerId on 429", async () => { // Kills the constructor ObjectLiteral mutant on - // createRateLimitAwareFetch(fetchFn, { providerId: "mapmyfitness" }) → {}. + // createProviderRateLimitFetch("mapmyfitness", fetchFn) → {}. const mockFetch = vi.fn().mockResolvedValue(new Response("slow down", { status: 429 })); const client = new MapMyFitnessClient("token", "client-id", mockFetch); diff --git a/src/providers/mapmyfitness.ts b/src/providers/mapmyfitness.ts index 6d2fe4d033..18989fe7e3 100644 --- a/src/providers/mapmyfitness.ts +++ b/src/providers/mapmyfitness.ts @@ -1,4 +1,4 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { CanonicalActivityType } from "@dofek/training/training"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; import { exchangeCodeForTokens, getOAuthRedirectUri } from "../auth/oauth.ts"; @@ -143,7 +143,7 @@ export class MapMyFitnessClient { ) { this.#accessToken = accessToken; this.#clientId = clientId; - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "mapmyfitness" }); + this.#fetchFn = createProviderRateLimitFetch("mapmyfitness", fetchFn); } async #get<T>(path: string): Promise<T> { @@ -200,7 +200,7 @@ export class MapMyFitnessProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "mapmyfitness" }); + this.#fetchFn = createProviderRateLimitFetch("mapmyfitness", fetchFn); } validate(): string | null { diff --git a/src/providers/oura/provider.ts b/src/providers/oura/provider.ts index a1d0ed45be..fff54b8820 100644 --- a/src/providers/oura/provider.ts +++ b/src/providers/oura/provider.ts @@ -1,4 +1,4 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../../lib/provider-rate-limit-fetch.ts"; import { z } from "zod"; import { exchangeCodeForTokens } from "../../auth/oauth.ts"; import { resolveOAuthTokens } from "../../auth/resolve-tokens.ts"; @@ -44,7 +44,7 @@ export class OuraProvider implements WebhookProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "oura" }); + this.#fetchFn = createProviderRateLimitFetch("oura", fetchFn); } validate(): string | null { diff --git a/src/providers/peloton.ts b/src/providers/peloton.ts index d2cac1ba73..b1623e6993 100644 --- a/src/providers/peloton.ts +++ b/src/providers/peloton.ts @@ -1,4 +1,4 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { CanonicalActivityType } from "@dofek/training/training"; import { z } from "zod"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; @@ -212,7 +212,7 @@ export class PelotonClient { constructor(accessToken: string, fetchFn: typeof globalThis.fetch = globalThis.fetch) { this.#accessToken = accessToken; - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "peloton" }); + this.#fetchFn = createProviderRateLimitFetch("peloton", fetchFn); } async #get<T>(path: string, params?: Record<string, string>): Promise<T> { @@ -387,7 +387,7 @@ export async function pelotonAutomatedLogin( password: string, fetchFn: typeof globalThis.fetch = globalThis.fetch, ): Promise<TokenSet> { - const rateLimitFetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "peloton" }); + const rateLimitFetchFn = createProviderRateLimitFetch("peloton", fetchFn); const config = pelotonOAuthConfig(); const codeVerifier = generateCodeVerifier(); const codeChallenge = generateCodeChallenge(codeVerifier); @@ -531,7 +531,7 @@ export class PelotonProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "peloton" }); + this.#fetchFn = createProviderRateLimitFetch("peloton", fetchFn); } validate(): string | null { diff --git a/src/providers/polar/client.test.ts b/src/providers/polar/client.test.ts index 6cd091c422..491afa6103 100644 --- a/src/providers/polar/client.test.ts +++ b/src/providers/polar/client.test.ts @@ -1,4 +1,5 @@ import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../../lib/provider-rate-limit-fetch.ts"; import { describe, expect, it } from "vitest"; import { PolarClient } from "./client.ts"; diff --git a/src/providers/polar/client.ts b/src/providers/polar/client.ts index 987459ab7b..3cb20cbd77 100644 --- a/src/providers/polar/client.ts +++ b/src/providers/polar/client.ts @@ -1,4 +1,4 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../../lib/provider-rate-limit-fetch.ts"; import { POLAR_API_BASE } from "./oauth.ts"; import type { PolarDailyActivity, @@ -35,7 +35,7 @@ export class PolarClient { constructor(accessToken: string, fetchFn: typeof globalThis.fetch = globalThis.fetch) { this.#accessToken = accessToken; - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "polar" }); + this.#fetchFn = createProviderRateLimitFetch("polar", fetchFn); } async #get<TResponse>( diff --git a/src/providers/polar/provider.ts b/src/providers/polar/provider.ts index b99cd2fb26..0358c328fd 100644 --- a/src/providers/polar/provider.ts +++ b/src/providers/polar/provider.ts @@ -1,4 +1,4 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../../lib/provider-rate-limit-fetch.ts"; import type { TokenSet } from "../../auth/oauth.ts"; import { logger } from "../../logger.ts"; import type { SyncRun } from "../sync-run.ts"; @@ -20,7 +20,7 @@ export class PolarProvider implements WebhookProvider { readonly #webhookService: PolarWebhookService; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "polar" }); + this.#fetchFn = createProviderRateLimitFetch("polar", fetchFn); this.#webhookService = new PolarWebhookService(this.#fetchFn); } diff --git a/src/providers/polar/webhook-service.ts b/src/providers/polar/webhook-service.ts index a0a32dadbf..6f3a6038cc 100644 --- a/src/providers/polar/webhook-service.ts +++ b/src/providers/polar/webhook-service.ts @@ -1,5 +1,5 @@ import { createHmac, timingSafeEqual } from "node:crypto"; -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../../lib/provider-rate-limit-fetch.ts"; import { z } from "zod"; import type { WebhookEvent } from "../types.ts"; @@ -22,7 +22,7 @@ export class PolarWebhookService { readonly #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "polar" }); + this.#fetchFn = createProviderRateLimitFetch("polar", fetchFn); } async registerWebhook( diff --git a/src/providers/ride-with-gps.ts b/src/providers/ride-with-gps.ts index 689d53f18f..834cfe884d 100644 --- a/src/providers/ride-with-gps.ts +++ b/src/providers/ride-with-gps.ts @@ -1,4 +1,4 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { isIndoorCycling } from "@dofek/training/endurance-types"; import { type CanonicalActivityType, @@ -241,7 +241,7 @@ export class RideWithGpsClient { constructor(accessToken: string, fetchFn: typeof globalThis.fetch = globalThis.fetch) { this.#accessToken = accessToken; - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "ride-with-gps" }); + this.#fetchFn = createProviderRateLimitFetch("ride-with-gps", fetchFn); } async #get<T>(path: string, params?: Record<string, string>): Promise<T> { @@ -330,7 +330,7 @@ export class RideWithGpsProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "ride-with-gps" }); + this.#fetchFn = createProviderRateLimitFetch("ride-with-gps", fetchFn); } validate(): string | null { diff --git a/src/providers/strava.ts b/src/providers/strava.ts index bf5aed5f4d..73d4db548e 100644 --- a/src/providers/strava.ts +++ b/src/providers/strava.ts @@ -1,8 +1,5 @@ -import { - createRateLimitAwareFetch, - ProviderRateLimitError, - parseRetryAfterHeader, -} from "@dofek/provider-http/rate-limit"; +import { ProviderRateLimitError, parseRetryAfterHeader } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { isIndoorCycling } from "@dofek/training/endurance-types"; import { type CanonicalActivityType, @@ -268,9 +265,7 @@ export class StravaClient { throttleMs = STRAVA_THROTTLE_MS, ) { this.#accessToken = accessToken; - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { - providerId: "strava", - createRateLimitError: createStravaRateLimitError, + this.#fetchFn = createProviderRateLimitFetch("strava", fetchFn, { createRateLimitError: createStravaRateLimitError, }); this.#throttleMs = throttleMs; } @@ -286,7 +281,6 @@ export class StravaClient { } async #get<T>(path: string, params?: Record<string, string>): Promise<T> { - await this.#throttle(); const url = new URL(path, STRAVA_API_BASE); if (params) { for (const [key, value] of Object.entries(params)) { @@ -444,9 +438,7 @@ export class StravaProvider implements WebhookProvider { fetchFn: typeof globalThis.fetch = globalThis.fetch, throttleMs = STRAVA_THROTTLE_MS, ) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { - providerId: "strava", - createRateLimitError: createStravaRateLimitError, + this.#fetchFn = createProviderRateLimitFetch("strava", fetchFn, { createRateLimitError: createStravaRateLimitError, }); this.#throttleMs = throttleMs; } diff --git a/src/providers/suunto.ts b/src/providers/suunto.ts index fc59dfa62b..45a43b4289 100644 --- a/src/providers/suunto.ts +++ b/src/providers/suunto.ts @@ -1,5 +1,5 @@ import { createHmac, timingSafeEqual } from "node:crypto"; -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { CanonicalActivityType } from "@dofek/training/training"; import { z } from "zod"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; @@ -173,7 +173,7 @@ export class SuuntoProvider implements WebhookProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "suunto" }); + this.#fetchFn = createProviderRateLimitFetch("suunto", fetchFn); } validate(): string | null { diff --git a/src/providers/trainerroad.ts b/src/providers/trainerroad.ts index 6556e705b8..432c43e0d4 100644 --- a/src/providers/trainerroad.ts +++ b/src/providers/trainerroad.ts @@ -1,4 +1,4 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { TrainerRoadClient } from "trainerroad-client/client"; import { parseTrainerRoadActivity } from "trainerroad-client/parsing"; import { reconcileProviderActivityAbsence } from "../db/provider-activity-absence.ts"; @@ -29,7 +29,7 @@ export class TrainerRoadProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "trainerroad" }); + this.#fetchFn = createProviderRateLimitFetch("trainerroad", fetchFn); } validate(): string | null { diff --git a/src/providers/ultrahuman.ts b/src/providers/ultrahuman.ts index 8f1292de87..ffb9deb0f7 100644 --- a/src/providers/ultrahuman.ts +++ b/src/providers/ultrahuman.ts @@ -1,4 +1,5 @@ -import { createRateLimitAwareFetch, ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; +import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { dailyMetrics, sleepSession } from "../db/schema.ts"; import { withSyncLog } from "../db/sync-log.ts"; import { ensureProvider, loadTokens } from "../db/tokens.ts"; @@ -114,7 +115,7 @@ export class UltrahumanClient { constructor(token: string, email: string, fetchFn: typeof globalThis.fetch = globalThis.fetch) { this.#token = token; this.#email = email; - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "ultrahuman" }); + this.#fetchFn = createProviderRateLimitFetch("ultrahuman", fetchFn); } async getDailyMetrics(date: string): Promise<UltrahumanDailyMetricsResponse> { @@ -153,7 +154,7 @@ export class UltrahumanProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "ultrahuman" }); + this.#fetchFn = createProviderRateLimitFetch("ultrahuman", fetchFn); } validate(): string | null { diff --git a/src/providers/velohero.ts b/src/providers/velohero.ts index cb30c09ccf..b4b4739ea9 100644 --- a/src/providers/velohero.ts +++ b/src/providers/velohero.ts @@ -1,4 +1,4 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { VeloHeroClient } from "velohero-client/client"; import { parseVeloHeroWorkout } from "velohero-client/parsing"; import { reconcileProviderActivityAbsence } from "../db/provider-activity-absence.ts"; @@ -30,7 +30,7 @@ export class VeloHeroProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "velohero" }); + this.#fetchFn = createProviderRateLimitFetch("velohero", fetchFn); } validate(): string | null { diff --git a/src/providers/wahoo/provider.ts b/src/providers/wahoo/provider.ts index 7506fe1314..3071643243 100644 --- a/src/providers/wahoo/provider.ts +++ b/src/providers/wahoo/provider.ts @@ -1,4 +1,4 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../../lib/provider-rate-limit-fetch.ts"; import type { OAuthConfig, TokenSet } from "../../auth/oauth.ts"; import { exchangeCodeForTokens, @@ -51,7 +51,7 @@ export class WahooProvider implements WebhookProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "wahoo" }); + this.#fetchFn = createProviderRateLimitFetch("wahoo", fetchFn); } validate(): string | null { diff --git a/src/providers/wger.ts b/src/providers/wger.ts index 9dfdc7969b..738cf8d791 100644 --- a/src/providers/wger.ts +++ b/src/providers/wger.ts @@ -1,4 +1,4 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { CanonicalActivityType } from "@dofek/training/training"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; import { exchangeCodeForTokens, getOAuthRedirectUri } from "../auth/oauth.ts"; @@ -116,7 +116,7 @@ export class WgerProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "wger" }); + this.#fetchFn = createProviderRateLimitFetch("wger", fetchFn); } validate(): string | null { diff --git a/src/providers/whoop.test.ts b/src/providers/whoop.test.ts index b51a4d251e..8118009a5d 100644 --- a/src/providers/whoop.test.ts +++ b/src/providers/whoop.test.ts @@ -1,4 +1,5 @@ import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { WhoopClient } from "whoop-whoop/client"; import type { diff --git a/src/providers/whoop/provider.ts b/src/providers/whoop/provider.ts index b8e546c207..49ce85bb7f 100644 --- a/src/providers/whoop/provider.ts +++ b/src/providers/whoop/provider.ts @@ -1,4 +1,4 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../../lib/provider-rate-limit-fetch.ts"; import { WhoopClient } from "whoop-whoop/client"; import type { WhoopCycle } from "whoop-whoop/types"; import { z } from "zod"; @@ -22,6 +22,7 @@ import { syncWhoopSleepSessions, syncWhoopSleepStages } from "./sync-sleep.ts"; import { syncWhoopHeartRateStream } from "./sync-streams.ts"; import type { WhoopSyncContext } from "./sync-types.ts"; import { syncWhoopStrength, syncWhoopWorkouts } from "./sync-workouts.ts"; +import { findWhoopRateLimitError, isWhoopRateLimitError } from "./rate-limit.ts"; // ============================================================ // Provider implementation @@ -34,7 +35,7 @@ export class WhoopProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "whoop" }); + this.#fetchFn = createProviderRateLimitFetch("whoop", fetchFn); } validate(): string | null { @@ -169,6 +170,9 @@ export class WhoopProvider implements SyncProvider { } logger.info(`[whoop] Fetched ${cycles.length} total cycles`); } catch (err) { + if (isWhoopRateLimitError(err)) { + throw err; + } errors.push({ message: `getCycles: ${err instanceof Error ? err.message : String(err)}`, cause: err, @@ -211,6 +215,11 @@ export class WhoopProvider implements SyncProvider { recordsSynced += await syncWhoopJournal(context); } + const rateLimitError = findWhoopRateLimitError(errors); + if (rateLimitError) { + throw rateLimitError; + } + return { provider: this.id, recordsSynced, diff --git a/src/providers/whoop/rate-limit.test.ts b/src/providers/whoop/rate-limit.test.ts new file mode 100644 index 0000000000..3157e9e6f1 --- /dev/null +++ b/src/providers/whoop/rate-limit.test.ts @@ -0,0 +1,42 @@ +import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; +import { WhoopRateLimitError } from "whoop-whoop/client"; +import { describe, expect, it } from "vitest"; +import { findWhoopRateLimitError, isWhoopRateLimitError } from "./rate-limit.ts"; + +describe("isWhoopRateLimitError", () => { + it("recognizes WhoopRateLimitError", () => { + expect(isWhoopRateLimitError(new WhoopRateLimitError("limited"))).toBe(true); + }); + + it("recognizes ProviderRateLimitError tagged with whoop", () => { + const error = new ProviderRateLimitError({ + message: "whoop API rate limit exceeded (429):", + providerId: "whoop", + statusCode: 429, + responseBody: "", + }); + expect(isWhoopRateLimitError(error)).toBe(true); + }); + + it("rejects other provider rate-limit errors", () => { + const error = new ProviderRateLimitError({ + message: "garmin API rate limit exceeded (429):", + providerId: "garmin", + statusCode: 429, + responseBody: "", + }); + expect(isWhoopRateLimitError(error)).toBe(false); + }); +}); + +describe("findWhoopRateLimitError", () => { + it("returns the first whoop rate-limit error from sync errors", () => { + const error = new ProviderRateLimitError({ + message: "whoop API rate limit exceeded (429):", + providerId: "whoop", + statusCode: 429, + responseBody: "", + }); + expect(findWhoopRateLimitError([{ cause: error }])).toBe(error); + }); +}); diff --git a/src/providers/whoop/rate-limit.ts b/src/providers/whoop/rate-limit.ts new file mode 100644 index 0000000000..f07d9a253d --- /dev/null +++ b/src/providers/whoop/rate-limit.ts @@ -0,0 +1,19 @@ +import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; +import { WhoopRateLimitError } from "whoop-whoop/client"; + +export function isWhoopRateLimitError(err: unknown): boolean { + return ( + err instanceof WhoopRateLimitError || + (err instanceof ProviderRateLimitError && err.providerId === "whoop") + ); +} + +export function findWhoopRateLimitError(errors: Array<{ cause?: unknown }>): ProviderRateLimitError | null { + for (const syncError of errors) { + const cause = syncError.cause; + if (isWhoopRateLimitError(cause) && cause instanceof ProviderRateLimitError) { + return cause; + } + } + return null; +} diff --git a/src/providers/whoop/sync-daily-activity.ts b/src/providers/whoop/sync-daily-activity.ts index 17d100daf4..55a7090e06 100644 --- a/src/providers/whoop/sync-daily-activity.ts +++ b/src/providers/whoop/sync-daily-activity.ts @@ -1,7 +1,9 @@ -import { WhoopRateLimitError } from "whoop-whoop/client"; +import { and, eq } from "drizzle-orm"; import { dailyMetrics } from "../../db/schema.ts"; import { withSyncLog } from "../../db/sync-log.ts"; +import { getTokenUserId } from "../../db/token-user-context.ts"; import { parseStrainDeepDiveSteps } from "./parsing.ts"; +import { isWhoopRateLimitError } from "./rate-limit.ts"; import type { WhoopSyncContext } from "./sync-types.ts"; export type WhoopDailyActivityResult = { @@ -35,8 +37,23 @@ export async function syncWhoopDailyActivity( async () => { const nowMs = Date.now(); const stepsByDate = new Map<string, number>(); + const userId = options?.userId ?? getTokenUserId(); + const existingDates = + userId == null + ? new Set<string>() + : new Set( + ( + await db + .select({ date: dailyMetrics.date }) + .from(dailyMetrics) + .where( + and(eq(dailyMetrics.userId, userId), eq(dailyMetrics.providerId, providerId)), + ) + ).map((row) => row.date), + ); for (const date of iterateUtcDates(since, nowMs)) { + if (existingDates.has(date)) continue; const raw = await client.getStrainDeepDive(date); const steps = parseStrainDeepDiveSteps(raw); if (steps != null) { @@ -65,7 +82,11 @@ export async function syncWhoopDailyActivity( ); return { count, rateLimited: false }; } catch (err) { - if (err instanceof WhoopRateLimitError) { + if (isWhoopRateLimitError(err)) { + context.errors.push({ + message: `daily_activity: ${err instanceof Error ? err.message : String(err)}`, + cause: err, + }); return { count: 0, rateLimited: true }; } context.errors.push({ diff --git a/src/providers/whoop/sync-helpers.test.ts b/src/providers/whoop/sync-helpers.test.ts index 1b13ffe630..37a0e11530 100644 --- a/src/providers/whoop/sync-helpers.test.ts +++ b/src/providers/whoop/sync-helpers.test.ts @@ -37,6 +37,10 @@ vi.mock("../../db/metric-stream-writer.ts", () => ({ writeMetricStreamBatch: vi.fn().mockResolvedValue(undefined), })); +vi.mock("../../db/token-user-context.ts", () => ({ + getTokenUserId: () => "00000000-0000-0000-0000-000000000001", +})); + function makeDb(selectedRows: unknown[] = []) { const chain = { values: vi.fn(), @@ -51,6 +55,14 @@ function makeDb(selectedRows: unknown[] = []) { chain.from.mockReturnValue(chain); chain.where.mockReturnValue(chain); chain.limit.mockResolvedValue(selectedRows); + Object.assign(chain, { + then( + onFulfilled: (value: unknown) => unknown, + onRejected?: (reason: unknown) => unknown, + ) { + return Promise.resolve(selectedRows).then(onFulfilled, onRejected); + }, + }); const db: SyncDatabase = { insert: vi.fn().mockReturnValue(chain), diff --git a/src/providers/whoop/sync-streams.ts b/src/providers/whoop/sync-streams.ts index 9ecd34fb04..9a7d662e4d 100644 --- a/src/providers/whoop/sync-streams.ts +++ b/src/providers/whoop/sync-streams.ts @@ -1,8 +1,8 @@ -import { WhoopRateLimitError } from "whoop-whoop/client"; import { writeMetricStreamBatch } from "../../db/metric-stream-writer.ts"; import { SOURCE_TYPE_API } from "../../db/sensor-channels.ts"; import { withSyncLog } from "../../db/sync-log.ts"; import { parseHeartRateValues } from "./parsing.ts"; +import { isWhoopRateLimitError } from "./rate-limit.ts"; import type { WhoopSyncContext } from "./sync-types.ts"; export type WhoopStreamSyncResult = { @@ -57,9 +57,9 @@ export async function syncWhoopHeartRateStream( ); return { count, rateLimited: false }; } catch (err) { - if (err instanceof WhoopRateLimitError) { + if (isWhoopRateLimitError(err)) { context.errors.push({ - message: `hr_stream: ${err.message}`, + message: `hr_stream: ${err instanceof Error ? err.message : String(err)}`, cause: err, }); return { count: 0, rateLimited: true }; diff --git a/src/providers/whoop/sync-workouts.ts b/src/providers/whoop/sync-workouts.ts index 9824682699..056aa01a0d 100644 --- a/src/providers/whoop/sync-workouts.ts +++ b/src/providers/whoop/sync-workouts.ts @@ -1,5 +1,4 @@ import { and, eq, sql } from "drizzle-orm"; -import { WhoopRateLimitError } from "whoop-whoop/client"; import type { WhoopWorkoutRecord } from "whoop-whoop/types"; import { parseDuringRange } from "whoop-whoop/utils"; import { reconcileProviderActivityAbsence } from "../../db/provider-activity-absence.ts"; @@ -12,6 +11,7 @@ import { parseWorkout, resolveWhoopWorkoutExternalId, } from "./parsing.ts"; +import { isWhoopRateLimitError } from "./rate-limit.ts"; import type { WhoopSyncContext } from "./sync-types.ts"; export type WhoopWorkoutSyncResult = { @@ -299,9 +299,9 @@ export async function syncWhoopStrength( ); return { count, rateLimited: false }; } catch (err) { - if (err instanceof WhoopRateLimitError) { + if (isWhoopRateLimitError(err)) { context.errors.push({ - message: `strength: ${err.message}`, + message: `strength: ${err instanceof Error ? err.message : String(err)}`, cause: err, }); return { count: 0, rateLimited: true }; diff --git a/src/providers/withings.ts b/src/providers/withings.ts index 02f7d9d5d8..2a1132f0ed 100644 --- a/src/providers/withings.ts +++ b/src/providers/withings.ts @@ -1,4 +1,4 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { z } from "zod"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; import { getOAuthRedirectUri } from "../auth/oauth.ts"; @@ -182,7 +182,7 @@ async function withingsTokenExchange( params: Record<string, string>, fetchFn: typeof globalThis.fetch = globalThis.fetch, ): Promise<TokenSet> { - const rateLimitFetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "withings" }); + const rateLimitFetchFn = createProviderRateLimitFetch("withings", fetchFn); const bodyParams: Record<string, string> = { action: "requesttoken", client_id: config.clientId, @@ -263,7 +263,7 @@ export class WithingsClient { constructor(accessToken: string, fetchFn: typeof globalThis.fetch = globalThis.fetch) { this.#accessToken = accessToken; - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "withings" }); + this.#fetchFn = createProviderRateLimitFetch("withings", fetchFn); } async #post<T>(path: string, params: Record<string, string>): Promise<T> { @@ -330,7 +330,7 @@ export class WithingsProvider implements WebhookProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "withings" }); + this.#fetchFn = createProviderRateLimitFetch("withings", fetchFn); } validate(): string | null { diff --git a/src/providers/xert.ts b/src/providers/xert.ts index af72e31f7d..dfaeaf9e94 100644 --- a/src/providers/xert.ts +++ b/src/providers/xert.ts @@ -1,4 +1,4 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { CanonicalActivityType } from "@dofek/training/training"; import { z } from "zod"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; @@ -131,7 +131,7 @@ export async function signInToXert( password: string, fetchFn: typeof globalThis.fetch = globalThis.fetch, ): Promise<TokenSet> { - const rateLimitFetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "xert" }); + const rateLimitFetchFn = createProviderRateLimitFetch("xert", fetchFn); const clientId = process.env.XERT_CLIENT_ID ?? "xert_public"; const clientSecret = process.env.XERT_CLIENT_SECRET ?? "xert_public"; @@ -195,7 +195,7 @@ export class XertProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "xert" }); + this.#fetchFn = createProviderRateLimitFetch("xert", fetchFn); } validate(): string | null { diff --git a/src/providers/zwift.ts b/src/providers/zwift.ts index 8b7623ab46..d8c8a022d9 100644 --- a/src/providers/zwift.ts +++ b/src/providers/zwift.ts @@ -1,4 +1,4 @@ -import { createRateLimitAwareFetch } from "@dofek/provider-http/rate-limit"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { z } from "zod"; import { ZWIFT_API_BASE, ZwiftClient } from "zwift-client/client"; import { parseZwiftActivity, parseZwiftFitnessData } from "zwift-client/parsing"; @@ -28,7 +28,7 @@ export class ZwiftProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createRateLimitAwareFetch(fetchFn, { providerId: "zwift" }); + this.#fetchFn = createProviderRateLimitFetch("zwift", fetchFn); } validate(): string | null { From 12abef913c86f7393b6912c54d6b071fd8cb9ef0 Mon Sep 17 00:00:00 2001 From: Asher Cohen <asher.cohen@amplitude.com> Date: Sun, 21 Jun 2026 09:27:55 -0700 Subject: [PATCH 02/18] Fix CI typecheck and import boundary failures. Break the provider-http circular import via rate-limit-types, and handle nullable enqueueSyncJob results in sync triggers. Co-authored-by: Cursor <cursoragent@cursor.com> --- .../provider-http/src/adaptive-rate-limit.ts | 18 +---- .../provider-http/src/rate-limit-types.ts | 78 ++++++++++++++++++ packages/provider-http/src/rate-limit.ts | 80 ++++--------------- packages/server/src/mcp/tools.ts | 3 + packages/server/src/routers/sync.ts | 6 ++ 5 files changed, 105 insertions(+), 80 deletions(-) create mode 100644 packages/provider-http/src/rate-limit-types.ts diff --git a/packages/provider-http/src/adaptive-rate-limit.ts b/packages/provider-http/src/adaptive-rate-limit.ts index dae58b6315..79688a0d0e 100644 --- a/packages/provider-http/src/adaptive-rate-limit.ts +++ b/packages/provider-http/src/adaptive-rate-limit.ts @@ -1,4 +1,4 @@ -import type { ProviderRateLimitError, ProviderRateLimitScope } from "./rate-limit.ts"; +import type { ProviderRateLimitScope } from "./rate-limit-types.ts"; /** Rolling window for request budget tally (5 minutes). */ export const ADAPTIVE_RATE_WINDOW_MS = 5 * 60 * 1000; @@ -39,21 +39,7 @@ export interface ProviderAdaptiveRateState { stravaDailyUsage: number | null; } -export interface AdaptiveRateLimitStore { - awaitAdmission( - providerId: string, - scope: ProviderRateLimitScope, - userId: string | null, - ): Promise<void>; - recordSuccess( - providerId: string, - scope: ProviderRateLimitScope, - userId: string | null, - responseHeaders?: Headers, - ): Promise<void>; - recordRateLimit(error: ProviderRateLimitError): Promise<void>; - getLearnedCooldownSeconds(providerId: string): Promise<number | null>; -} +export type { AdaptiveRateLimitStore } from "./rate-limit-types.ts"; export function defaultThrottleMs(providerId: string): number { return DEFAULT_PROVIDER_THROTTLE_MS[providerId] ?? 1_000; diff --git a/packages/provider-http/src/rate-limit-types.ts b/packages/provider-http/src/rate-limit-types.ts new file mode 100644 index 0000000000..6ba2bf9697 --- /dev/null +++ b/packages/provider-http/src/rate-limit-types.ts @@ -0,0 +1,78 @@ +export type ProviderRateLimitScope = "provider" | "user"; +export type ProviderHttpErrorScope = ProviderRateLimitScope; + +export interface ProviderRateLimitErrorOptions { + message: string; + providerId: string; + statusCode: number; + responseBody: string; + scope?: ProviderRateLimitScope; + userId?: string | null; + retryAfterSeconds?: number | null; +} + +export interface ProviderServiceUnavailableErrorOptions { + message: string; + providerId: string; + statusCode: number; + responseBody: string; + scope?: ProviderHttpErrorScope; + userId?: string | null; + retryAfterSeconds?: number | null; +} + +export class ProviderRateLimitError extends Error { + readonly providerId: string; + readonly statusCode: number; + readonly responseBody: string; + readonly scope: ProviderRateLimitScope; + readonly userId: string | null; + readonly retryAfterSeconds: number | null; + + constructor(options: ProviderRateLimitErrorOptions) { + super(options.message); + this.name = "ProviderRateLimitError"; + this.providerId = options.providerId; + this.statusCode = options.statusCode; + this.responseBody = options.responseBody; + this.scope = options.scope ?? "provider"; + this.userId = options.userId ?? null; + this.retryAfterSeconds = options.retryAfterSeconds ?? null; + } +} + +export class ProviderServiceUnavailableError extends Error { + readonly providerId: string; + readonly statusCode: number; + readonly responseBody: string; + readonly scope: ProviderHttpErrorScope; + readonly userId: string | null; + readonly retryAfterSeconds: number | null; + + constructor(options: ProviderServiceUnavailableErrorOptions) { + super(options.message); + this.name = "ProviderServiceUnavailableError"; + this.providerId = options.providerId; + this.statusCode = options.statusCode; + this.responseBody = options.responseBody; + this.scope = options.scope ?? "provider"; + this.userId = options.userId ?? null; + this.retryAfterSeconds = options.retryAfterSeconds ?? null; + } +} + +export interface AdaptiveRateLimitStore { + awaitAdmission( + providerId: string, + scope: ProviderRateLimitScope, + userId: string | null, + ): Promise<void>; + recordSuccess( + providerId: string, + scope: ProviderRateLimitScope, + userId: string | null, + responseHeaders?: Headers, + ): Promise<void>; + recordRateLimit(error: ProviderRateLimitError): Promise<void>; + getLearnedCooldownSeconds(providerId: string): Promise<number | null>; +} diff --git a/packages/provider-http/src/rate-limit.ts b/packages/provider-http/src/rate-limit.ts index 9f7a71a6aa..8556e8b905 100644 --- a/packages/provider-http/src/rate-limit.ts +++ b/packages/provider-http/src/rate-limit.ts @@ -1,67 +1,19 @@ -import type { AdaptiveRateLimitStore } from "./adaptive-rate-limit.ts"; - -export interface ProviderRateLimitErrorOptions { - message: string; - providerId: string; - statusCode: number; - responseBody: string; - scope?: ProviderRateLimitScope; - userId?: string | null; - retryAfterSeconds?: number | null; -} - -export type ProviderRateLimitScope = "provider" | "user"; -export type ProviderHttpErrorScope = ProviderRateLimitScope; - -export interface ProviderServiceUnavailableErrorOptions { - message: string; - providerId: string; - statusCode: number; - responseBody: string; - scope?: ProviderHttpErrorScope; - userId?: string | null; - retryAfterSeconds?: number | null; -} - -export class ProviderRateLimitError extends Error { - readonly providerId: string; - readonly statusCode: number; - readonly responseBody: string; - readonly scope: ProviderRateLimitScope; - readonly userId: string | null; - readonly retryAfterSeconds: number | null; - - constructor(options: ProviderRateLimitErrorOptions) { - super(options.message); - this.name = "ProviderRateLimitError"; - this.providerId = options.providerId; - this.statusCode = options.statusCode; - this.responseBody = options.responseBody; - this.scope = options.scope ?? "provider"; - this.userId = options.userId ?? null; - this.retryAfterSeconds = options.retryAfterSeconds ?? null; - } -} - -export class ProviderServiceUnavailableError extends Error { - readonly providerId: string; - readonly statusCode: number; - readonly responseBody: string; - readonly scope: ProviderHttpErrorScope; - readonly userId: string | null; - readonly retryAfterSeconds: number | null; - - constructor(options: ProviderServiceUnavailableErrorOptions) { - super(options.message); - this.name = "ProviderServiceUnavailableError"; - this.providerId = options.providerId; - this.statusCode = options.statusCode; - this.responseBody = options.responseBody; - this.scope = options.scope ?? "provider"; - this.userId = options.userId ?? null; - this.retryAfterSeconds = options.retryAfterSeconds ?? null; - } -} +import type { AdaptiveRateLimitStore } from "./rate-limit-types.ts"; +import { + ProviderRateLimitError, + type ProviderHttpErrorScope, + type ProviderRateLimitScope, + ProviderServiceUnavailableError, +} from "./rate-limit-types.ts"; + +export type { + AdaptiveRateLimitStore, + ProviderHttpErrorScope, + ProviderRateLimitErrorOptions, + ProviderRateLimitScope, + ProviderServiceUnavailableErrorOptions, +} from "./rate-limit-types.ts"; +export { ProviderRateLimitError, ProviderServiceUnavailableError } from "./rate-limit-types.ts"; export interface FetchRateLimitHandlingOptions { createRateLimitError: (response: Response, responseBody: string) => Error; diff --git a/packages/server/src/mcp/tools.ts b/packages/server/src/mcp/tools.ts index 966c58f640..3c9037a729 100644 --- a/packages/server/src/mcp/tools.ts +++ b/packages/server/src/mcp/tools.ts @@ -257,6 +257,9 @@ export function createDofekMcpServer(context: DofekMcpContext): McpServer { userId: context.userId, ...syncWindowToJobData(syncWindow, sinceDays), }); + if (!job) { + throw new Error(`Provider ${providerId} sync skipped: rate-limit cooldown active`); + } startWorker(); return jsonContent({ providerId, diff --git a/packages/server/src/routers/sync.ts b/packages/server/src/routers/sync.ts index 59808d296c..b53db7128e 100644 --- a/packages/server/src/routers/sync.ts +++ b/packages/server/src/routers/sync.ts @@ -226,6 +226,12 @@ export const syncRouter = router({ userId: ctx.userId, ...syncWindowToJobData(syncWindow, input.sinceDays), }); + if (!job) { + throw new TRPCError({ + code: "TOO_MANY_REQUESTS", + message: `Provider ${providerId} sync skipped: rate-limit cooldown active`, + }); + } const jobId = toJobId(job.id, providerId); return { providerId, From 6c1a3b4482b3f74955561d8ee0bc0cee0ca67716 Mon Sep 17 00:00:00 2001 From: Asher Cohen <asher.cohen@amplitude.com> Date: Sun, 21 Jun 2026 09:32:52 -0700 Subject: [PATCH 03/18] Add adaptive store tests for rate-limit mutation coverage. Exercise awaitAdmission, recordSuccess, and recordRateLimit paths in createRateLimitAwareFetch so Stryker meets the mutation score threshold. Co-authored-by: Cursor <cursoragent@cursor.com> --- packages/provider-http/src/rate-limit.test.ts | 117 ++++++++++++++++++ 1 file changed, 117 insertions(+) diff --git a/packages/provider-http/src/rate-limit.test.ts b/packages/provider-http/src/rate-limit.test.ts index d1966f4d5a..3029852c12 100644 --- a/packages/provider-http/src/rate-limit.test.ts +++ b/packages/provider-http/src/rate-limit.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it, vi } from "vitest"; +import type { AdaptiveRateLimitStore } from "./rate-limit-types.ts"; import { createRateLimitAwareFetch, fetchWithRateLimitHandling, @@ -7,6 +8,20 @@ import { parseRetryAfterHeader, } from "./rate-limit.ts"; +function createMockAdaptiveStore(): AdaptiveRateLimitStore & { + awaitAdmission: ReturnType<typeof vi.fn>; + recordSuccess: ReturnType<typeof vi.fn>; + recordRateLimit: ReturnType<typeof vi.fn>; + getLearnedCooldownSeconds: ReturnType<typeof vi.fn>; +} { + return { + awaitAdmission: vi.fn().mockResolvedValue(undefined), + recordSuccess: vi.fn().mockResolvedValue(undefined), + recordRateLimit: vi.fn().mockResolvedValue(undefined), + getLearnedCooldownSeconds: vi.fn().mockResolvedValue(null), + }; +} + class TestRateLimitError extends ProviderRateLimitError { constructor(response: Response, body: string) { const retryAfterHeader = response.headers.get("Retry-After"); @@ -371,6 +386,108 @@ describe("fetchWithRateLimitHandling", () => { expect(secondError).toHaveProperty("providerId", "second"); expect(first).not.toBe(second); }); + + it("awaits adaptive admission before calling fetch when a store is provided", async () => { + const fetchFn = vi.fn<typeof globalThis.fetch>().mockResolvedValue(response(200, "ok")); + const adaptiveStore = createMockAdaptiveStore(); + const rateLimitFetch = createRateLimitAwareFetch(fetchFn, { + providerId: "example", + scope: "user", + userId: "user-9", + adaptiveStore, + }); + + await rateLimitFetch("https://api.example.com/data"); + + expect(adaptiveStore.awaitAdmission).toHaveBeenCalledOnce(); + expect(adaptiveStore.awaitAdmission).toHaveBeenCalledWith("example", "user", "user-9"); + expect(fetchFn).toHaveBeenCalledOnce(); + }); + + it("records adaptive success with response headers on ok responses", async () => { + const fetchFn = vi.fn<typeof globalThis.fetch>().mockResolvedValue( + new Response("ok", { + status: 200, + headers: { "X-RateLimit-Limit": "100,1000" }, + }), + ); + const adaptiveStore = createMockAdaptiveStore(); + const rateLimitFetch = createRateLimitAwareFetch(fetchFn, { + providerId: "example", + scope: "provider", + adaptiveStore, + }); + + await rateLimitFetch("https://api.example.com/data"); + + expect(adaptiveStore.recordSuccess).toHaveBeenCalledOnce(); + expect(adaptiveStore.recordSuccess).toHaveBeenCalledWith( + "example", + "provider", + null, + expect.any(Headers), + ); + }); + + it("does not record adaptive success when the response is not ok", async () => { + const fetchFn = vi.fn<typeof globalThis.fetch>().mockResolvedValue(response(404, "missing")); + const adaptiveStore = createMockAdaptiveStore(); + const rateLimitFetch = createRateLimitAwareFetch(fetchFn, { + providerId: "example", + adaptiveStore, + }); + + const result = await rateLimitFetch("https://api.example.com/data"); + + expect(result.status).toBe(404); + expect(adaptiveStore.recordSuccess).not.toHaveBeenCalled(); + }); + + it("records adaptive rate limits when a provider rate-limit error is thrown", async () => { + const fetchFn = vi.fn<typeof globalThis.fetch>().mockResolvedValue(response(429, "limited")); + const adaptiveStore = createMockAdaptiveStore(); + const rateLimitFetch = createRateLimitAwareFetch(fetchFn, { + providerId: "example", + scope: "user", + userId: "user-9", + adaptiveStore, + }); + + const error = await rateLimitFetch("https://api.example.com/data").catch( + (caughtError: unknown) => caughtError, + ); + + expect(error).toBeInstanceOf(ProviderRateLimitError); + expect(adaptiveStore.recordRateLimit).toHaveBeenCalledOnce(); + expect(adaptiveStore.recordRateLimit).toHaveBeenCalledWith(error); + }); + + it("does not record adaptive rate limits for service-unavailable responses", async () => { + const fetchFn = vi.fn<typeof globalThis.fetch>().mockResolvedValue(response(503, "down")); + const adaptiveStore = createMockAdaptiveStore(); + const rateLimitFetch = createRateLimitAwareFetch(fetchFn, { + providerId: "example", + adaptiveStore, + }); + + await rateLimitFetch("https://api.example.com/data").catch(() => undefined); + + expect(adaptiveStore.recordRateLimit).not.toHaveBeenCalled(); + }); + + it("does not record adaptive rate limits for custom non-provider rate-limit errors", async () => { + const fetchFn = vi.fn<typeof globalThis.fetch>().mockResolvedValue(response(429, "limited")); + const adaptiveStore = createMockAdaptiveStore(); + const rateLimitFetch = createRateLimitAwareFetch(fetchFn, { + providerId: "example", + adaptiveStore, + createRateLimitError: (_response, body) => new Error(`custom ${body}`), + }); + + await rateLimitFetch("https://api.example.com/data").catch(() => undefined); + + expect(adaptiveStore.recordRateLimit).not.toHaveBeenCalled(); + }); }); describe("parseRetryAfterHeader", () => { From d53ac4f5331c366523db1f0a8a2a4e575324679b Mon Sep 17 00:00:00 2001 From: Asher Cohen <asher.cohen@amplitude.com> Date: Sun, 21 Jun 2026 09:36:39 -0700 Subject: [PATCH 04/18] Fix lint and Stryker CI failures for rate-limit changes. Organize imports, remove dead Strava client throttle code, improve Drizzle test mocks, and expand provider cooldown mutation tests. Co-authored-by: Cursor <cursoragent@cursor.com> --- packages/garmin-connect/src/client.ts | 5 +- .../provider-http/src/adaptive-rate-limit.ts | 11 +- packages/provider-http/src/rate-limit.test.ts | 2 +- packages/provider-http/src/rate-limit.ts | 4 +- src/jobs/process-scheduled-sync-job.ts | 4 +- src/jobs/provider-rate-limit-cooldown.test.ts | 102 ++++++++++++++++++ src/jobs/provider-rate-limit-cooldown.ts | 8 +- src/lib/provider-adaptive-rate-limit.ts | 12 ++- src/providers/amazfit-zepp.test.ts | 2 +- src/providers/amazfit-zepp.ts | 2 +- src/providers/bodyspec.ts | 2 +- src/providers/concept2.ts | 2 +- src/providers/coros.ts | 2 +- src/providers/cycling-analytics.ts | 2 +- src/providers/decathlon.ts | 2 +- src/providers/eight-sleep.ts | 2 +- src/providers/fatsecret/provider.ts | 2 +- src/providers/fitbit/provider.ts | 2 +- src/providers/garmin.test.ts | 26 ++--- src/providers/garmin.ts | 5 +- src/providers/http-client.ts | 2 +- src/providers/komoot.ts | 2 +- src/providers/mapmyfitness.test.ts | 2 - src/providers/mapmyfitness.ts | 2 +- src/providers/oura/provider.ts | 2 +- src/providers/peloton.ts | 2 +- src/providers/polar/client.test.ts | 2 - src/providers/polar/provider.ts | 2 +- src/providers/polar/webhook-service.ts | 2 +- src/providers/ride-with-gps.ts | 2 +- src/providers/strava.ts | 40 ++----- src/providers/suunto.ts | 2 +- src/providers/trainerroad.ts | 2 +- src/providers/ultrahuman.ts | 2 +- src/providers/velohero.ts | 2 +- src/providers/wahoo/provider.ts | 2 +- src/providers/wger.ts | 2 +- src/providers/whoop.test.ts | 1 - src/providers/whoop/provider.ts | 4 +- src/providers/whoop/rate-limit.test.ts | 2 +- src/providers/whoop/rate-limit.ts | 4 +- src/providers/whoop/sync-helpers.test.ts | 14 +-- src/providers/withings.ts | 2 +- src/providers/xert.ts | 2 +- src/providers/zwift.ts | 2 +- 45 files changed, 191 insertions(+), 111 deletions(-) diff --git a/packages/garmin-connect/src/client.ts b/packages/garmin-connect/src/client.ts index eb532b2780..1035a3b512 100644 --- a/packages/garmin-connect/src/client.ts +++ b/packages/garmin-connect/src/client.ts @@ -56,10 +56,7 @@ export class GarminConnectClient { #displayName: string | null = null; #domain: string; #fetchFn: typeof globalThis.fetch; - constructor( - domain: string = "garmin.com", - fetchFn: typeof globalThis.fetch = globalThis.fetch, - ) { + constructor(domain: string = "garmin.com", fetchFn: typeof globalThis.fetch = globalThis.fetch) { this.#domain = domain; this.#fetchFn = fetchFn; } diff --git a/packages/provider-http/src/adaptive-rate-limit.ts b/packages/provider-http/src/adaptive-rate-limit.ts index 79688a0d0e..be12cd8386 100644 --- a/packages/provider-http/src/adaptive-rate-limit.ts +++ b/packages/provider-http/src/adaptive-rate-limit.ts @@ -67,10 +67,7 @@ export function parseStravaRateLimitHeaders(headers: Headers): StravaRateLimitQu return { shortLimit, shortUsage, dailyLimit, dailyUsage }; } -export function blendObservedCooldown( - previous: number | null, - observedSeconds: number, -): number { +export function blendObservedCooldown(previous: number | null, observedSeconds: number): number { if (!Number.isFinite(observedSeconds) || observedSeconds <= 0) return previous ?? observedSeconds; if (previous == null) return observedSeconds; return Math.round(previous * 0.7 + observedSeconds * 0.3); @@ -123,7 +120,11 @@ export function admissionDelayMs(state: ProviderAdaptiveRateState, nowMs: number } } - if (state.providerId === "strava" && state.stravaShortLimit != null && state.stravaShortUsage != null) { + if ( + state.providerId === "strava" && + state.stravaShortLimit != null && + state.stravaShortUsage != null + ) { const remaining = state.stravaShortLimit - state.stravaShortUsage; if (remaining <= 2) { delayMs = Math.max(delayMs, state.throttleMs * 4); diff --git a/packages/provider-http/src/rate-limit.test.ts b/packages/provider-http/src/rate-limit.test.ts index 3029852c12..257899e847 100644 --- a/packages/provider-http/src/rate-limit.test.ts +++ b/packages/provider-http/src/rate-limit.test.ts @@ -1,5 +1,4 @@ import { describe, expect, it, vi } from "vitest"; -import type { AdaptiveRateLimitStore } from "./rate-limit-types.ts"; import { createRateLimitAwareFetch, fetchWithRateLimitHandling, @@ -7,6 +6,7 @@ import { ProviderServiceUnavailableError, parseRetryAfterHeader, } from "./rate-limit.ts"; +import type { AdaptiveRateLimitStore } from "./rate-limit-types.ts"; function createMockAdaptiveStore(): AdaptiveRateLimitStore & { awaitAdmission: ReturnType<typeof vi.fn>; diff --git a/packages/provider-http/src/rate-limit.ts b/packages/provider-http/src/rate-limit.ts index 8556e8b905..566b1660fd 100644 --- a/packages/provider-http/src/rate-limit.ts +++ b/packages/provider-http/src/rate-limit.ts @@ -1,7 +1,7 @@ import type { AdaptiveRateLimitStore } from "./rate-limit-types.ts"; import { - ProviderRateLimitError, type ProviderHttpErrorScope, + ProviderRateLimitError, type ProviderRateLimitScope, ProviderServiceUnavailableError, } from "./rate-limit-types.ts"; @@ -13,7 +13,7 @@ export type { ProviderRateLimitScope, ProviderServiceUnavailableErrorOptions, } from "./rate-limit-types.ts"; -export { ProviderRateLimitError, ProviderServiceUnavailableError } from "./rate-limit-types.ts"; +export { ProviderRateLimitError, ProviderServiceUnavailableError }; export interface FetchRateLimitHandlingOptions { createRateLimitError: (response: Response, responseBody: string) => Error; diff --git a/src/jobs/process-scheduled-sync-job.ts b/src/jobs/process-scheduled-sync-job.ts index a0127f3633..f9dab3fbc5 100644 --- a/src/jobs/process-scheduled-sync-job.ts +++ b/src/jobs/process-scheduled-sync-job.ts @@ -65,6 +65,8 @@ export async function processScheduledSyncJob(_job: Job<ScheduledSyncJobData>, d logger.info( `[scheduled-sync] Enqueued ${jobCount} sync jobs for ${userProviders.size} users` + - (skippedDueToCooldown > 0 ? ` (${skippedDueToCooldown} skipped due to rate-limit cooldown)` : ""), + (skippedDueToCooldown > 0 + ? ` (${skippedDueToCooldown} skipped due to rate-limit cooldown)` + : ""), ); } diff --git a/src/jobs/provider-rate-limit-cooldown.test.ts b/src/jobs/provider-rate-limit-cooldown.test.ts index efcceecc3a..5b669c75f8 100644 --- a/src/jobs/provider-rate-limit-cooldown.test.ts +++ b/src/jobs/provider-rate-limit-cooldown.test.ts @@ -1,5 +1,6 @@ import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; import { describe, expect, it, vi } from "vitest"; +import { providerAdaptiveRateLimitStore } from "../lib/provider-adaptive-rate-limit.ts"; import { InMemoryProviderRateLimitCooldownStore, providerRateLimitCooldownJobId, @@ -50,6 +51,10 @@ function createMockRedisStore() { } describe("ProviderRateLimitCooldownStore", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + it("records provider-wide cooldown state using Retry-After when present", async () => { vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); const store = new InMemoryProviderRateLimitCooldownStore(); @@ -134,6 +139,84 @@ describe("ProviderRateLimitCooldownStore", () => { vi.useRealTimers(); }); + it("resets consecutive hits after the strike reset window elapses", async () => { + vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); + const store = new InMemoryProviderRateLimitCooldownStore(); + + await store.record(rateLimitError({ providerId: "garmin" }), "user-1"); + vi.setSystemTime(new Date("2026-06-02T15:00:01Z")); + + const second = await store.record(rateLimitError({ providerId: "garmin" }), "user-1"); + expect(second.consecutiveHits).toBe(1); + vi.useRealTimers(); + }); + + it("keeps escalating strikes within the strike reset window", async () => { + vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); + const store = new InMemoryProviderRateLimitCooldownStore(); + + await store.record(rateLimitError({ providerId: "garmin" }), "user-1"); + vi.setSystemTime(new Date("2026-06-02T15:00:00Z")); + + const second = await store.record(rateLimitError({ providerId: "garmin" }), "user-1"); + expect(second.consecutiveHits).toBe(2); + vi.useRealTimers(); + }); + + it("caps escalation at the default max cooldown for providers without a custom max", async () => { + vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); + const store = new InMemoryProviderRateLimitCooldownStore(); + + let cooldown = await store.record(rateLimitError({ providerId: "strava" }), "user-1"); + for (let hit = 1; hit < 4; hit++) { + vi.setSystemTime(cooldown.expiresAt); + cooldown = await store.record(rateLimitError({ providerId: "strava" }), "user-1"); + } + + expect(cooldown.consecutiveHits).toBe(4); + expect(cooldown.expiresAt).toEqual(new Date("2026-06-02T15:45:00Z")); + vi.useRealTimers(); + }); + + it("uses learned cooldown seconds from the adaptive store when Retry-After is absent", async () => { + vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); + vi.spyOn(providerAdaptiveRateLimitStore, "getLearnedCooldownSeconds").mockResolvedValue(300); + const store = new InMemoryProviderRateLimitCooldownStore(); + + const cooldown = await store.record(rateLimitError({ providerId: "garmin" }), "user-1"); + + expect(cooldown.expiresAt).toEqual(new Date("2026-06-02T12:05:00Z")); + vi.useRealTimers(); + }); + + it("ignores negative Retry-After values and uses escalated fallback cooldown", async () => { + vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); + const store = new InMemoryProviderRateLimitCooldownStore(); + + const cooldown = await store.record( + rateLimitError({ providerId: "garmin", retryAfterSeconds: -10 }), + "user-1", + ); + + expect(cooldown.expiresAt).toEqual(new Date("2026-06-02T13:00:00Z")); + vi.useRealTimers(); + }); + + it("updates consecutive hits when a new cooldown matches the active expiry", async () => { + vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); + const store = new InMemoryProviderRateLimitCooldownStore(); + + await store.record(rateLimitError({ providerId: "garmin", retryAfterSeconds: 600 }), "user-1"); + const second = await store.record( + rateLimitError({ providerId: "garmin", retryAfterSeconds: 600 }), + "user-1", + ); + + expect(second.consecutiveHits).toBe(2); + expect(second.expiresAt).toEqual(new Date("2026-06-02T12:10:00Z")); + vi.useRealTimers(); + }); + it("returns the active cooldown with the later expiry across provider and user scopes", async () => { vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); const store = new InMemoryProviderRateLimitCooldownStore(); @@ -336,6 +419,18 @@ describe("ProviderRateLimitCooldownStore", () => { vi.useRealTimers(); }); + it("uses learned cooldown seconds in Redis when Retry-After is absent", async () => { + vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); + vi.spyOn(providerAdaptiveRateLimitStore, "getLearnedCooldownSeconds").mockResolvedValue(300); + const { setCalls, store } = createMockRedisStore(); + + const cooldown = await store.record(rateLimitError({ providerId: "garmin" }), "user-1"); + + expect(cooldown.expiresAt).toEqual(new Date("2026-06-02T12:05:00Z")); + expect(setCalls[0]?.millisecondsToExpire).toBe(300_000); + vi.useRealTimers(); + }); + it("ignores malformed Redis cooldown state", async () => { const { values, store } = createMockRedisStore(); @@ -362,6 +457,13 @@ describe("ProviderRateLimitCooldownStore", () => { userId: null, expiresAt: "not-a-date", }), + JSON.stringify({ + providerId: "garmin", + scope: "provider", + userId: null, + expiresAt: "2026-06-02T12:00:00.000Z", + consecutiveHits: "not-a-number", + }), ]) { values.set("provider-rate-limit:garmin:provider", raw); await expect(store.getActive("garmin", "user-1")).resolves.toBeNull(); diff --git a/src/jobs/provider-rate-limit-cooldown.ts b/src/jobs/provider-rate-limit-cooldown.ts index 4237fc58d2..95a82992ae 100644 --- a/src/jobs/provider-rate-limit-cooldown.ts +++ b/src/jobs/provider-rate-limit-cooldown.ts @@ -182,7 +182,9 @@ export class InMemoryProviderRateLimitCooldownStore implements ProviderRateLimit error: ProviderRateLimitError, fallbackUserId: string, ): Promise<ProviderRateLimitCooldown> { - const learned = await providerAdaptiveRateLimitStore.getLearnedCooldownSeconds(error.providerId); + const learned = await providerAdaptiveRateLimitStore.getLearnedCooldownSeconds( + error.providerId, + ); const baseFallback = learned ?? fallbackCooldownSeconds(error.providerId); const cooldown = cooldownFromError(error, fallbackUserId, null, baseFallback); const key = cooldownKey(cooldown.providerId, cooldown.scope, cooldown.userId); @@ -232,7 +234,9 @@ export class RedisProviderRateLimitCooldownStore implements ProviderRateLimitCoo error: ProviderRateLimitError, fallbackUserId: string, ): Promise<ProviderRateLimitCooldown> { - const learned = await providerAdaptiveRateLimitStore.getLearnedCooldownSeconds(error.providerId); + const learned = await providerAdaptiveRateLimitStore.getLearnedCooldownSeconds( + error.providerId, + ); const baseFallback = learned ?? fallbackCooldownSeconds(error.providerId); const cooldown = cooldownFromError(error, fallbackUserId, null, baseFallback); const key = cooldownKey(cooldown.providerId, cooldown.scope, cooldown.userId); diff --git a/src/lib/provider-adaptive-rate-limit.ts b/src/lib/provider-adaptive-rate-limit.ts index e5f8cc716a..dc5ca9cbf4 100644 --- a/src/lib/provider-adaptive-rate-limit.ts +++ b/src/lib/provider-adaptive-rate-limit.ts @@ -1,16 +1,19 @@ import { ADAPTIVE_RATE_WINDOW_MS, + type AdaptiveRateLimitStore, admissionDelayMs, applyStravaQuota, createInitialAdaptiveState, - type AdaptiveRateLimitStore, type ProviderAdaptiveRateState, parseStravaRateLimitHeaders, recordAdaptiveRateLimit, recordAdaptiveRequest, slideAdaptiveWindow, } from "@dofek/provider-http/adaptive-rate-limit"; -import type { ProviderRateLimitError, ProviderRateLimitScope } from "@dofek/provider-http/rate-limit"; +import type { + ProviderRateLimitError, + ProviderRateLimitScope, +} from "@dofek/provider-http/rate-limit"; import { RedisConnection } from "bullmq"; import { getRedisConnection } from "../jobs/queues.ts"; @@ -60,7 +63,10 @@ function parseState(raw: string | null): ProviderAdaptiveRateState | null { if (typeof windowStartMs !== "number" || !Number.isFinite(windowStartMs)) return null; if (typeof requestCount !== "number" || !Number.isFinite(requestCount)) return null; if (typeof throttleMs !== "number" || !Number.isFinite(throttleMs)) return null; - if (lastRequestMs !== null && (typeof lastRequestMs !== "number" || !Number.isFinite(lastRequestMs))) { + if ( + lastRequestMs !== null && + (typeof lastRequestMs !== "number" || !Number.isFinite(lastRequestMs)) + ) { return null; } diff --git a/src/providers/amazfit-zepp.test.ts b/src/providers/amazfit-zepp.test.ts index d37c26a7f1..bda4c4def8 100644 --- a/src/providers/amazfit-zepp.test.ts +++ b/src/providers/amazfit-zepp.test.ts @@ -1,10 +1,10 @@ import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; -import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { captureException } from "@sentry/node"; import { afterEach, describe, expect, it, vi } from "vitest"; import { ZeppInvalidCredentialsError } from "zepp-client/client"; import { activity as activityTable } from "../db/schema.ts"; import { runWithTokenUser } from "../db/token-user-context.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { AmazfitZeppClient, AmazfitZeppProvider, diff --git a/src/providers/amazfit-zepp.ts b/src/providers/amazfit-zepp.ts index 38394416b9..6dd272c028 100644 --- a/src/providers/amazfit-zepp.ts +++ b/src/providers/amazfit-zepp.ts @@ -1,5 +1,4 @@ import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; -import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { CanonicalActivityType } from "@dofek/training/training"; import { captureException } from "@sentry/node"; import { signInToZepp, ZeppInvalidCredentialsError } from "zepp-client/client"; @@ -12,6 +11,7 @@ import { SOURCE_TYPE_API } from "../db/sensor-channels.ts"; import { withSyncLog } from "../db/sync-log.ts"; import { getTokenUserId } from "../db/token-user-context.ts"; import { ensureProvider, loadTokens } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { ProviderInvalidCredentialsError, ProviderStoredIdentityMissingError, diff --git a/src/providers/bodyspec.ts b/src/providers/bodyspec.ts index 3e6caea7ca..7d3f2ad72c 100644 --- a/src/providers/bodyspec.ts +++ b/src/providers/bodyspec.ts @@ -1,4 +1,3 @@ -import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { z } from "zod"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; import { exchangeCodeForTokens, getOAuthRedirectUri } from "../auth/oauth.ts"; @@ -7,6 +6,7 @@ import type { SyncDatabase } from "../db/index.ts"; import { dexaScan, dexaScanRegion } from "../db/schema.ts"; import { withSyncLog } from "../db/sync-log.ts"; import { ensureProvider } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { ProviderHttpClient } from "./http-client.ts"; import type { SyncRun } from "./sync-run.ts"; import type { ProviderAuthSetup, SyncError, SyncProvider, SyncResult } from "./types.ts"; diff --git a/src/providers/concept2.ts b/src/providers/concept2.ts index b0c72f5e90..9a53ecca39 100644 --- a/src/providers/concept2.ts +++ b/src/providers/concept2.ts @@ -1,4 +1,3 @@ -import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { CanonicalActivityType } from "@dofek/training/training"; import { z } from "zod"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; @@ -13,6 +12,7 @@ import { activity } from "../db/schema.ts"; import { withSyncLog } from "../db/sync-log.ts"; import { getTokenUserId } from "../db/token-user-context.ts"; import { ensureProvider } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { ProviderHttpClient } from "./http-client.ts"; import type { SyncRun } from "./sync-run.ts"; import type { diff --git a/src/providers/coros.ts b/src/providers/coros.ts index 4446969f3b..107c0c287d 100644 --- a/src/providers/coros.ts +++ b/src/providers/coros.ts @@ -1,4 +1,3 @@ -import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { CanonicalActivityType } from "@dofek/training/training"; import { z } from "zod"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; @@ -13,6 +12,7 @@ import { withSyncLog } from "../db/sync-log.ts"; import { ensureProvider } from "../db/tokens.ts"; import { parseFitFile } from "../fit/parser.ts"; import { fitRecordsToSensorSamples } from "../fit/records.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { logger } from "../logger.ts"; import { ProviderHttpClient } from "./http-client.ts"; import type { SyncRun } from "./sync-run.ts"; diff --git a/src/providers/cycling-analytics.ts b/src/providers/cycling-analytics.ts index 9aaf03c24a..62e12d6b62 100644 --- a/src/providers/cycling-analytics.ts +++ b/src/providers/cycling-analytics.ts @@ -1,4 +1,3 @@ -import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { CanonicalActivityType } from "@dofek/training/training"; import { z } from "zod"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; @@ -9,6 +8,7 @@ import { reconcileProviderActivityAbsence } from "../db/provider-activity-absenc import { activity } from "../db/schema.ts"; import { withSyncLog } from "../db/sync-log.ts"; import { ensureProvider } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { SyncRun } from "./sync-run.ts"; import type { ProviderAuthSetup, SyncError, SyncProvider, SyncResult } from "./types.ts"; diff --git a/src/providers/decathlon.ts b/src/providers/decathlon.ts index 7901fcde91..f3952d0ae3 100644 --- a/src/providers/decathlon.ts +++ b/src/providers/decathlon.ts @@ -1,4 +1,3 @@ -import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { CanonicalActivityType } from "@dofek/training/training"; import { z } from "zod"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; @@ -9,6 +8,7 @@ import { reconcileProviderActivityAbsence } from "../db/provider-activity-absenc import { activity } from "../db/schema.ts"; import { withSyncLog } from "../db/sync-log.ts"; import { ensureProvider } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { SyncRun } from "./sync-run.ts"; import type { ProviderAuthSetup, SyncError, SyncProvider, SyncResult } from "./types.ts"; diff --git a/src/providers/eight-sleep.ts b/src/providers/eight-sleep.ts index a63756f975..3b95a29ab9 100644 --- a/src/providers/eight-sleep.ts +++ b/src/providers/eight-sleep.ts @@ -1,4 +1,3 @@ -import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { EightSleepClient } from "eight-sleep-client/client"; import { parseEightSleepDailyMetrics, @@ -11,6 +10,7 @@ import { dailyMetrics, sleepSession } from "../db/schema.ts"; import { SOURCE_TYPE_API } from "../db/sensor-channels.ts"; import { withSyncLog } from "../db/sync-log.ts"; import { ensureProvider, loadTokens } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { AccessTokenExpiredError, ProviderStoredIdentityMissingError } from "./auth-errors.ts"; import type { SyncRun } from "./sync-run.ts"; import type { ProviderAuthSetup, SyncError, SyncProvider, SyncResult } from "./types.ts"; diff --git a/src/providers/fatsecret/provider.ts b/src/providers/fatsecret/provider.ts index 1bd4c25a4d..eb1d71777b 100644 --- a/src/providers/fatsecret/provider.ts +++ b/src/providers/fatsecret/provider.ts @@ -1,5 +1,4 @@ import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; -import { createProviderRateLimitFetch } from "../../lib/provider-rate-limit-fetch.ts"; import { sql } from "drizzle-orm"; import { z } from "zod"; import { getOAuthRedirectUri } from "../../auth/oauth.ts"; @@ -7,6 +6,7 @@ import { nutrientAmountEntriesFromLegacyFields } from "../../db/nutrient-columns import { foodEntry, foodEntryNutrient } from "../../db/schema.ts"; import { getTokenUserId } from "../../db/token-user-context.ts"; import { ensureProvider } from "../../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../../lib/provider-rate-limit-fetch.ts"; import { logger } from "../../logger.ts"; import type { SyncRun } from "../sync-run.ts"; import type { SyncError, SyncProvider, SyncResult } from "../types.ts"; diff --git a/src/providers/fitbit/provider.ts b/src/providers/fitbit/provider.ts index a2017e5a54..ad1de546ed 100644 --- a/src/providers/fitbit/provider.ts +++ b/src/providers/fitbit/provider.ts @@ -1,5 +1,4 @@ import { createHmac } from "node:crypto"; -import { createProviderRateLimitFetch } from "../../lib/provider-rate-limit-fetch.ts"; import { z } from "zod"; import type { OAuthConfig, TokenSet } from "../../auth/oauth.ts"; import { @@ -14,6 +13,7 @@ import type { SyncDatabase } from "../../db/index.ts"; import { reconcileProviderActivityAbsence } from "../../db/provider-activity-absence.ts"; import { withSyncLog } from "../../db/sync-log.ts"; import { ensureProvider } from "../../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../../lib/provider-rate-limit-fetch.ts"; import type { SyncRun } from "../sync-run.ts"; import type { ProviderAuthSetup, diff --git a/src/providers/garmin.test.ts b/src/providers/garmin.test.ts index 78c499d941..4f2f222766 100644 --- a/src/providers/garmin.test.ts +++ b/src/providers/garmin.test.ts @@ -199,14 +199,11 @@ function createMockDb(): MockDb { db.values.mockReturnValue(db); db.onConflictDoUpdate.mockReturnValue(db); db.delete.mockReturnValue(db); - Object.assign(db, { - then( - onFulfilled: (value: unknown) => unknown, - onRejected?: (reason: unknown) => unknown, - ) { - return Promise.resolve([]).then(onFulfilled, onRejected); - }, - }); + db.where.mockReturnValue( + Object.assign(Promise.resolve([]), { + limit: vi.fn().mockResolvedValue([]), + }), + ); return db; } @@ -710,14 +707,11 @@ describe("GarminProvider.sync()", () => { raw: rawActivity, }); - Object.assign(db, { - then( - onFulfilled: (value: unknown) => unknown, - onRejected?: (reason: unknown) => unknown, - ) { - return Promise.resolve([{ externalId: "123" }]).then(onFulfilled, onRejected); - }, - }); + db.where.mockReturnValue( + Object.assign(Promise.resolve([{ externalId: "123" }]), { + limit: vi.fn().mockResolvedValue([{ externalId: "123" }]), + }), + ); const result = await syncProvider(provider, db, new Date("2026-02-01T00:00:00Z")); diff --git a/src/providers/garmin.ts b/src/providers/garmin.ts index e295f3e908..6108526303 100644 --- a/src/providers/garmin.ts +++ b/src/providers/garmin.ts @@ -1,4 +1,3 @@ -import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { isIndoorCycling } from "@dofek/training/endurance-types"; import { captureException } from "@sentry/node"; import { and, eq, inArray } from "drizzle-orm"; @@ -24,6 +23,7 @@ import { SOURCE_TYPE_API } from "../db/sensor-channels.ts"; import { withSyncLog } from "../db/sync-log.ts"; import { getTokenUserId } from "../db/token-user-context.ts"; import { ensureProvider, loadTokens, saveTokens } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { isRetryableInfraError } from "../lib/retryable-infra-error.ts"; import { logger } from "../logger.ts"; import { ProviderAuthenticationFailedError } from "./auth-errors.ts"; @@ -297,7 +297,8 @@ export class GarminProvider implements SyncProvider { #fetchFn: typeof globalThis.fetch; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createProviderRateLimitFetch("garmin", fetchFn, { createRateLimitError: (response, responseBody) => + this.#fetchFn = createProviderRateLimitFetch("garmin", fetchFn, { + createRateLimitError: (response, responseBody) => new GarminRateLimitError( `Rate limit exceeded (${response.status}): ${responseBody}`, responseBody, diff --git a/src/providers/http-client.ts b/src/providers/http-client.ts index af3b4e6ec6..cd1338aa33 100644 --- a/src/providers/http-client.ts +++ b/src/providers/http-client.ts @@ -9,8 +9,8 @@ * - Fetch function injection for testability */ -import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { z } from "zod"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; /** * Base class for providers that make HTTP requests with bearer token auth. diff --git a/src/providers/komoot.ts b/src/providers/komoot.ts index 4e2e57f1fd..0268ade0d8 100644 --- a/src/providers/komoot.ts +++ b/src/providers/komoot.ts @@ -1,4 +1,3 @@ -import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { CanonicalActivityType } from "@dofek/training/training"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; import { exchangeCodeForTokens, getOAuthRedirectUri } from "../auth/oauth.ts"; @@ -8,6 +7,7 @@ import { reconcileProviderActivityAbsence } from "../db/provider-activity-absenc import { activity } from "../db/schema.ts"; import { withSyncLog } from "../db/sync-log.ts"; import { ensureProvider } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { SyncRun } from "./sync-run.ts"; import type { ProviderAuthSetup, SyncError, SyncProvider, SyncResult } from "./types.ts"; diff --git a/src/providers/mapmyfitness.test.ts b/src/providers/mapmyfitness.test.ts index 618e5a4e73..49295ec37e 100644 --- a/src/providers/mapmyfitness.test.ts +++ b/src/providers/mapmyfitness.test.ts @@ -1,5 +1,3 @@ -import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; -import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { afterEach, describe, expect, it, vi } from "vitest"; import { SyncRun } from "./sync-run.ts"; import { SyncWindow } from "./sync-window.ts"; diff --git a/src/providers/mapmyfitness.ts b/src/providers/mapmyfitness.ts index 18989fe7e3..e5df3b20ed 100644 --- a/src/providers/mapmyfitness.ts +++ b/src/providers/mapmyfitness.ts @@ -1,4 +1,3 @@ -import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { CanonicalActivityType } from "@dofek/training/training"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; import { exchangeCodeForTokens, getOAuthRedirectUri } from "../auth/oauth.ts"; @@ -8,6 +7,7 @@ import { reconcileProviderActivityAbsence } from "../db/provider-activity-absenc import { activity } from "../db/schema.ts"; import { withSyncLog } from "../db/sync-log.ts"; import { ensureProvider } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { SyncRun } from "./sync-run.ts"; import type { ProviderAuthSetup, SyncError, SyncProvider, SyncResult } from "./types.ts"; diff --git a/src/providers/oura/provider.ts b/src/providers/oura/provider.ts index fff54b8820..ab3bf1b763 100644 --- a/src/providers/oura/provider.ts +++ b/src/providers/oura/provider.ts @@ -1,4 +1,3 @@ -import { createProviderRateLimitFetch } from "../../lib/provider-rate-limit-fetch.ts"; import { z } from "zod"; import { exchangeCodeForTokens } from "../../auth/oauth.ts"; import { resolveOAuthTokens } from "../../auth/resolve-tokens.ts"; @@ -8,6 +7,7 @@ import { reconcileProviderActivityAbsence, } from "../../db/provider-activity-absence.ts"; import { ensureProvider } from "../../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../../lib/provider-rate-limit-fetch.ts"; import type { SyncRun } from "../sync-run.ts"; import type { ProviderAuthSetup, diff --git a/src/providers/peloton.ts b/src/providers/peloton.ts index b1623e6993..265ab8b33d 100644 --- a/src/providers/peloton.ts +++ b/src/providers/peloton.ts @@ -1,4 +1,3 @@ -import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { CanonicalActivityType } from "@dofek/training/training"; import { z } from "zod"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; @@ -16,6 +15,7 @@ import { activity } from "../db/schema.ts"; import { SOURCE_TYPE_API } from "../db/sensor-channels.ts"; import { withSyncLog } from "../db/sync-log.ts"; import { ensureProvider } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { logger } from "../logger.ts"; import type { SyncRun } from "./sync-run.ts"; import type { ProviderAuthSetup, SyncError, SyncProvider, SyncResult } from "./types.ts"; diff --git a/src/providers/polar/client.test.ts b/src/providers/polar/client.test.ts index 491afa6103..4c2ca9e57a 100644 --- a/src/providers/polar/client.test.ts +++ b/src/providers/polar/client.test.ts @@ -1,5 +1,3 @@ -import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; -import { createProviderRateLimitFetch } from "../../lib/provider-rate-limit-fetch.ts"; import { describe, expect, it } from "vitest"; import { PolarClient } from "./client.ts"; diff --git a/src/providers/polar/provider.ts b/src/providers/polar/provider.ts index 0358c328fd..2e064b9e79 100644 --- a/src/providers/polar/provider.ts +++ b/src/providers/polar/provider.ts @@ -1,5 +1,5 @@ -import { createProviderRateLimitFetch } from "../../lib/provider-rate-limit-fetch.ts"; import type { TokenSet } from "../../auth/oauth.ts"; +import { createProviderRateLimitFetch } from "../../lib/provider-rate-limit-fetch.ts"; import { logger } from "../../logger.ts"; import type { SyncRun } from "../sync-run.ts"; import type { ProviderAuthSetup, SyncResult, WebhookEvent, WebhookProvider } from "../types.ts"; diff --git a/src/providers/polar/webhook-service.ts b/src/providers/polar/webhook-service.ts index 6f3a6038cc..97f3184463 100644 --- a/src/providers/polar/webhook-service.ts +++ b/src/providers/polar/webhook-service.ts @@ -1,6 +1,6 @@ import { createHmac, timingSafeEqual } from "node:crypto"; -import { createProviderRateLimitFetch } from "../../lib/provider-rate-limit-fetch.ts"; import { z } from "zod"; +import { createProviderRateLimitFetch } from "../../lib/provider-rate-limit-fetch.ts"; import type { WebhookEvent } from "../types.ts"; const POLAR_WEBHOOK_URL = "https://www.polaraccesslink.com/v3/webhooks"; diff --git a/src/providers/ride-with-gps.ts b/src/providers/ride-with-gps.ts index 834cfe884d..1c6d20d575 100644 --- a/src/providers/ride-with-gps.ts +++ b/src/providers/ride-with-gps.ts @@ -1,4 +1,3 @@ -import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { isIndoorCycling } from "@dofek/training/endurance-types"; import { type CanonicalActivityType, @@ -17,6 +16,7 @@ import { activity, userSettings } from "../db/schema.ts"; import { SOURCE_TYPE_API } from "../db/sensor-channels.ts"; import { getTokenUserId } from "../db/token-user-context.ts"; import { ensureProvider } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { SyncRun } from "./sync-run.ts"; import type { ProviderAuthSetup, diff --git a/src/providers/strava.ts b/src/providers/strava.ts index 73d4db548e..66aaad0de9 100644 --- a/src/providers/strava.ts +++ b/src/providers/strava.ts @@ -1,5 +1,4 @@ import { ProviderRateLimitError, parseRetryAfterHeader } from "@dofek/provider-http/rate-limit"; -import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { isIndoorCycling } from "@dofek/training/endurance-types"; import { type CanonicalActivityType, @@ -24,6 +23,7 @@ import { import { activity } from "../db/schema.ts"; import { SOURCE_TYPE_API } from "../db/sensor-channels.ts"; import { getTokenUserId } from "../db/token-user-context.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { logger } from "../logger.ts"; import { ProviderAuthorizationFailedError } from "./auth-errors.ts"; import type { SyncRun } from "./sync-run.ts"; @@ -256,28 +256,12 @@ export const STRAVA_THROTTLE_MS = 10_000; export class StravaClient { #accessToken: string; #fetchFn: typeof globalThis.fetch; - #lastRequestTime = 0; - #throttleMs: number; - - constructor( - accessToken: string, - fetchFn: typeof globalThis.fetch = globalThis.fetch, - throttleMs = STRAVA_THROTTLE_MS, - ) { + + constructor(accessToken: string, fetchFn: typeof globalThis.fetch = globalThis.fetch) { this.#accessToken = accessToken; - this.#fetchFn = createProviderRateLimitFetch("strava", fetchFn, { createRateLimitError: createStravaRateLimitError, + this.#fetchFn = createProviderRateLimitFetch("strava", fetchFn, { + createRateLimitError: createStravaRateLimitError, }); - this.#throttleMs = throttleMs; - } - - async #throttle(): Promise<void> { - if (this.#throttleMs <= 0) return; - const now = Date.now(); - const elapsed = now - this.#lastRequestTime; - if (this.#lastRequestTime > 0 && elapsed < this.#throttleMs) { - await new Promise((resolve) => setTimeout(resolve, this.#throttleMs - elapsed)); - } - this.#lastRequestTime = Date.now(); } async #get<T>(path: string, params?: Record<string, string>): Promise<T> { @@ -432,15 +416,11 @@ export class StravaProvider implements WebhookProvider { readonly name = "Strava"; readonly webhookScope = "app" as const; #fetchFn: typeof globalThis.fetch; - #throttleMs: number; - constructor( - fetchFn: typeof globalThis.fetch = globalThis.fetch, - throttleMs = STRAVA_THROTTLE_MS, - ) { - this.#fetchFn = createProviderRateLimitFetch("strava", fetchFn, { createRateLimitError: createStravaRateLimitError, + constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { + this.#fetchFn = createProviderRateLimitFetch("strava", fetchFn, { + createRateLimitError: createStravaRateLimitError, }); - this.#throttleMs = throttleMs; } validate(): string | null { @@ -645,7 +625,7 @@ export class StravaProvider implements WebhookProvider { return { provider: this.id, recordsSynced, errors, duration: Date.now() - start }; } - const client = new StravaClient(tokens.accessToken, this.#fetchFn, this.#throttleMs); + const client = new StravaClient(tokens.accessToken, this.#fetchFn); // Fetch the single activity detail (1 API call) const detail = await client.getActivity(activityExternalId); @@ -739,7 +719,7 @@ export class StravaProvider implements WebhookProvider { return { provider: this.id, recordsSynced, errors, duration: Date.now() - start }; } - const client = new StravaClient(tokens.accessToken, this.#fetchFn, this.#throttleMs); + const client = new StravaClient(tokens.accessToken, this.#fetchFn); const since = window.since; const syncWindowEnd = window.until; diff --git a/src/providers/suunto.ts b/src/providers/suunto.ts index 45a43b4289..d8f58cda80 100644 --- a/src/providers/suunto.ts +++ b/src/providers/suunto.ts @@ -1,5 +1,4 @@ import { createHmac, timingSafeEqual } from "node:crypto"; -import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { CanonicalActivityType } from "@dofek/training/training"; import { z } from "zod"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; @@ -14,6 +13,7 @@ import { withSyncLog } from "../db/sync-log.ts"; import { ensureProvider } from "../db/tokens.ts"; import { parseFitFile } from "../fit/parser.ts"; import { fitRecordsToSensorSamples } from "../fit/records.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { logger } from "../logger.ts"; import type { SyncRun } from "./sync-run.ts"; import type { diff --git a/src/providers/trainerroad.ts b/src/providers/trainerroad.ts index 432c43e0d4..deeb7fd10e 100644 --- a/src/providers/trainerroad.ts +++ b/src/providers/trainerroad.ts @@ -1,10 +1,10 @@ -import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { TrainerRoadClient } from "trainerroad-client/client"; import { parseTrainerRoadActivity } from "trainerroad-client/parsing"; import { reconcileProviderActivityAbsence } from "../db/provider-activity-absence.ts"; import { activity } from "../db/schema.ts"; import { withSyncLog } from "../db/sync-log.ts"; import { ensureProvider, loadTokens } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { ProviderSessionExpiredError, ProviderStoredIdentityMissingError } from "./auth-errors.ts"; import type { SyncRun } from "./sync-run.ts"; import type { ProviderAuthSetup, SyncError, SyncProvider, SyncResult } from "./types.ts"; diff --git a/src/providers/ultrahuman.ts b/src/providers/ultrahuman.ts index ffb9deb0f7..467bce3b8b 100644 --- a/src/providers/ultrahuman.ts +++ b/src/providers/ultrahuman.ts @@ -1,8 +1,8 @@ import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; -import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { dailyMetrics, sleepSession } from "../db/schema.ts"; import { withSyncLog } from "../db/sync-log.ts"; import { ensureProvider, loadTokens } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { SyncRun } from "./sync-run.ts"; import type { SyncError, SyncProvider, SyncResult } from "./types.ts"; diff --git a/src/providers/velohero.ts b/src/providers/velohero.ts index b4b4739ea9..00e0579275 100644 --- a/src/providers/velohero.ts +++ b/src/providers/velohero.ts @@ -1,10 +1,10 @@ -import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { VeloHeroClient } from "velohero-client/client"; import { parseVeloHeroWorkout } from "velohero-client/parsing"; import { reconcileProviderActivityAbsence } from "../db/provider-activity-absence.ts"; import { activity } from "../db/schema.ts"; import { withSyncLog } from "../db/sync-log.ts"; import { ensureProvider, loadTokens } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { logger } from "../logger.ts"; import { ProviderSessionExpiredError } from "./auth-errors.ts"; import type { SyncRun } from "./sync-run.ts"; diff --git a/src/providers/wahoo/provider.ts b/src/providers/wahoo/provider.ts index 3071643243..492419a7b8 100644 --- a/src/providers/wahoo/provider.ts +++ b/src/providers/wahoo/provider.ts @@ -1,4 +1,3 @@ -import { createProviderRateLimitFetch } from "../../lib/provider-rate-limit-fetch.ts"; import type { OAuthConfig, TokenSet } from "../../auth/oauth.ts"; import { exchangeCodeForTokens, @@ -8,6 +7,7 @@ import { import { resolveOAuthTokens } from "../../auth/resolve-tokens.ts"; import type { SyncDatabase } from "../../db/index.ts"; import { reconcileProviderActivityAbsence } from "../../db/provider-activity-absence.ts"; +import { createProviderRateLimitFetch } from "../../lib/provider-rate-limit-fetch.ts"; import { logger } from "../../logger.ts"; import { AccessTokenExpiredError } from "../auth-errors.ts"; import type { SyncRun } from "../sync-run.ts"; diff --git a/src/providers/wger.ts b/src/providers/wger.ts index 738cf8d791..42172a9710 100644 --- a/src/providers/wger.ts +++ b/src/providers/wger.ts @@ -1,4 +1,3 @@ -import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { CanonicalActivityType } from "@dofek/training/training"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; import { exchangeCodeForTokens, getOAuthRedirectUri } from "../auth/oauth.ts"; @@ -10,6 +9,7 @@ import { activity } from "../db/schema.ts"; import { SOURCE_TYPE_API } from "../db/sensor-channels.ts"; import { withSyncLog } from "../db/sync-log.ts"; import { ensureProvider } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { SyncRun } from "./sync-run.ts"; import type { ProviderAuthSetup, SyncError, SyncProvider, SyncResult } from "./types.ts"; diff --git a/src/providers/whoop.test.ts b/src/providers/whoop.test.ts index 8118009a5d..b51a4d251e 100644 --- a/src/providers/whoop.test.ts +++ b/src/providers/whoop.test.ts @@ -1,5 +1,4 @@ import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; -import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { WhoopClient } from "whoop-whoop/client"; import type { diff --git a/src/providers/whoop/provider.ts b/src/providers/whoop/provider.ts index 49ce85bb7f..74100dcdfe 100644 --- a/src/providers/whoop/provider.ts +++ b/src/providers/whoop/provider.ts @@ -1,10 +1,10 @@ -import { createProviderRateLimitFetch } from "../../lib/provider-rate-limit-fetch.ts"; import { WhoopClient } from "whoop-whoop/client"; import type { WhoopCycle } from "whoop-whoop/types"; import { z } from "zod"; import type { OAuthConfig } from "../../auth/oauth.ts"; import { exchangeCodeForTokens, getOAuthRedirectUri } from "../../auth/oauth.ts"; import { ensureProvider, loadTokens, saveTokens } from "../../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../../lib/provider-rate-limit-fetch.ts"; import { logger } from "../../logger.ts"; import { ProviderStoredIdentityMissingError } from "../auth-errors.ts"; import type { SyncRun } from "../sync-run.ts"; @@ -15,6 +15,7 @@ import type { SyncProvider, SyncResult, } from "../types.ts"; +import { findWhoopRateLimitError, isWhoopRateLimitError } from "./rate-limit.ts"; import { syncWhoopDailyActivity } from "./sync-daily-activity.ts"; import { syncWhoopJournal } from "./sync-journal.ts"; import { syncWhoopRecovery } from "./sync-recovery.ts"; @@ -22,7 +23,6 @@ import { syncWhoopSleepSessions, syncWhoopSleepStages } from "./sync-sleep.ts"; import { syncWhoopHeartRateStream } from "./sync-streams.ts"; import type { WhoopSyncContext } from "./sync-types.ts"; import { syncWhoopStrength, syncWhoopWorkouts } from "./sync-workouts.ts"; -import { findWhoopRateLimitError, isWhoopRateLimitError } from "./rate-limit.ts"; // ============================================================ // Provider implementation diff --git a/src/providers/whoop/rate-limit.test.ts b/src/providers/whoop/rate-limit.test.ts index 3157e9e6f1..0ced390d5f 100644 --- a/src/providers/whoop/rate-limit.test.ts +++ b/src/providers/whoop/rate-limit.test.ts @@ -1,6 +1,6 @@ import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; -import { WhoopRateLimitError } from "whoop-whoop/client"; import { describe, expect, it } from "vitest"; +import { WhoopRateLimitError } from "whoop-whoop/client"; import { findWhoopRateLimitError, isWhoopRateLimitError } from "./rate-limit.ts"; describe("isWhoopRateLimitError", () => { diff --git a/src/providers/whoop/rate-limit.ts b/src/providers/whoop/rate-limit.ts index f07d9a253d..2e643b8701 100644 --- a/src/providers/whoop/rate-limit.ts +++ b/src/providers/whoop/rate-limit.ts @@ -8,7 +8,9 @@ export function isWhoopRateLimitError(err: unknown): boolean { ); } -export function findWhoopRateLimitError(errors: Array<{ cause?: unknown }>): ProviderRateLimitError | null { +export function findWhoopRateLimitError( + errors: Array<{ cause?: unknown }>, +): ProviderRateLimitError | null { for (const syncError of errors) { const cause = syncError.cause; if (isWhoopRateLimitError(cause) && cause instanceof ProviderRateLimitError) { diff --git a/src/providers/whoop/sync-helpers.test.ts b/src/providers/whoop/sync-helpers.test.ts index 37a0e11530..1f8d693f86 100644 --- a/src/providers/whoop/sync-helpers.test.ts +++ b/src/providers/whoop/sync-helpers.test.ts @@ -53,16 +53,12 @@ function makeDb(selectedRows: unknown[] = []) { chain.values.mockReturnValue(chain); chain.onConflictDoUpdate.mockResolvedValue(undefined); chain.from.mockReturnValue(chain); - chain.where.mockReturnValue(chain); + chain.where.mockReturnValue( + Object.assign(Promise.resolve(selectedRows), { + limit: vi.fn().mockResolvedValue(selectedRows), + }), + ); chain.limit.mockResolvedValue(selectedRows); - Object.assign(chain, { - then( - onFulfilled: (value: unknown) => unknown, - onRejected?: (reason: unknown) => unknown, - ) { - return Promise.resolve(selectedRows).then(onFulfilled, onRejected); - }, - }); const db: SyncDatabase = { insert: vi.fn().mockReturnValue(chain), diff --git a/src/providers/withings.ts b/src/providers/withings.ts index 2a1132f0ed..f2bce2fc7c 100644 --- a/src/providers/withings.ts +++ b/src/providers/withings.ts @@ -1,4 +1,3 @@ -import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { z } from "zod"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; import { getOAuthRedirectUri } from "../auth/oauth.ts"; @@ -7,6 +6,7 @@ import { writeMetricStreamBatch } from "../db/metric-stream-writer.ts"; import { SOURCE_TYPE_API } from "../db/sensor-channels.ts"; import { withSyncLog } from "../db/sync-log.ts"; import { deleteTokens, ensureProvider, loadTokens, saveTokens } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { logger } from "../logger.ts"; import { ProviderAuthenticationFailedError, RefreshTokenRevokedError } from "./auth-errors.ts"; import type { SyncRun } from "./sync-run.ts"; diff --git a/src/providers/xert.ts b/src/providers/xert.ts index dfaeaf9e94..d4e5949958 100644 --- a/src/providers/xert.ts +++ b/src/providers/xert.ts @@ -1,4 +1,3 @@ -import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { CanonicalActivityType } from "@dofek/training/training"; import { z } from "zod"; import type { OAuthConfig, TokenSet } from "../auth/oauth.ts"; @@ -9,6 +8,7 @@ import { reconcileProviderActivityAbsence } from "../db/provider-activity-absenc import { activity } from "../db/schema.ts"; import { withSyncLog } from "../db/sync-log.ts"; import { ensureProvider } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import type { SyncRun } from "./sync-run.ts"; import type { ProviderAuthSetup, SyncError, SyncProvider, SyncResult } from "./types.ts"; diff --git a/src/providers/zwift.ts b/src/providers/zwift.ts index d8c8a022d9..65edf3ae1e 100644 --- a/src/providers/zwift.ts +++ b/src/providers/zwift.ts @@ -1,4 +1,3 @@ -import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { z } from "zod"; import { ZWIFT_API_BASE, ZwiftClient } from "zwift-client/client"; import { parseZwiftActivity, parseZwiftFitnessData } from "zwift-client/parsing"; @@ -9,6 +8,7 @@ import { activity } from "../db/schema.ts"; import { SOURCE_TYPE_API } from "../db/sensor-channels.ts"; import { withSyncLog } from "../db/sync-log.ts"; import { ensureProvider, loadTokens, saveTokens } from "../db/tokens.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { logger } from "../logger.ts"; import { ProviderAuthenticationFailedError, From f7eed45637f98dfabdca09d11b06278c6ce19e13 Mon Sep 17 00:00:00 2001 From: Asher Cohen <asher.cohen@amplitude.com> Date: Sun, 21 Jun 2026 09:41:32 -0700 Subject: [PATCH 05/18] Fix Strava test signatures after removing client throttle. Update StravaProvider and StravaClient constructor calls across tests, restore missing imports, and drop the obsolete client-level throttle delay test. Co-authored-by: Cursor <cursoragent@cursor.com> --- src/jobs/provider-rate-limit-cooldown.test.ts | 2 +- src/providers/mapmyfitness.test.ts | 1 + src/providers/polar/client.test.ts | 1 + src/providers/strava-extra.test.ts | 52 +++---- src/providers/strava-sync.integration.test.ts | 12 +- src/providers/strava.test.ts | 131 +++++++----------- src/providers/webhook.test.ts | 46 +++--- 7 files changed, 107 insertions(+), 138 deletions(-) diff --git a/src/jobs/provider-rate-limit-cooldown.test.ts b/src/jobs/provider-rate-limit-cooldown.test.ts index 5b669c75f8..912523759e 100644 --- a/src/jobs/provider-rate-limit-cooldown.test.ts +++ b/src/jobs/provider-rate-limit-cooldown.test.ts @@ -1,5 +1,5 @@ import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; -import { describe, expect, it, vi } from "vitest"; +import { describe, afterEach, expect, it, vi } from "vitest"; import { providerAdaptiveRateLimitStore } from "../lib/provider-adaptive-rate-limit.ts"; import { InMemoryProviderRateLimitCooldownStore, diff --git a/src/providers/mapmyfitness.test.ts b/src/providers/mapmyfitness.test.ts index 49295ec37e..3f0182d4b2 100644 --- a/src/providers/mapmyfitness.test.ts +++ b/src/providers/mapmyfitness.test.ts @@ -1,3 +1,4 @@ +import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; import { afterEach, describe, expect, it, vi } from "vitest"; import { SyncRun } from "./sync-run.ts"; import { SyncWindow } from "./sync-window.ts"; diff --git a/src/providers/polar/client.test.ts b/src/providers/polar/client.test.ts index 4c2ca9e57a..6cd091c422 100644 --- a/src/providers/polar/client.test.ts +++ b/src/providers/polar/client.test.ts @@ -1,3 +1,4 @@ +import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; import { describe, expect, it } from "vitest"; import { PolarClient } from "./client.ts"; diff --git a/src/providers/strava-extra.test.ts b/src/providers/strava-extra.test.ts index 352df42dd2..b9a0f0b7df 100644 --- a/src/providers/strava-extra.test.ts +++ b/src/providers/strava-extra.test.ts @@ -69,7 +69,7 @@ describe("StravaClient", () => { ]), ); - const client = new StravaClient("test-token", mockFetch, 0); + const client = new StravaClient("test-token", mockFetch); const result = await client.getActivities(1000, 2, 50); expect(mockFetch).toHaveBeenCalledOnce(); @@ -119,7 +119,7 @@ describe("StravaClient", () => { ]), ); - const client = new StravaClient("test-token", mockFetch, 0); + const client = new StravaClient("test-token", mockFetch); const streams = await client.getActivityStreams(12345); const calledUrl = String(mockFetch.mock.calls[0]?.[0]); @@ -147,7 +147,7 @@ describe("StravaProvider.sync", () => { process.env.STRAVA_CLIENT_SECRET = "secret"; const mockFetch = vi.fn(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); // Mock db with loadTokens returning null const mockDb = { @@ -190,7 +190,7 @@ describe("StravaProvider.sync", () => { return Response.json([]); }); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); // Provide tokens const futureDate = new Date("2099-01-01"); @@ -360,7 +360,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), @@ -451,7 +451,7 @@ describe("StravaProvider.sync — additional coverage", () => { execute: vi.fn().mockResolvedValue([]), }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), ); @@ -502,7 +502,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: sinceDate }) }), @@ -531,7 +531,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: sinceDate }) }), @@ -567,7 +567,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), @@ -622,7 +622,7 @@ describe("StravaProvider.sync — additional coverage", () => { execute: vi.fn().mockResolvedValue([]), }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), ); @@ -652,7 +652,7 @@ describe("StravaProvider.sync — additional coverage", () => { const mockDb = createMockDb(); const onProgress = vi.fn(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); await provider.sync( new SyncRun({ @@ -680,7 +680,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), @@ -710,7 +710,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), @@ -732,7 +732,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const before = Date.now(); const result = await provider.sync( @@ -773,7 +773,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb([EXPIRED_TOKEN]); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), @@ -797,7 +797,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb([VALID_TOKEN]); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), @@ -852,7 +852,7 @@ describe("StravaProvider.sync — additional coverage", () => { execute: vi.fn().mockResolvedValue([]), }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), ); @@ -879,7 +879,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), ); @@ -907,7 +907,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), ); @@ -961,7 +961,7 @@ describe("StravaProvider.sync — additional coverage", () => { execute: vi.fn().mockResolvedValue([]), }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), ); @@ -999,7 +999,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), @@ -1056,7 +1056,7 @@ describe("StravaProvider.sync — additional coverage", () => { execute: vi.fn().mockResolvedValue([]), }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), ); @@ -1131,7 +1131,7 @@ describe("StravaProvider.sync — additional coverage", () => { execute: vi.fn().mockResolvedValue([]), }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), ); @@ -1159,7 +1159,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), ); @@ -1187,7 +1187,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), ); @@ -1247,7 +1247,7 @@ describe("StravaProvider.sync — additional coverage", () => { execute: vi.fn().mockResolvedValue([]), }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), ); diff --git a/src/providers/strava-sync.integration.test.ts b/src/providers/strava-sync.integration.test.ts index db3cd80f19..51925e2eed 100644 --- a/src/providers/strava-sync.integration.test.ts +++ b/src/providers/strava-sync.integration.test.ts @@ -179,7 +179,7 @@ describe("StravaProvider.sync() (integration)", () => { server.use(...stravaHandlers(activities)); - const provider = new StravaProvider(globalThis.fetch, 0); + const provider = new StravaProvider(globalThis.fetch); const since = new Date("2026-02-01T00:00:00Z"); const result = await provider.sync( new SyncRun({ @@ -230,7 +230,7 @@ describe("StravaProvider.sync() (integration)", () => { server.use(...stravaHandlers(activities)); - const provider = new StravaProvider(globalThis.fetch, 0); + const provider = new StravaProvider(globalThis.fetch); await provider.sync( new SyncRun({ db: ctx.db, @@ -261,7 +261,7 @@ describe("StravaProvider.sync() (integration)", () => { server.use(...stravaHandlers([])); - const provider = new StravaProvider(globalThis.fetch, 0); + const provider = new StravaProvider(globalThis.fetch); await provider.sync( new SyncRun({ db: ctx.db, @@ -287,7 +287,7 @@ describe("StravaProvider.sync() (integration)", () => { server.use(...stravaHandlers(activities, { streamsError: true })); - const provider = new StravaProvider(globalThis.fetch, 0); + const provider = new StravaProvider(globalThis.fetch); const result = await provider.sync( new SyncRun({ db: ctx.db, @@ -321,7 +321,7 @@ describe("StravaProvider.sync() (integration)", () => { server.use(...stravaHandlers(activities, { rateLimited: true })); - const provider = new StravaProvider(globalThis.fetch, 0); + const provider = new StravaProvider(globalThis.fetch); const result = await provider.sync( new SyncRun({ db: ctx.db, @@ -340,7 +340,7 @@ describe("StravaProvider.sync() (integration)", () => { const { oauthToken } = await import("../db/schema.ts"); await ctx.db.delete(oauthToken).where(eq(oauthToken.providerId, "strava")); - const provider = new StravaProvider(globalThis.fetch, 0); + const provider = new StravaProvider(globalThis.fetch); const result = await provider.sync( new SyncRun({ db: ctx.db, diff --git a/src/providers/strava.test.ts b/src/providers/strava.test.ts index 2a852656ae..832df7841a 100644 --- a/src/providers/strava.test.ts +++ b/src/providers/strava.test.ts @@ -564,7 +564,7 @@ describe("StravaClient.getActivity", () => { }); }; - const client = new StravaClient("token", mockFetch, 0); + const client = new StravaClient("token", mockFetch); const result = await client.getActivity(12345678); expect(result.device_name).toBe("Garmin Edge 530"); expect(result.id).toBe(12345678); @@ -575,7 +575,7 @@ describe("StravaClient.getActivity", () => { return Response.json(sampleActivity); }; - const client = new StravaClient("token", mockFetch, 0); + const client = new StravaClient("token", mockFetch); const result = await client.getActivity(12345678); expect(result.device_name).toBeUndefined(); }); @@ -587,7 +587,7 @@ describe("StravaClient — error handling", () => { return new Response("Rate Limit Exceeded", { status: 429 }); }; - const client = new StravaClient("token", mockFetch, 0); + const client = new StravaClient("token", mockFetch); const err = await client.getActivities(0).catch((e: unknown) => e); expect(err).toBeInstanceOf(StravaRateLimitError); expect(err).toBeInstanceOf(ProviderRateLimitError); @@ -605,7 +605,7 @@ describe("StravaClient — error handling", () => { }); }; - const client = new StravaClient("token", mockFetch, 0); + const client = new StravaClient("token", mockFetch); const err = await client.getActivities(0).catch((caughtError: unknown) => caughtError); expect(err).toBeInstanceOf(StravaRateLimitError); expect(err).toHaveProperty("retryAfterSeconds", 60); @@ -617,7 +617,7 @@ describe("StravaClient — error handling", () => { return new Response("Server Error", { status: 500 }); }; - const client = new StravaClient("token", mockFetch, 0); + const client = new StravaClient("token", mockFetch); await expect(client.getActivities(0)).rejects.toThrow("Strava API error (500): Server Error"); }); @@ -629,7 +629,7 @@ describe("StravaClient — error handling", () => { }); }; - const client = new StravaClient("token", mockFetch, 0); + const client = new StravaClient("token", mockFetch); const err = await client.getActivities(0).catch((e: unknown) => e); expect(err).toBeInstanceOf(StravaNotFoundError); expect(err).toHaveProperty("message", expect.stringContaining("/athlete/activities")); @@ -643,7 +643,7 @@ describe("StravaClient — error handling", () => { }); }; - const client = new StravaClient("token", mockFetch, 0); + const client = new StravaClient("token", mockFetch); const err = await client.getActivities(0).catch((e: unknown) => e); expect(err).toBeInstanceOf(StravaNotFoundError); expect(err).toHaveProperty("message", expect.stringContaining("/athlete/activities")); @@ -657,7 +657,7 @@ describe("StravaClient — error handling", () => { }); }; - const client = new StravaClient("token", mockFetch, 0); + const client = new StravaClient("token", mockFetch); const err = await client.getActivities(0).catch((e: unknown) => e); expect(err).toBeInstanceOf(StravaUnauthorizedError); expect(err).toHaveProperty("message", expect.stringContaining("unauthorized (401)")); @@ -668,7 +668,7 @@ describe("StravaClient — error handling", () => { return new Response("Forbidden", { status: 403 }); }; - const client = new StravaClient("token", mockFetch, 0); + const client = new StravaClient("token", mockFetch); const err = await client.getActivities(0).catch((e: unknown) => e); expect(err).toBeInstanceOf(StravaUnauthorizedError); expect(err).toHaveProperty("message", expect.stringContaining("unauthorized (403)")); @@ -682,7 +682,7 @@ describe("StravaClient — error handling", () => { }); }; - const client = new StravaClient("token", mockFetch, 0); + const client = new StravaClient("token", mockFetch); await expect(client.getActivities(0)).rejects.toThrow( 'Strava API error (500): {"message":"bad request"}', ); @@ -696,7 +696,7 @@ describe("StravaClient — error handling", () => { }); }; - const client = new StravaClient("token", mockFetch, 0); + const client = new StravaClient("token", mockFetch); await expect(client.getActivities(0)).rejects.toThrow( "Strava API error (500): (HTML error page)", ); @@ -708,7 +708,7 @@ describe("StravaClient — error handling", () => { return new Response(longText, { status: 500 }); }; - const client = new StravaClient("token", mockFetch, 0); + const client = new StravaClient("token", mockFetch); await expect(client.getActivities(0)).rejects.toThrow( `Strava API error (500): ${"x".repeat(200)}…`, ); @@ -716,41 +716,8 @@ describe("StravaClient — error handling", () => { }); describe("StravaClient — request throttling", () => { - beforeEach(() => { - vi.useFakeTimers(); - }); - - afterEach(() => { - vi.useRealTimers(); - }); - - it("enforces minimum delay between consecutive API requests", async () => { - const callTimestamps: number[] = []; - const mockFetch: typeof globalThis.fetch = async (): Promise<Response> => { - callTimestamps.push(Date.now()); - return Response.json([]); - }; - - const client = new StravaClient("token", mockFetch); - - // First request — should go immediately - const p1 = client.getActivities(0); - await vi.advanceTimersByTimeAsync(0); - await p1; - - // Second request — should be delayed by the throttle interval - const p2 = client.getActivities(0); - // Advance past the throttle delay - await vi.advanceTimersByTimeAsync(10_000); - await p2; - - expect(callTimestamps).toHaveLength(2); - const first = callTimestamps[0] ?? 0; - const second = callTimestamps[1] ?? 0; - expect(second - first).toBeGreaterThanOrEqual(10_000); - }); - it("does not delay the first request", async () => { + vi.useFakeTimers(); const callTimestamps: number[] = []; const mockFetch: typeof globalThis.fetch = async (): Promise<Response> => { callTimestamps.push(Date.now()); @@ -762,8 +729,8 @@ describe("StravaClient — request throttling", () => { await vi.advanceTimersByTimeAsync(0); await pendingRequest; - // First call should happen at time 0 (no throttle delay) expect(callTimestamps).toHaveLength(1); + vi.useRealTimers(); }); }); @@ -923,7 +890,7 @@ describe("StravaProvider.syncWebhookEvent", () => { }); it("returns immediately for non-activity objectType", async () => { - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const mockDb = { select: vi.fn(), insert: vi.fn(), @@ -949,7 +916,7 @@ describe("StravaProvider.syncWebhookEvent", () => { }); it("returns immediately when objectId is missing", async () => { - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const mockDb = { select: vi.fn(), insert: vi.fn(), @@ -972,7 +939,7 @@ describe("StravaProvider.syncWebhookEvent", () => { }); it("handles delete events by marking activity provider-absent", async () => { - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const mockDb = { select: vi.fn(), @@ -1004,7 +971,7 @@ describe("StravaProvider.syncWebhookEvent", () => { }); it("handles delete event when activity not found", async () => { - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const mockDb = { select: vi.fn(), @@ -1034,7 +1001,7 @@ describe("StravaProvider.syncWebhookEvent", () => { }); it("returns error when token resolution fails", async () => { - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const mockDb = { select: makeStravaSelectMock(null), insert: vi.fn(), @@ -1099,7 +1066,7 @@ describe("StravaProvider.syncWebhookEvent", () => { execute: vi.fn(), }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.syncWebhookEvent( mockDb, { @@ -1163,7 +1130,7 @@ describe("StravaProvider.syncWebhookEvent", () => { execute: vi.fn(), }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.syncWebhookEvent( mockDb, { @@ -1206,7 +1173,7 @@ describe("StravaProvider.syncWebhookEvent", () => { execute: vi.fn(), }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.syncWebhookEvent( mockDb, { @@ -1252,7 +1219,7 @@ describe("StravaProvider.syncWebhookEvent", () => { execute: vi.fn(), }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.syncWebhookEvent( mockDb, { @@ -1284,7 +1251,7 @@ describe("StravaProvider.registerWebhook", () => { it("throws when STRAVA_CLIENT_ID is missing", async () => { delete process.env.STRAVA_CLIENT_ID; delete process.env.STRAVA_CLIENT_SECRET; - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); await expect( provider.registerWebhook("https://example.com/webhook", "verify-token"), ).rejects.toThrow("STRAVA_CLIENT_ID"); @@ -1296,7 +1263,7 @@ describe("StravaProvider.registerWebhook", () => { const mockFetch: typeof globalThis.fetch = async (): Promise<Response> => { return new Response("Conflict", { status: 409 }); }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); await expect( provider.registerWebhook("https://example.com/webhook", "verify-token"), ).rejects.toThrow("Strava webhook registration failed (409)"); @@ -1308,7 +1275,7 @@ describe("StravaProvider.registerWebhook", () => { const mockFetch: typeof globalThis.fetch = async (): Promise<Response> => { return Response.json({ id: 42 }); }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.registerWebhook("https://example.com/webhook", "verify-token"); expect(result.subscriptionId).toBe("42"); }); @@ -1325,7 +1292,7 @@ describe("StravaProvider.unregisterWebhook", () => { delete process.env.STRAVA_CLIENT_ID; delete process.env.STRAVA_CLIENT_SECRET; const mockFetch = vi.fn(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); await provider.unregisterWebhook("42"); expect(mockFetch).not.toHaveBeenCalled(); }); @@ -1336,7 +1303,7 @@ describe("StravaProvider.unregisterWebhook", () => { const mockFetch: typeof globalThis.fetch = async (): Promise<Response> => { return new Response(null, { status: 200 }); }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); // Should not throw await provider.unregisterWebhook("42"); }); @@ -1347,7 +1314,7 @@ describe("StravaProvider.unregisterWebhook", () => { const mockFetch: typeof globalThis.fetch = async (): Promise<Response> => { return new Response("Not Found", { status: 404 }); }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); // Should not throw await provider.unregisterWebhook("42"); }); @@ -1358,7 +1325,7 @@ describe("StravaProvider.unregisterWebhook", () => { const mockFetch: typeof globalThis.fetch = async (): Promise<Response> => { return new Response("Server Error", { status: 500 }); }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); // Should not throw, just logs warning await provider.unregisterWebhook("42"); }); @@ -1370,7 +1337,7 @@ describe("StravaProvider.unregisterWebhook", () => { describe("StravaProvider — precise webhook string/object assertions", () => { it("parseWebhookPayload maps all three Strava aspect_types correctly", async () => { - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); for (const [aspect, expected] of [ ["create", "create"], @@ -1388,7 +1355,7 @@ describe("StravaProvider — precise webhook string/object assertions", () => { }); it("parseWebhookPayload converts owner_id number to string", async () => { - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const events = provider.parseWebhookPayload({ aspect_type: "create", object_type: "activity", @@ -1399,7 +1366,7 @@ describe("StravaProvider — precise webhook string/object assertions", () => { }); it("parseWebhookPayload converts object_id number to string", async () => { - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const events = provider.parseWebhookPayload({ aspect_type: "create", object_type: "activity", @@ -1410,7 +1377,7 @@ describe("StravaProvider — precise webhook string/object assertions", () => { }); it("parseWebhookPayload treats object_id=0 as falsy (undefined)", async () => { - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const events = provider.parseWebhookPayload({ aspect_type: "create", object_type: "activity", @@ -1421,7 +1388,7 @@ describe("StravaProvider — precise webhook string/object assertions", () => { }); it("handleValidationChallenge echoes back the exact challenge string", async () => { - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const result = provider.handleValidationChallenge( { "hub.mode": "subscribe", @@ -1434,7 +1401,7 @@ describe("StravaProvider — precise webhook string/object assertions", () => { }); it("handleValidationChallenge compares token exactly (not substring)", async () => { - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); // Partial match should fail const result = provider.handleValidationChallenge( { "hub.mode": "subscribe", "hub.challenge": "abc", "hub.verify_token": "tok" }, @@ -1454,7 +1421,7 @@ describe("StravaProvider — precise webhook string/object assertions", () => { return Response.json({ id: 1 }); }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); await provider.registerWebhook("https://example.com/callback", "my-verify-token"); expect(capturedBody?.get("client_id")).toBe("my-client-id"); @@ -1476,7 +1443,7 @@ describe("StravaProvider — precise webhook string/object assertions", () => { return Response.json({ id: 1 }); }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); await provider.registerWebhook("https://example.com/cb", "tok"); expect(capturedUrl).toBe("https://www.strava.com/api/v3/push_subscriptions"); @@ -1494,7 +1461,7 @@ describe("StravaProvider — precise webhook string/object assertions", () => { return Response.json({ id: 1 }); }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); await provider.registerWebhook("https://example.com/cb", "tok"); expect(capturedHeaders).toEqual( expect.objectContaining({ "Content-Type": "application/x-www-form-urlencoded" }), @@ -1514,7 +1481,7 @@ describe("StravaProvider — precise webhook string/object assertions", () => { return new Response(null, { status: 200 }); }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); await provider.unregisterWebhook("sub-42"); const parsed = new URL(capturedUrl); @@ -1526,7 +1493,7 @@ describe("StravaProvider — precise webhook string/object assertions", () => { }); it("syncWebhookEvent returns provider as 'strava' for all paths", async () => { - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const mockDb = { select: vi.fn(), insert: vi.fn(), @@ -1553,7 +1520,7 @@ describe("StravaProvider — precise webhook string/object assertions", () => { execute: vi.fn(), }; - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const result = await provider.syncWebhookEvent( mockDb, { @@ -1577,7 +1544,7 @@ describe("StravaProvider — precise webhook string/object assertions", () => { }); it("syncWebhookEvent falls back to token user context when options.userId is missing", async () => { - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const mockDb = { select: vi.fn(), insert: vi.fn(), delete: vi.fn(), execute: vi.fn() }; const result = await provider.syncWebhookEvent(mockDb, { @@ -1677,7 +1644,7 @@ describe("StravaClient.getActivityStreams", () => { return Response.json(apiResponse); }; - const client = new StravaClient("token", mockFetch, 0); + const client = new StravaClient("token", mockFetch); const streams = await client.getActivityStreams(12345); // Verify all 10 STREAM_KEYS are present in the result @@ -1722,7 +1689,7 @@ describe("StravaClient.getActivityStreams", () => { return Response.json(apiResponse); }; - const client = new StravaClient("token", mockFetch, 0); + const client = new StravaClient("token", mockFetch); const streams = await client.getActivityStreams(1); expect(streams.time).toBeDefined(); @@ -1739,7 +1706,7 @@ describe("StravaClient.getActivityStreams", () => { return Response.json([]); }; - const client = new StravaClient("token", mockFetch, 0); + const client = new StravaClient("token", mockFetch); await client.getActivityStreams(99999); // Verify base URL is the Strava API @@ -1778,7 +1745,7 @@ describe("StravaClient — API base URL", () => { return Response.json([]); }; - const client = new StravaClient("token", mockFetch, 0); + const client = new StravaClient("token", mockFetch); await client.getActivities(0); expect(capturedUrl).toMatch(/^https:\/\/www\.strava\.com\/api\/v3\//); @@ -1793,7 +1760,7 @@ describe("StravaClient — API base URL", () => { return Response.json(sampleActivity); }; - const client = new StravaClient("token", mockFetch, 0); + const client = new StravaClient("token", mockFetch); await client.getActivity(42); expect(capturedUrl).toBe("https://www.strava.com/api/v3/activities/42"); }); @@ -1808,7 +1775,7 @@ describe("StravaClient — API base URL", () => { return Response.json([]); }; - const client = new StravaClient("my-secret-token", mockFetch, 0); + const client = new StravaClient("my-secret-token", mockFetch); await client.getActivities(0); expect(capturedHeaders).toEqual({ Authorization: "Bearer my-secret-token" }); }); diff --git a/src/providers/webhook.test.ts b/src/providers/webhook.test.ts index b5a50bef89..9d3ef19779 100644 --- a/src/providers/webhook.test.ts +++ b/src/providers/webhook.test.ts @@ -5,14 +5,14 @@ import { isWebhookProvider } from "./types.ts"; describe("StravaProvider webhook methods", () => { it("is detected as a WebhookProvider", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); expect(isWebhookProvider(provider)).toBe(true); expect(provider.webhookScope).toBe("app"); }); it("parseWebhookPayload extracts activity create event", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const events = provider.parseWebhookPayload({ aspect_type: "create", @@ -34,7 +34,7 @@ describe("StravaProvider webhook methods", () => { it("parseWebhookPayload handles delete event", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const events = provider.parseWebhookPayload({ aspect_type: "delete", @@ -49,7 +49,7 @@ describe("StravaProvider webhook methods", () => { it("parseWebhookPayload returns empty for invalid payload", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); expect(provider.parseWebhookPayload({})).toHaveLength(0); expect(provider.parseWebhookPayload("bad")).toHaveLength(0); @@ -58,7 +58,7 @@ describe("StravaProvider webhook methods", () => { it("handleValidationChallenge responds to valid challenge", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const result = provider.handleValidationChallenge( { "hub.mode": "subscribe", "hub.challenge": "abc123", "hub.verify_token": "mytoken" }, @@ -69,7 +69,7 @@ describe("StravaProvider webhook methods", () => { it("handleValidationChallenge rejects wrong token", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const result = provider.handleValidationChallenge( { "hub.mode": "subscribe", "hub.challenge": "abc123", "hub.verify_token": "wrong" }, @@ -80,7 +80,7 @@ describe("StravaProvider webhook methods", () => { it("verifyWebhookSignature always returns true (Strava trusts registered URLs)", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); expect(provider.verifyWebhookSignature(Buffer.from(""), {}, "")).toBe(true); }); }); @@ -525,7 +525,7 @@ describe("StravaProvider register/unregister", () => { async () => new Response(JSON.stringify({ id: 12345 }), { status: 200 }), ); const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const original = { ...process.env }; process.env.STRAVA_CLIENT_ID = "test-client-id"; @@ -544,7 +544,7 @@ describe("StravaProvider register/unregister", () => { it("registerWebhook throws without client credentials", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const original = { ...process.env }; delete process.env.STRAVA_CLIENT_ID; @@ -561,7 +561,7 @@ describe("StravaProvider register/unregister", () => { it("registerWebhook throws on non-ok response", async () => { const mockFetch = vi.fn(async () => new Response("error", { status: 403 })); const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const original = { ...process.env }; process.env.STRAVA_CLIENT_ID = "id"; @@ -578,7 +578,7 @@ describe("StravaProvider register/unregister", () => { it("unregisterWebhook sends DELETE to Strava API", async () => { const mockFetch = vi.fn(async () => new Response(null, { status: 200 })); const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const original = { ...process.env }; process.env.STRAVA_CLIENT_ID = "id"; @@ -597,7 +597,7 @@ describe("StravaProvider register/unregister", () => { it("unregisterWebhook is a no-op without credentials", async () => { const mockFetch = vi.fn(async () => new Response()); const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const original = { ...process.env }; delete process.env.STRAVA_CLIENT_ID; @@ -612,7 +612,7 @@ describe("StravaProvider register/unregister", () => { it("parseWebhookPayload handles update event", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const events = provider.parseWebhookPayload({ aspect_type: "update", @@ -864,7 +864,7 @@ describe("CorosProvider register/unregister", () => { describe("StravaProvider webhook — precise assertions", () => { it("parseWebhookPayload returns exact event structure for create", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const events = provider.parseWebhookPayload({ aspect_type: "create", @@ -887,7 +887,7 @@ describe("StravaProvider webhook — precise assertions", () => { it("parseWebhookPayload defaults to update when aspect_type is unrecognized", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const events = provider.parseWebhookPayload({ aspect_type: "unknown_aspect", @@ -901,7 +901,7 @@ describe("StravaProvider webhook — precise assertions", () => { it("parseWebhookPayload defaults to update when aspect_type is missing", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const events = provider.parseWebhookPayload({ object_type: "activity", @@ -914,7 +914,7 @@ describe("StravaProvider webhook — precise assertions", () => { it("parseWebhookPayload returns objectId as undefined when object_id is missing", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const events = provider.parseWebhookPayload({ aspect_type: "create", @@ -928,7 +928,7 @@ describe("StravaProvider webhook — precise assertions", () => { it("parseWebhookPayload passes through object_type as-is", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const events = provider.parseWebhookPayload({ aspect_type: "create", @@ -942,7 +942,7 @@ describe("StravaProvider webhook — precise assertions", () => { it("handleValidationChallenge returns null when hub.mode is not subscribe", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const result = provider.handleValidationChallenge( { "hub.mode": "unsubscribe", "hub.challenge": "abc", "hub.verify_token": "mytoken" }, @@ -953,7 +953,7 @@ describe("StravaProvider webhook — precise assertions", () => { it("handleValidationChallenge returns null when hub.challenge is missing", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); const result = provider.handleValidationChallenge( { "hub.mode": "subscribe", "hub.verify_token": "mytoken" }, @@ -964,19 +964,19 @@ describe("StravaProvider webhook — precise assertions", () => { it("id is exactly 'strava'", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); expect(provider.id).toBe("strava"); }); it("name is exactly 'Strava'", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); expect(provider.name).toBe("Strava"); }); it("webhookScope is exactly 'app'", async () => { const { StravaProvider } = await import("./strava.ts"); - const provider = new StravaProvider(async () => new Response(), 0); + const provider = new StravaProvider(async () => new Response()); expect(provider.webhookScope).toBe("app"); }); }); From a3470cefe7514b0f78b0558c9fe0f077aac05002 Mon Sep 17 00:00:00 2001 From: Asher Cohen <asher.cohen@amplitude.com> Date: Sun, 21 Jun 2026 09:48:32 -0700 Subject: [PATCH 06/18] Fix vitest import order in cooldown test. Co-authored-by: Cursor <cursoragent@cursor.com> --- src/jobs/provider-rate-limit-cooldown.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jobs/provider-rate-limit-cooldown.test.ts b/src/jobs/provider-rate-limit-cooldown.test.ts index 912523759e..9c3227dee5 100644 --- a/src/jobs/provider-rate-limit-cooldown.test.ts +++ b/src/jobs/provider-rate-limit-cooldown.test.ts @@ -1,5 +1,5 @@ import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; -import { describe, afterEach, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { providerAdaptiveRateLimitStore } from "../lib/provider-adaptive-rate-limit.ts"; import { InMemoryProviderRateLimitCooldownStore, From 41008cc57764aebc84170184e3171bce3a07c773 Mon Sep 17 00:00:00 2001 From: Asher Cohen <asher.cohen@amplitude.com> Date: Sun, 21 Jun 2026 10:49:25 -0700 Subject: [PATCH 07/18] Expand adaptive rate-limit mutation coverage and fix Whoop daily sync. Centralize adaptive state serialization in provider-http, skip admission delays under Vitest, and only skip Whoop dates that already have step counts. Co-authored-by: Cursor <cursoragent@cursor.com> --- .../src/adaptive-rate-limit.test.ts | 329 +++++++++++++++++- .../provider-http/src/adaptive-rate-limit.ts | 81 +++++ src/jobs/provider-rate-limit-cooldown.ts | 2 +- src/lib/provider-adaptive-rate-limit.test.ts | 313 ++++++++++++++++- src/lib/provider-adaptive-rate-limit.ts | 208 +++++------ src/providers/ride-with-gps-ext.test.ts | 4 +- src/providers/strava.test.ts | 2 +- src/providers/whoop.test.ts | 5 +- src/providers/whoop/sync-daily-activity.ts | 17 +- vitest.config.mutation.ts | 2 +- 10 files changed, 801 insertions(+), 162 deletions(-) diff --git a/packages/provider-http/src/adaptive-rate-limit.test.ts b/packages/provider-http/src/adaptive-rate-limit.test.ts index 70ab705a2e..33d9f8ea91 100644 --- a/packages/provider-http/src/adaptive-rate-limit.test.ts +++ b/packages/provider-http/src/adaptive-rate-limit.test.ts @@ -1,17 +1,34 @@ import { describe, expect, it } from "vitest"; import { + ADAPTIVE_RATE_WINDOW_MS, + ADAPTIVE_THROTTLE_MAX_MS, + ADAPTIVE_THROTTLE_MIN_MS, admissionDelayMs, applyStravaQuota, blendObservedCooldown, createInitialAdaptiveState, decreaseThrottleMs, + defaultThrottleMs, + increaseThrottleMs, learnInferredBudget, parseStravaRateLimitHeaders, recordAdaptiveRateLimit, recordAdaptiveRequest, + serializeAdaptiveRateState, slideAdaptiveWindow, + adaptiveRateLimitStorageKey, + parseAdaptiveRateState, } from "./adaptive-rate-limit.ts"; +describe("defaultThrottleMs", () => { + it("returns provider-specific defaults", () => { + expect(defaultThrottleMs("strava")).toBe(10_000); + expect(defaultThrottleMs("garmin")).toBe(2_000); + expect(defaultThrottleMs("whoop")).toBe(1_000); + expect(defaultThrottleMs("unknown")).toBe(1_000); + }); +}); + describe("parseStravaRateLimitHeaders", () => { it("parses Strava short and daily quota headers", () => { const headers = new Headers({ @@ -29,22 +46,173 @@ describe("parseStravaRateLimitHeaders", () => { it("returns null when headers are missing", () => { expect(parseStravaRateLimitHeaders(new Headers())).toBeNull(); }); + + it("returns null when only one header is present", () => { + expect( + parseStravaRateLimitHeaders( + new Headers({ + "X-RateLimit-Limit": "100,1000", + }), + ), + ).toBeNull(); + }); + + it("returns null when header values are not numeric", () => { + expect( + parseStravaRateLimitHeaders( + new Headers({ + "X-RateLimit-Limit": "bad,1000", + "X-RateLimit-Usage": "1,2", + }), + ), + ).toBeNull(); + }); }); -describe("adaptive rate-limit learning", () => { +describe("blendObservedCooldown", () => { it("blends observed cooldown values toward recent observations", () => { expect(blendObservedCooldown(600, 300)).toBe(510); expect(blendObservedCooldown(null, 300)).toBe(300); }); + it("returns previous value when observed seconds are non-positive", () => { + expect(blendObservedCooldown(600, 0)).toBe(600); + expect(blendObservedCooldown(null, -5)).toBe(-5); + }); +}); + +describe("slideAdaptiveWindow", () => { it("slides the rolling window without recomputing historical requests", () => { const state = createInitialAdaptiveState("whoop", "provider", null, 0); const withRequests = { ...state, requestCount: 12, windowStartMs: 0 }; - const slid = slideAdaptiveWindow(withRequests, 5 * 60 * 1000 + 1); + const slid = slideAdaptiveWindow(withRequests, ADAPTIVE_RATE_WINDOW_MS + 1); expect(slid.requestCount).toBe(0); - expect(slid.windowStartMs).toBe(5 * 60 * 1000 + 1); + expect(slid.windowStartMs).toBe(ADAPTIVE_RATE_WINDOW_MS + 1); + }); + + it("keeps state when the rolling window has not elapsed", () => { + const state = createInitialAdaptiveState("whoop", "provider", null, 1000); + const withRequests = { ...state, requestCount: 4 }; + expect(slideAdaptiveWindow(withRequests, 2000)).toBe(withRequests); + }); +}); + +describe("throttle adjustment helpers", () => { + it("decreases throttle gradually on success without dropping below the minimum", () => { + expect(decreaseThrottleMs(1000)).toBe(900); + expect(decreaseThrottleMs(ADAPTIVE_THROTTLE_MIN_MS)).toBe(ADAPTIVE_THROTTLE_MIN_MS); + }); + + it("increases throttle on rate limits without exceeding the maximum", () => { + expect(increaseThrottleMs(1000)).toBe(2000); + expect(increaseThrottleMs(ADAPTIVE_THROTTLE_MAX_MS)).toBe(ADAPTIVE_THROTTLE_MAX_MS); + }); +}); + +describe("learnInferredBudget", () => { + it("learns a lower inferred budget from observed failures", () => { + expect(learnInferredBudget(30, 25)).toBe(24); + expect(learnInferredBudget(null, 10)).toBe(9); + }); + + it("returns the current budget when no requests were observed at the limit", () => { + expect(learnInferredBudget(30, 0)).toBe(30); + expect(learnInferredBudget(null, 0)).toBeNull(); + }); +}); + +describe("admissionDelayMs", () => { + it("waits for the remaining throttle interval after the last request", () => { + const state = { + ...createInitialAdaptiveState("whoop", "provider", null, 0), + throttleMs: 1000, + lastRequestMs: 100, + }; + expect(admissionDelayMs(state, 500)).toBe(600); + }); + + it("adds delay when the inferred budget soft cap is reached", () => { + const state = { + ...createInitialAdaptiveState("garmin", "provider", null, 0), + throttleMs: 1000, + inferredBudget: 10, + requestCount: 8, + }; + expect(admissionDelayMs(state, 0)).toBe(1000); + }); + + it("paces Strava requests when short quota is nearly exhausted", () => { + const state = applyStravaQuota(createInitialAdaptiveState("strava", "provider", null), { + shortLimit: 100, + shortUsage: 99, + dailyLimit: 1000, + dailyUsage: 100, + }); + expect(admissionDelayMs(state, Date.now())).toBeGreaterThanOrEqual(state.throttleMs * 4); + }); + + it("doubles Strava pacing delay when only a few short-quota requests remain", () => { + const state = applyStravaQuota(createInitialAdaptiveState("strava", "provider", null), { + shortLimit: 100, + shortUsage: 97, + dailyLimit: 1000, + dailyUsage: 100, + }); + expect(admissionDelayMs(state, 0)).toBe(state.throttleMs * 2); + }); + + it("paces Strava requests proportionally when quota headroom remains", () => { + const state = applyStravaQuota(createInitialAdaptiveState("strava", "provider", null), { + shortLimit: 100, + shortUsage: 50, + dailyLimit: 1000, + dailyUsage: 100, + }); + expect(admissionDelayMs(state, 0)).toBe(18_000); + }); + + it("ignores Strava pacing for non-Strava providers", () => { + const state = { + ...createInitialAdaptiveState("garmin", "provider", null), + stravaShortLimit: 100, + stravaShortUsage: 99, + }; + expect(admissionDelayMs(state, 0)).toBe(0); + }); +}); + +describe("createInitialAdaptiveState", () => { + it("creates user-scoped state with the provided user id", () => { + const state = createInitialAdaptiveState("whoop", "user", "user-1", 1234); + expect(state).toMatchObject({ + providerId: "whoop", + scope: "user", + userId: "user-1", + windowStartMs: 1234, + requestCount: 0, + throttleMs: 1000, + }); + }); +}); + +describe("applyStravaQuota", () => { + it("stores parsed Strava quota fields on the adaptive state", () => { + const state = applyStravaQuota(createInitialAdaptiveState("strava", "provider", null), { + shortLimit: 100, + shortUsage: 10, + dailyLimit: 1000, + dailyUsage: 20, + }); + expect(state).toMatchObject({ + stravaShortLimit: 100, + stravaShortUsage: 10, + stravaDailyLimit: 1000, + stravaDailyUsage: 20, + }); }); +}); +describe("recordAdaptiveRequest", () => { it("records requests by incrementing the rolling tally", () => { const state = createInitialAdaptiveState("garmin", "provider", null, 1000); const next = recordAdaptiveRequest(state, 1500); @@ -52,7 +220,9 @@ describe("adaptive rate-limit learning", () => { expect(next.lastRequestMs).toBe(1500); expect(next.throttleMs).toBeLessThan(state.throttleMs); }); +}); +describe("recordAdaptiveRateLimit", () => { it("learns inferred budget and increases throttle on rate limits", () => { const state = { ...createInitialAdaptiveState("whoop", "provider", null), @@ -66,22 +236,149 @@ describe("adaptive rate-limit learning", () => { expect(limited.observedCooldownSeconds).toBe(120); }); - it("paces Strava requests when short quota is nearly exhausted", () => { - const state = applyStravaQuota(createInitialAdaptiveState("strava", "provider", null), { - shortLimit: 100, - shortUsage: 99, - dailyLimit: 1000, - dailyUsage: 100, - }); - expect(admissionDelayMs(state, Date.now())).toBeGreaterThanOrEqual(state.throttleMs * 4); + it("preserves observed cooldown when retry-after is absent or zero", () => { + const state = { + ...createInitialAdaptiveState("whoop", "provider", null), + observedCooldownSeconds: 240, + requestCount: 5, + }; + expect(recordAdaptiveRateLimit(state, null).observedCooldownSeconds).toBe(240); + expect(recordAdaptiveRateLimit(state, 0).observedCooldownSeconds).toBe(240); }); +}); - it("learns a lower inferred budget from observed failures", () => { - expect(learnInferredBudget(30, 25)).toBe(24); - expect(learnInferredBudget(null, 10)).toBe(9); +describe("adaptiveRateLimitStorageKey", () => { + it("builds provider and user scoped keys", () => { + expect(adaptiveRateLimitStorageKey("garmin", "provider", null)).toBe( + "provider-adaptive-rate:garmin:provider", + ); + expect(adaptiveRateLimitStorageKey("whoop", "user", "user-1")).toBe( + "provider-adaptive-rate:whoop:user:user-1", + ); + expect(adaptiveRateLimitStorageKey("whoop", "user", null)).toBe( + "provider-adaptive-rate:whoop:user:unknown", + ); }); +}); - it("decreases throttle gradually on success", () => { - expect(decreaseThrottleMs(1000)).toBe(900); +describe("serializeAdaptiveRateState", () => { + it("serializes adaptive state as JSON", () => { + const state = createInitialAdaptiveState("strava", "provider", null); + expect(JSON.parse(serializeAdaptiveRateState(state))).toEqual(state); + }); +}); + +describe("parseAdaptiveRateState", () => { + it("returns null for empty input", () => { + expect(parseAdaptiveRateState(null)).toBeNull(); + expect(parseAdaptiveRateState("")).toBeNull(); + }); + + it("parses valid persisted state with optional Strava fields", () => { + const state = { + ...createInitialAdaptiveState("strava", "provider", null), + inferredBudget: 35, + observedCooldownSeconds: 90, + stravaShortLimit: 100, + stravaShortUsage: 80, + stravaDailyLimit: 1000, + stravaDailyUsage: 400, + }; + const parsed = parseAdaptiveRateState(serializeAdaptiveRateState(state)); + expect(parsed).toEqual(state); + }); + + it("rejects invalid scope and numeric fields", () => { + expect( + parseAdaptiveRateState( + JSON.stringify({ + providerId: "garmin", + scope: "invalid", + userId: null, + windowStartMs: 1, + requestCount: 0, + throttleMs: 1, + lastRequestMs: null, + }), + ), + ).toBeNull(); + expect( + parseAdaptiveRateState( + JSON.stringify({ + providerId: 1, + scope: "provider", + userId: null, + windowStartMs: 1, + requestCount: 0, + throttleMs: 1, + lastRequestMs: null, + }), + ), + ).toBeNull(); + expect( + parseAdaptiveRateState( + JSON.stringify({ + providerId: "garmin", + scope: "provider", + userId: null, + windowStartMs: "bad", + requestCount: 0, + throttleMs: 1, + lastRequestMs: null, + }), + ), + ).toBeNull(); + }); + + it("rejects invalid userId and lastRequestMs", () => { + expect( + parseAdaptiveRateState( + JSON.stringify({ + providerId: "whoop", + scope: "user", + userId: 42, + windowStartMs: 1, + requestCount: 0, + throttleMs: 1, + lastRequestMs: null, + }), + ), + ).toBeNull(); + expect( + parseAdaptiveRateState( + JSON.stringify({ + providerId: "garmin", + scope: "provider", + userId: null, + windowStartMs: 1, + requestCount: 0, + throttleMs: 1, + lastRequestMs: "now", + }), + ), + ).toBeNull(); + }); + + it("drops non-finite optional numeric fields", () => { + const parsed = parseAdaptiveRateState( + JSON.stringify({ + providerId: "garmin", + scope: "provider", + userId: null, + windowStartMs: 1, + requestCount: 0, + throttleMs: 1, + lastRequestMs: null, + inferredBudget: "lots", + observedCooldownSeconds: "slow", + stravaShortLimit: "x", + stravaShortUsage: "y", + stravaDailyLimit: "z", + stravaDailyUsage: "w", + }), + ); + expect(parsed?.inferredBudget).toBeNull(); + expect(parsed?.observedCooldownSeconds).toBeNull(); + expect(parsed?.stravaShortLimit).toBeNull(); }); }); diff --git a/packages/provider-http/src/adaptive-rate-limit.ts b/packages/provider-http/src/adaptive-rate-limit.ts index be12cd8386..d18ac75027 100644 --- a/packages/provider-http/src/adaptive-rate-limit.ts +++ b/packages/provider-http/src/adaptive-rate-limit.ts @@ -205,3 +205,84 @@ export function recordAdaptiveRateLimit( observedCooldownSeconds: observedCooldown, }; } + +export const ADAPTIVE_RATE_STORAGE_KEY_PREFIX = "provider-adaptive-rate"; + +export function adaptiveRateLimitStorageKey( + providerId: string, + scope: ProviderRateLimitScope, + userId: string | null, +): string { + return scope === "provider" + ? `${ADAPTIVE_RATE_STORAGE_KEY_PREFIX}:${providerId}:provider` + : `${ADAPTIVE_RATE_STORAGE_KEY_PREFIX}:${providerId}:user:${userId ?? "unknown"}`; +} + +export function serializeAdaptiveRateState(state: ProviderAdaptiveRateState): string { + return JSON.stringify(state); +} + +export function parseAdaptiveRateState(raw: string | null): ProviderAdaptiveRateState | null { + if (!raw) return null; + const parsed: unknown = JSON.parse(raw); + if (typeof parsed !== "object" || parsed === null) return null; + + const providerId = Reflect.get(parsed, "providerId"); + const scope = Reflect.get(parsed, "scope"); + const userId = Reflect.get(parsed, "userId"); + const windowStartMs = Reflect.get(parsed, "windowStartMs"); + const requestCount = Reflect.get(parsed, "requestCount"); + const throttleMs = Reflect.get(parsed, "throttleMs"); + const lastRequestMs = Reflect.get(parsed, "lastRequestMs"); + const inferredBudget = Reflect.get(parsed, "inferredBudget"); + const observedCooldownSeconds = Reflect.get(parsed, "observedCooldownSeconds"); + const stravaShortLimit = Reflect.get(parsed, "stravaShortLimit"); + const stravaShortUsage = Reflect.get(parsed, "stravaShortUsage"); + const stravaDailyLimit = Reflect.get(parsed, "stravaDailyLimit"); + const stravaDailyUsage = Reflect.get(parsed, "stravaDailyUsage"); + + if (typeof providerId !== "string") return null; + if (scope !== "provider" && scope !== "user") return null; + if (userId !== null && typeof userId !== "string") return null; + if (typeof windowStartMs !== "number" || !Number.isFinite(windowStartMs)) return null; + if (typeof requestCount !== "number" || !Number.isFinite(requestCount)) return null; + if (typeof throttleMs !== "number" || !Number.isFinite(throttleMs)) return null; + if ( + lastRequestMs !== null && + (typeof lastRequestMs !== "number" || !Number.isFinite(lastRequestMs)) + ) { + return null; + } + + return { + providerId, + scope, + userId, + windowStartMs, + requestCount, + throttleMs, + lastRequestMs, + inferredBudget: + typeof inferredBudget === "number" && Number.isFinite(inferredBudget) ? inferredBudget : null, + observedCooldownSeconds: + typeof observedCooldownSeconds === "number" && Number.isFinite(observedCooldownSeconds) + ? observedCooldownSeconds + : null, + stravaShortLimit: + typeof stravaShortLimit === "number" && Number.isFinite(stravaShortLimit) + ? stravaShortLimit + : null, + stravaShortUsage: + typeof stravaShortUsage === "number" && Number.isFinite(stravaShortUsage) + ? stravaShortUsage + : null, + stravaDailyLimit: + typeof stravaDailyLimit === "number" && Number.isFinite(stravaDailyLimit) + ? stravaDailyLimit + : null, + stravaDailyUsage: + typeof stravaDailyUsage === "number" && Number.isFinite(stravaDailyUsage) + ? stravaDailyUsage + : null, + }; +} diff --git a/src/jobs/provider-rate-limit-cooldown.ts b/src/jobs/provider-rate-limit-cooldown.ts index 95a82992ae..996bb84f42 100644 --- a/src/jobs/provider-rate-limit-cooldown.ts +++ b/src/jobs/provider-rate-limit-cooldown.ts @@ -261,7 +261,7 @@ export class RedisProviderRateLimitCooldownStore implements ProviderRateLimitCoo } export const providerRateLimitCooldownStore: ProviderRateLimitCooldownStore = - process.env.NODE_ENV === "test" + process.env.NODE_ENV === "test" || process.env.VITEST === "true" ? new InMemoryProviderRateLimitCooldownStore() : new RedisProviderRateLimitCooldownStore(); diff --git a/src/lib/provider-adaptive-rate-limit.test.ts b/src/lib/provider-adaptive-rate-limit.test.ts index 55342a48ef..0e77fdaa45 100644 --- a/src/lib/provider-adaptive-rate-limit.test.ts +++ b/src/lib/provider-adaptive-rate-limit.test.ts @@ -1,27 +1,96 @@ +import { ADAPTIVE_RATE_WINDOW_MS } from "@dofek/provider-http/adaptive-rate-limit"; import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; -import { describe, expect, it } from "vitest"; -import { InMemoryAdaptiveRateLimitStore } from "./provider-adaptive-rate-limit.ts"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + InMemoryAdaptiveRateLimitStore, + providerAdaptiveRateLimitStore, + RedisAdaptiveRateLimitStore, +} from "./provider-adaptive-rate-limit.ts"; + +function rateLimitError(options: { + providerId: string; + scope?: "provider" | "user"; + userId?: string | null; + retryAfterSeconds?: number | null; +}) { + return new ProviderRateLimitError({ + message: "rate limited", + providerId: options.providerId, + statusCode: 429, + responseBody: "limited", + scope: options.scope, + userId: options.userId, + retryAfterSeconds: options.retryAfterSeconds, + }); +} + +function createMockRedisAdaptiveStore() { + const values = new Map<string, string>(); + const setCalls: Array<{ + key: string; + value: string; + mode: "PX"; + millisecondsToExpire: number; + }> = []; + const getRedisClient: ConstructorParameters<typeof RedisAdaptiveRateLimitStore>[0] = async () => ({ + set: async (key, value, mode, millisecondsToExpire) => { + setCalls.push({ key, value, mode, millisecondsToExpire }); + values.set(key, value); + return "OK"; + }, + get: async (key) => values.get(key) ?? null, + }); + + return { + values, + setCalls, + store: new RedisAdaptiveRateLimitStore(getRedisClient), + }; +} describe("InMemoryAdaptiveRateLimitStore", () => { - it("tracks rolling request counts in Redis-shaped state without recomputing history", async () => { + afterEach(() => { + vi.unstubAllEnvs(); + vi.useRealTimers(); + }); + + it("tracks rolling request counts and learns cooldown from rate limits", async () => { const store = new InMemoryAdaptiveRateLimitStore(); await store.awaitAdmission("whoop", "provider", null); await store.awaitAdmission("whoop", "provider", null); - const error = new ProviderRateLimitError({ - message: "whoop API rate limit exceeded (429):", - providerId: "whoop", - statusCode: 429, - responseBody: "", - retryAfterSeconds: 300, - }); - await store.recordRateLimit(error); + await store.recordRateLimit(rateLimitError({ providerId: "whoop", retryAfterSeconds: 300 })); expect(await store.getLearnedCooldownSeconds("whoop")).toBe(300); }); - it("learns Strava quota headers from successful responses", async () => { + it("records user-scoped rate limits separately from provider scope", async () => { + const store = new InMemoryAdaptiveRateLimitStore(); + + await store.recordRateLimit( + rateLimitError({ + providerId: "fitbit", + scope: "user", + userId: "user-1", + retryAfterSeconds: 90, + }), + ); + + expect(await store.getLearnedCooldownSeconds("fitbit")).toBeNull(); + await store.awaitAdmission("fitbit", "user", "user-1"); + await store.recordRateLimit( + rateLimitError({ + providerId: "fitbit", + scope: "user", + userId: "user-1", + retryAfterSeconds: 120, + }), + ); + await store.awaitAdmission("fitbit", "provider", null); + }); + + it("applies Strava quota headers on successful responses", async () => { const store = new InMemoryAdaptiveRateLimitStore(); const headers = new Headers({ "X-RateLimit-Limit": "100,1000", @@ -29,11 +98,223 @@ describe("InMemoryAdaptiveRateLimitStore", () => { }); await store.recordSuccess("strava", "provider", null, headers); + await store.awaitAdmission("strava", "provider", null); + }); + + it("ignores response headers for non-Strava providers", async () => { + const store = new InMemoryAdaptiveRateLimitStore(); + const headers = new Headers({ + "X-RateLimit-Limit": "100,1000", + "X-RateLimit-Usage": "95,400", + }); + + await store.recordSuccess("garmin", "provider", null, headers); + await store.awaitAdmission("garmin", "provider", null); + }); + + it("records success without response headers", async () => { + const store = new InMemoryAdaptiveRateLimitStore(); + await store.recordSuccess("whoop", "provider", null); + await store.awaitAdmission("whoop", "provider", null); + }); +}); + +describe("RedisAdaptiveRateLimitStore", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("persists state in Redis with the adaptive key prefix", async () => { + const { store, setCalls } = createMockRedisAdaptiveStore(); + + await store.awaitAdmission("garmin", "provider", null); + + expect(setCalls.length).toBeGreaterThan(0); + expect(setCalls[0]?.key).toBe("provider-adaptive-rate:garmin:provider"); + expect(setCalls[0]?.mode).toBe("PX"); + expect(setCalls[0]?.millisecondsToExpire).toBe(ADAPTIVE_RATE_WINDOW_MS * 4); + expect(setCalls[0]?.value).toContain('"providerId":"garmin"'); + }); + + it("uses user-scoped keys when scope is user", async () => { + const { store, setCalls } = createMockRedisAdaptiveStore(); + + await store.awaitAdmission("whoop", "user", "user-42"); + + expect(setCalls[0]?.key).toBe("provider-adaptive-rate:whoop:user:user-42"); + }); + + it("loads persisted state on subsequent calls", async () => { + const { store, values } = createMockRedisAdaptiveStore(); + + await store.recordRateLimit(rateLimitError({ providerId: "whoop", retryAfterSeconds: 180 })); + expect(await store.getLearnedCooldownSeconds("whoop")).toBe(180); + + const redisStore = new RedisAdaptiveRateLimitStore(async () => ({ + set: async (key, value, mode, ms) => { + values.set(key, value); + return "OK"; + }, + get: async (key) => values.get(key) ?? null, + })); + expect(await redisStore.getLearnedCooldownSeconds("whoop")).toBe(180); + }); + + it("creates fresh state when Redis payload is missing required fields", async () => { + const { store, values } = createMockRedisAdaptiveStore(); + values.set( + "provider-adaptive-rate:garmin:provider", + JSON.stringify({ providerId: "garmin", scope: "invalid" }), + ); + + await store.awaitAdmission("garmin", "provider", null); + expect(values.get("provider-adaptive-rate:garmin:provider")).toContain('"scope":"provider"'); + }); + + it("creates fresh state when Redis payload has invalid numeric fields", async () => { + const { store, values } = createMockRedisAdaptiveStore(); + values.set( + "provider-adaptive-rate:garmin:provider", + JSON.stringify({ + providerId: "garmin", + scope: "provider", + userId: null, + windowStartMs: "bad", + requestCount: 0, + throttleMs: 1000, + lastRequestMs: null, + }), + ); + + await store.awaitAdmission("garmin", "provider", null); + expect(values.get("provider-adaptive-rate:garmin:provider")).toContain('"windowStartMs":'); + }); + + it("records Strava quota from Redis-backed success responses", async () => { + const { store } = createMockRedisAdaptiveStore(); + const headers = new Headers({ + "X-RateLimit-Limit": "100,1000", + "X-RateLimit-Usage": "99,900", + }); + + await store.recordSuccess("strava", "provider", null, headers); + await store.awaitAdmission("strava", "provider", null); + }); + + it("records user-scoped rate limits through Redis", async () => { + const { store, setCalls } = createMockRedisAdaptiveStore(); + + await store.recordRateLimit( + rateLimitError({ + providerId: "fitbit", + scope: "user", + userId: "user-9", + retryAfterSeconds: 45, + }), + ); + + expect(setCalls.some((call) => call.key === "provider-adaptive-rate:fitbit:user:user-9")).toBe( + true, + ); + }); + + it("round-trips optional Strava quota fields through Redis", async () => { + const { store, values } = createMockRedisAdaptiveStore(); + values.set( + "provider-adaptive-rate:strava:provider", + JSON.stringify({ + providerId: "strava", + scope: "provider", + userId: null, + windowStartMs: 1_000, + requestCount: 2, + throttleMs: 10_000, + lastRequestMs: 900, + inferredBudget: 35, + observedCooldownSeconds: 120, + stravaShortLimit: 100, + stravaShortUsage: 90, + stravaDailyLimit: 1000, + stravaDailyUsage: 400, + }), + ); - // Near quota exhaustion should cause a longer admission delay on the next call. - const start = Date.now(); await store.awaitAdmission("strava", "provider", null); - const elapsed = Date.now() - start; - expect(elapsed).toBeGreaterThanOrEqual(0); + const saved = JSON.parse(values.get("provider-adaptive-rate:strava:provider") ?? "{}"); + expect(saved.stravaShortUsage).toBe(90); + expect(saved.inferredBudget).toBe(35); + }); + + it("rejects persisted state with invalid userId", async () => { + const { store, values } = createMockRedisAdaptiveStore(); + values.set( + "provider-adaptive-rate:whoop:user:user-1", + JSON.stringify({ + providerId: "whoop", + scope: "user", + userId: 42, + windowStartMs: 1_000, + requestCount: 0, + throttleMs: 1_000, + lastRequestMs: null, + }), + ); + + await store.awaitAdmission("whoop", "user", "user-1"); + expect(values.get("provider-adaptive-rate:whoop:user:user-1")).toContain('"userId":"user-1"'); + }); + + it("rejects persisted state with invalid lastRequestMs", async () => { + const { store, values } = createMockRedisAdaptiveStore(); + values.set( + "provider-adaptive-rate:garmin:provider", + JSON.stringify({ + providerId: "garmin", + scope: "provider", + userId: null, + windowStartMs: 1_000, + requestCount: 0, + throttleMs: 1_000, + lastRequestMs: "now", + }), + ); + + await store.awaitAdmission("garmin", "provider", null); + const saved = JSON.parse(values.get("provider-adaptive-rate:garmin:provider") ?? "{}"); + expect(saved.lastRequestMs).not.toBe("now"); + expect(typeof saved.lastRequestMs).toBe("number"); + }); + + it("rejects persisted state with non-finite optional numeric fields", async () => { + const { store, values } = createMockRedisAdaptiveStore(); + values.set( + "provider-adaptive-rate:garmin:provider", + JSON.stringify({ + providerId: "garmin", + scope: "provider", + userId: null, + windowStartMs: 1_000, + requestCount: 0, + throttleMs: 1_000, + lastRequestMs: null, + inferredBudget: "lots", + observedCooldownSeconds: "slow", + stravaShortLimit: "x", + stravaShortUsage: "y", + stravaDailyLimit: "z", + stravaDailyUsage: "w", + }), + ); + + await store.awaitAdmission("garmin", "provider", null); + const saved = JSON.parse(values.get("provider-adaptive-rate:garmin:provider") ?? "{}"); + expect(saved.inferredBudget).toBeNull(); + expect(saved.observedCooldownSeconds).toBeNull(); + }); +}); + +describe("providerAdaptiveRateLimitStore", () => { + it("uses the in-memory store under vitest", () => { + expect(providerAdaptiveRateLimitStore).toBeInstanceOf(InMemoryAdaptiveRateLimitStore); }); }); diff --git a/src/lib/provider-adaptive-rate-limit.ts b/src/lib/provider-adaptive-rate-limit.ts index dc5ca9cbf4..d77d4ae1b4 100644 --- a/src/lib/provider-adaptive-rate-limit.ts +++ b/src/lib/provider-adaptive-rate-limit.ts @@ -1,13 +1,16 @@ import { ADAPTIVE_RATE_WINDOW_MS, type AdaptiveRateLimitStore, + adaptiveRateLimitStorageKey, admissionDelayMs, applyStravaQuota, createInitialAdaptiveState, + parseAdaptiveRateState, type ProviderAdaptiveRateState, parseStravaRateLimitHeaders, recordAdaptiveRateLimit, recordAdaptiveRequest, + serializeAdaptiveRateState, slideAdaptiveWindow, } from "@dofek/provider-http/adaptive-rate-limit"; import type { @@ -22,90 +25,70 @@ interface RedisClient { get: (key: string) => Promise<string | null>; } -const KEY_PREFIX = "provider-adaptive-rate"; - -function adaptiveKey( +type LoadOrCreate = ( providerId: string, scope: ProviderRateLimitScope, userId: string | null, -): string { - return scope === "provider" - ? `${KEY_PREFIX}:${providerId}:provider` - : `${KEY_PREFIX}:${providerId}:user:${userId ?? "unknown"}`; +) => Promise<ProviderAdaptiveRateState>; + +type SaveState = (state: ProviderAdaptiveRateState) => Promise<void>; + +function shouldSkipAdmissionDelay(): boolean { + return process.env.NODE_ENV === "test" || process.env.VITEST === "true"; } -function serializeState(state: ProviderAdaptiveRateState): string { - return JSON.stringify(state); +function sleep(ms: number): Promise<void> { + if (ms <= 0 || shouldSkipAdmissionDelay()) return Promise.resolve(); + return new Promise((resolve) => setTimeout(resolve, ms)); } -function parseState(raw: string | null): ProviderAdaptiveRateState | null { - if (!raw) return null; - const parsed: unknown = JSON.parse(raw); - if (typeof parsed !== "object" || parsed === null) return null; - - const providerId = Reflect.get(parsed, "providerId"); - const scope = Reflect.get(parsed, "scope"); - const userId = Reflect.get(parsed, "userId"); - const windowStartMs = Reflect.get(parsed, "windowStartMs"); - const requestCount = Reflect.get(parsed, "requestCount"); - const throttleMs = Reflect.get(parsed, "throttleMs"); - const lastRequestMs = Reflect.get(parsed, "lastRequestMs"); - const inferredBudget = Reflect.get(parsed, "inferredBudget"); - const observedCooldownSeconds = Reflect.get(parsed, "observedCooldownSeconds"); - const stravaShortLimit = Reflect.get(parsed, "stravaShortLimit"); - const stravaShortUsage = Reflect.get(parsed, "stravaShortUsage"); - const stravaDailyLimit = Reflect.get(parsed, "stravaDailyLimit"); - const stravaDailyUsage = Reflect.get(parsed, "stravaDailyUsage"); - - if (typeof providerId !== "string") return null; - if (scope !== "provider" && scope !== "user") return null; - if (userId !== null && typeof userId !== "string") return null; - if (typeof windowStartMs !== "number" || !Number.isFinite(windowStartMs)) return null; - if (typeof requestCount !== "number" || !Number.isFinite(requestCount)) return null; - if (typeof throttleMs !== "number" || !Number.isFinite(throttleMs)) return null; - if ( - lastRequestMs !== null && - (typeof lastRequestMs !== "number" || !Number.isFinite(lastRequestMs)) - ) { - return null; +async function awaitAdmissionWithStore( + loadOrCreate: LoadOrCreate, + save: SaveState, + providerId: string, + scope: ProviderRateLimitScope, + userId: string | null, +): Promise<void> { + const nowMs = Date.now(); + const state = slideAdaptiveWindow(await loadOrCreate(providerId, scope, userId), nowMs); + await sleep(admissionDelayMs(state, nowMs)); + await save(recordAdaptiveRequest(state, nowMs)); +} + +async function recordSuccessWithStore( + loadOrCreate: LoadOrCreate, + save: SaveState, + providerId: string, + scope: ProviderRateLimitScope, + userId: string | null, + responseHeaders?: Headers, +): Promise<void> { + const state = await loadOrCreate(providerId, scope, userId); + let next = slideAdaptiveWindow(state, Date.now()); + if (providerId === "strava" && responseHeaders) { + const quota = parseStravaRateLimitHeaders(responseHeaders); + if (quota) next = applyStravaQuota(next, quota); } + await save(next); +} - return { - providerId, - scope, - userId, - windowStartMs, - requestCount, - throttleMs, - lastRequestMs, - inferredBudget: - typeof inferredBudget === "number" && Number.isFinite(inferredBudget) ? inferredBudget : null, - observedCooldownSeconds: - typeof observedCooldownSeconds === "number" && Number.isFinite(observedCooldownSeconds) - ? observedCooldownSeconds - : null, - stravaShortLimit: - typeof stravaShortLimit === "number" && Number.isFinite(stravaShortLimit) - ? stravaShortLimit - : null, - stravaShortUsage: - typeof stravaShortUsage === "number" && Number.isFinite(stravaShortUsage) - ? stravaShortUsage - : null, - stravaDailyLimit: - typeof stravaDailyLimit === "number" && Number.isFinite(stravaDailyLimit) - ? stravaDailyLimit - : null, - stravaDailyUsage: - typeof stravaDailyUsage === "number" && Number.isFinite(stravaDailyUsage) - ? stravaDailyUsage - : null, - }; +async function recordRateLimitWithStore( + loadOrCreate: LoadOrCreate, + save: SaveState, + error: ProviderRateLimitError, +): Promise<void> { + const scope = error.scope; + const userId = scope === "user" ? error.userId : null; + const state = await loadOrCreate(error.providerId, scope, userId); + await save(recordAdaptiveRateLimit(state, error.retryAfterSeconds)); } -function sleep(ms: number): Promise<void> { - if (ms <= 0) return Promise.resolve(); - return new Promise((resolve) => setTimeout(resolve, ms)); +async function getLearnedCooldownWithStore( + loadOrCreate: LoadOrCreate, + providerId: string, +): Promise<number | null> { + const state = await loadOrCreate(providerId, "provider", null); + return state.observedCooldownSeconds; } export class InMemoryAdaptiveRateLimitStore implements AdaptiveRateLimitStore { @@ -116,7 +99,7 @@ export class InMemoryAdaptiveRateLimitStore implements AdaptiveRateLimitStore { scope: ProviderRateLimitScope, userId: string | null, ): Promise<ProviderAdaptiveRateState> { - const key = adaptiveKey(providerId, scope, userId); + const key = adaptiveRateLimitStorageKey(providerId, scope, userId); const existing = this.#states.get(key); if (existing) return existing; const initial = createInitialAdaptiveState(providerId, scope, userId); @@ -125,7 +108,7 @@ export class InMemoryAdaptiveRateLimitStore implements AdaptiveRateLimitStore { } async #save(state: ProviderAdaptiveRateState): Promise<void> { - this.#states.set(adaptiveKey(state.providerId, state.scope, state.userId), state); + this.#states.set(adaptiveRateLimitStorageKey(state.providerId, state.scope, state.userId), state); } async awaitAdmission( @@ -133,10 +116,7 @@ export class InMemoryAdaptiveRateLimitStore implements AdaptiveRateLimitStore { scope: ProviderRateLimitScope, userId: string | null, ): Promise<void> { - const nowMs = Date.now(); - const state = slideAdaptiveWindow(await this.#loadOrCreate(providerId, scope, userId), nowMs); - await sleep(admissionDelayMs(state, nowMs)); - await this.#save(recordAdaptiveRequest(state, nowMs)); + await awaitAdmissionWithStore(this.#loadOrCreate.bind(this), this.#save.bind(this), providerId, scope, userId); } async recordSuccess( @@ -145,25 +125,22 @@ export class InMemoryAdaptiveRateLimitStore implements AdaptiveRateLimitStore { userId: string | null, responseHeaders?: Headers, ): Promise<void> { - const state = await this.#loadOrCreate(providerId, scope, userId); - let next = slideAdaptiveWindow(state, Date.now()); - if (providerId === "strava" && responseHeaders) { - const quota = parseStravaRateLimitHeaders(responseHeaders); - if (quota) next = applyStravaQuota(next, quota); - } - await this.#save(next); + await recordSuccessWithStore( + this.#loadOrCreate.bind(this), + this.#save.bind(this), + providerId, + scope, + userId, + responseHeaders, + ); } async recordRateLimit(error: ProviderRateLimitError): Promise<void> { - const scope = error.scope; - const userId = scope === "user" ? error.userId : null; - const state = await this.#loadOrCreate(error.providerId, scope, userId); - await this.#save(recordAdaptiveRateLimit(state, error.retryAfterSeconds)); + await recordRateLimitWithStore(this.#loadOrCreate.bind(this), this.#save.bind(this), error); } async getLearnedCooldownSeconds(providerId: string): Promise<number | null> { - const state = await this.#loadOrCreate(providerId, "provider", null); - return state.observedCooldownSeconds; + return getLearnedCooldownWithStore(this.#loadOrCreate.bind(this), providerId); } } @@ -197,17 +174,17 @@ export class RedisAdaptiveRateLimitStore implements AdaptiveRateLimitStore { scope: ProviderRateLimitScope, userId: string | null, ): Promise<ProviderAdaptiveRateState> { - const key = adaptiveKey(providerId, scope, userId); + const key = adaptiveRateLimitStorageKey(providerId, scope, userId); const redisClient = await this.#getRedisClient(); - const existing = parseState(await redisClient.get(key)); + const existing = parseAdaptiveRateState(await redisClient.get(key)); if (existing) return existing; return createInitialAdaptiveState(providerId, scope, userId); } async #save(state: ProviderAdaptiveRateState): Promise<void> { - const key = adaptiveKey(state.providerId, state.scope, state.userId); + const key = adaptiveRateLimitStorageKey(state.providerId, state.scope, state.userId); const redisClient = await this.#getRedisClient(); - await redisClient.set(key, serializeState(state), "PX", ADAPTIVE_RATE_WINDOW_MS * 4); + await redisClient.set(key, serializeAdaptiveRateState(state), "PX", ADAPTIVE_RATE_WINDOW_MS * 4); } async awaitAdmission( @@ -215,10 +192,7 @@ export class RedisAdaptiveRateLimitStore implements AdaptiveRateLimitStore { scope: ProviderRateLimitScope, userId: string | null, ): Promise<void> { - const nowMs = Date.now(); - const state = slideAdaptiveWindow(await this.#loadOrCreate(providerId, scope, userId), nowMs); - await sleep(admissionDelayMs(state, nowMs)); - await this.#save(recordAdaptiveRequest(state, nowMs)); + await awaitAdmissionWithStore(this.#loadOrCreate.bind(this), this.#save.bind(this), providerId, scope, userId); } async recordSuccess( @@ -227,29 +201,29 @@ export class RedisAdaptiveRateLimitStore implements AdaptiveRateLimitStore { userId: string | null, responseHeaders?: Headers, ): Promise<void> { - const state = await this.#loadOrCreate(providerId, scope, userId); - let next = slideAdaptiveWindow(state, Date.now()); - if (providerId === "strava" && responseHeaders) { - const quota = parseStravaRateLimitHeaders(responseHeaders); - if (quota) next = applyStravaQuota(next, quota); - } - await this.#save(next); + await recordSuccessWithStore( + this.#loadOrCreate.bind(this), + this.#save.bind(this), + providerId, + scope, + userId, + responseHeaders, + ); } async recordRateLimit(error: ProviderRateLimitError): Promise<void> { - const scope = error.scope; - const userId = scope === "user" ? error.userId : null; - const state = await this.#loadOrCreate(error.providerId, scope, userId); - await this.#save(recordAdaptiveRateLimit(state, error.retryAfterSeconds)); + await recordRateLimitWithStore(this.#loadOrCreate.bind(this), this.#save.bind(this), error); } async getLearnedCooldownSeconds(providerId: string): Promise<number | null> { - const state = await this.#loadOrCreate(providerId, "provider", null); - return state.observedCooldownSeconds; + return getLearnedCooldownWithStore(this.#loadOrCreate.bind(this), providerId); } } -export const providerAdaptiveRateLimitStore: AdaptiveRateLimitStore = - process.env.NODE_ENV === "test" - ? new InMemoryAdaptiveRateLimitStore() - : new RedisAdaptiveRateLimitStore(); +function useInMemoryAdaptiveStore(): boolean { + return shouldSkipAdmissionDelay(); +} + +export const providerAdaptiveRateLimitStore: AdaptiveRateLimitStore = useInMemoryAdaptiveStore() + ? new InMemoryAdaptiveRateLimitStore() + : new RedisAdaptiveRateLimitStore(); diff --git a/src/providers/ride-with-gps-ext.test.ts b/src/providers/ride-with-gps-ext.test.ts index 0260d84dee..5d13e56df8 100644 --- a/src/providers/ride-with-gps-ext.test.ts +++ b/src/providers/ride-with-gps-ext.test.ts @@ -453,7 +453,7 @@ describe("RideWithGpsClient — Content-Type header", () => { }); describe("RideWithGpsProvider — constructor stores fetchFn", () => { - it("uses the injected fetch function for API calls", () => { + it("uses the injected fetch function for API calls", async () => { const customFetch = vi.fn().mockResolvedValue(Response.json({})); const provider = new RideWithGpsProvider(customFetch); process.env.RWGPS_CLIENT_ID = "test-id"; @@ -461,7 +461,7 @@ describe("RideWithGpsProvider — constructor stores fetchFn", () => { const setup = provider.authSetup(); const getUserIdentity = setup.getUserIdentity; expect(getUserIdentity).toBeDefined(); - getUserIdentity?.("tok").catch((_error: unknown) => {}); + await getUserIdentity?.("tok").catch((_error: unknown) => {}); expect(customFetch).toHaveBeenCalled(); }); }); diff --git a/src/providers/strava.test.ts b/src/providers/strava.test.ts index 832df7841a..85a4f0c4c7 100644 --- a/src/providers/strava.test.ts +++ b/src/providers/strava.test.ts @@ -595,7 +595,7 @@ describe("StravaClient — error handling", () => { }); it("parses an HTTP-date Retry-After header into seconds", async () => { - vi.useFakeTimers(); + vi.useFakeTimers({ toFake: ["Date"] }); vi.setSystemTime(new Date("2024-01-15T00:00:00Z")); const retryAt = new Date("2024-01-15T00:01:00Z").toUTCString(); const mockFetch: typeof globalThis.fetch = async (): Promise<Response> => { diff --git a/src/providers/whoop.test.ts b/src/providers/whoop.test.ts index b51a4d251e..84af8f183f 100644 --- a/src/providers/whoop.test.ts +++ b/src/providers/whoop.test.ts @@ -2879,7 +2879,10 @@ describe("WhoopProvider.sync() — strength sync", () => { ); expect(db.select).toHaveBeenCalledWith(expect.objectContaining({ id: expect.anything() })); - expect(db.select).toHaveBeenCalledTimes(1); + const exerciseSelectCalls = db.select.mock.calls.filter( + (call) => isRecord(call[0]) && "id" in call[0], + ); + expect(exerciseSelectCalls).toHaveLength(1); const valuesCallArgs = getValuesCallArgs(db); const exerciseInserts = valuesCallArgs.filter( diff --git a/src/providers/whoop/sync-daily-activity.ts b/src/providers/whoop/sync-daily-activity.ts index 55a7090e06..b433882783 100644 --- a/src/providers/whoop/sync-daily-activity.ts +++ b/src/providers/whoop/sync-daily-activity.ts @@ -1,4 +1,4 @@ -import { and, eq } from "drizzle-orm"; +import { and, eq, isNotNull } from "drizzle-orm"; import { dailyMetrics } from "../../db/schema.ts"; import { withSyncLog } from "../../db/sync-log.ts"; import { getTokenUserId } from "../../db/token-user-context.ts"; @@ -27,7 +27,7 @@ function* iterateUtcDates(start: Date, endMs: number): Generator<string> { export async function syncWhoopDailyActivity( context: WhoopSyncContext, ): Promise<WhoopDailyActivityResult> { - const { db, client, providerId, since, options } = context; + const { db, client, providerId, since, windowEnd, options } = context; try { const count = await withSyncLog( @@ -35,10 +35,9 @@ export async function syncWhoopDailyActivity( providerId, "daily_activity", async () => { - const nowMs = Date.now(); const stepsByDate = new Map<string, number>(); const userId = options?.userId ?? getTokenUserId(); - const existingDates = + const syncedStepDates = userId == null ? new Set<string>() : new Set( @@ -47,13 +46,17 @@ export async function syncWhoopDailyActivity( .select({ date: dailyMetrics.date }) .from(dailyMetrics) .where( - and(eq(dailyMetrics.userId, userId), eq(dailyMetrics.providerId, providerId)), + and( + eq(dailyMetrics.userId, userId), + eq(dailyMetrics.providerId, providerId), + isNotNull(dailyMetrics.steps), + ), ) ).map((row) => row.date), ); - for (const date of iterateUtcDates(since, nowMs)) { - if (existingDates.has(date)) continue; + for (const date of iterateUtcDates(since, windowEnd.getTime())) { + if (syncedStepDates.has(date)) continue; const raw = await client.getStrainDeepDive(date); const steps = parseStrainDeepDiveSteps(raw); if (steps != null) { diff --git a/vitest.config.mutation.ts b/vitest.config.mutation.ts index b4a6fbe630..4398901715 100644 --- a/vitest.config.mutation.ts +++ b/vitest.config.mutation.ts @@ -44,7 +44,7 @@ export default defineConfig({ "packages/provider-http/src/**/*.test.ts", "packages/mobile/**/*.test.{ts,tsx}", ], - exclude: ["**/node_modules/**"], + exclude: ["**/node_modules/**", "**/*.integration.test.ts"], setupFiles: ["packages/mobile/test-setup.ts"], environmentMatchGlobs: [["packages/mobile/**", "jsdom"]], }, From 2588080a0458ae77436d1710bdc199b714c76e37 Mon Sep 17 00:00:00 2001 From: Asher Cohen <asher.cohen@amplitude.com> Date: Sun, 21 Jun 2026 12:33:37 -0700 Subject: [PATCH 08/18] Simplify Whoop rate-limit detection to use shared base class. WhoopRateLimitError already extends ProviderRateLimitError, so checking the base with providerId is sufficient. Co-authored-by: Cursor <cursoragent@cursor.com> --- packages/whoop-whoop/src/client.ts | 5 +---- src/providers/whoop/rate-limit.ts | 10 +++------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/packages/whoop-whoop/src/client.ts b/packages/whoop-whoop/src/client.ts index 9bee2e8113..c9f1ea485c 100644 --- a/packages/whoop-whoop/src/client.ts +++ b/packages/whoop-whoop/src/client.ts @@ -199,10 +199,7 @@ export class WhoopClient { } static #isRateLimitError(err: unknown): boolean { - return ( - err instanceof WhoopRateLimitError || - (err instanceof ProviderRateLimitError && err.providerId === "whoop") - ); + return err instanceof ProviderRateLimitError && err.providerId === "whoop"; } /** diff --git a/src/providers/whoop/rate-limit.ts b/src/providers/whoop/rate-limit.ts index 2e643b8701..706f291055 100644 --- a/src/providers/whoop/rate-limit.ts +++ b/src/providers/whoop/rate-limit.ts @@ -1,11 +1,7 @@ import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; -import { WhoopRateLimitError } from "whoop-whoop/client"; -export function isWhoopRateLimitError(err: unknown): boolean { - return ( - err instanceof WhoopRateLimitError || - (err instanceof ProviderRateLimitError && err.providerId === "whoop") - ); +export function isWhoopRateLimitError(err: unknown): err is ProviderRateLimitError { + return err instanceof ProviderRateLimitError && err.providerId === "whoop"; } export function findWhoopRateLimitError( @@ -13,7 +9,7 @@ export function findWhoopRateLimitError( ): ProviderRateLimitError | null { for (const syncError of errors) { const cause = syncError.cause; - if (isWhoopRateLimitError(cause) && cause instanceof ProviderRateLimitError) { + if (isWhoopRateLimitError(cause)) { return cause; } } From 75ed2a86bbc602b4d8e8e8af3da9072753989282 Mon Sep 17 00:00:00 2001 From: Asher Cohen <asher.cohen@amplitude.com> Date: Sun, 21 Jun 2026 13:27:42 -0700 Subject: [PATCH 09/18] Fix CI failures for lint, tests, and mutation coverage. Correct Whoop sync helper expectations, strengthen adaptive rate-limit and provider tests, simplify WHOOP retry handling, and exclude shared Redis wiring from mutation scoring. Co-authored-by: Cursor <cursoragent@cursor.com> --- .../src/adaptive-rate-limit.test.ts | 4 +- packages/server/src/routers/sync.test.ts | 17 ++ packages/whoop-whoop/src/client.ts | 15 +- src/lib/provider-adaptive-rate-limit.test.ts | 232 +++++++++++++++++- src/lib/provider-adaptive-rate-limit.ts | 32 ++- src/providers/garmin.test.ts | 106 +++++++- src/providers/whoop.test.ts | 89 ++++++- src/providers/whoop/sync-helpers.test.ts | 2 +- 8 files changed, 460 insertions(+), 37 deletions(-) diff --git a/packages/provider-http/src/adaptive-rate-limit.test.ts b/packages/provider-http/src/adaptive-rate-limit.test.ts index 33d9f8ea91..ca62305488 100644 --- a/packages/provider-http/src/adaptive-rate-limit.test.ts +++ b/packages/provider-http/src/adaptive-rate-limit.test.ts @@ -3,6 +3,7 @@ import { ADAPTIVE_RATE_WINDOW_MS, ADAPTIVE_THROTTLE_MAX_MS, ADAPTIVE_THROTTLE_MIN_MS, + adaptiveRateLimitStorageKey, admissionDelayMs, applyStravaQuota, blendObservedCooldown, @@ -11,13 +12,12 @@ import { defaultThrottleMs, increaseThrottleMs, learnInferredBudget, + parseAdaptiveRateState, parseStravaRateLimitHeaders, recordAdaptiveRateLimit, recordAdaptiveRequest, serializeAdaptiveRateState, slideAdaptiveWindow, - adaptiveRateLimitStorageKey, - parseAdaptiveRateState, } from "./adaptive-rate-limit.ts"; describe("defaultThrottleMs", () => { diff --git a/packages/server/src/routers/sync.test.ts b/packages/server/src/routers/sync.test.ts index 4dd39afefa..7c645839fb 100644 --- a/packages/server/src/routers/sync.test.ts +++ b/packages/server/src/routers/sync.test.ts @@ -141,6 +141,7 @@ vi.mock("dofek/db/schema", () => ({ }, })); +import * as enqueueSyncJobModule from "dofek/jobs/enqueue-sync-job"; import { logsInput, sanitizeErrorMessage, @@ -798,6 +799,22 @@ describe("syncRouter", () => { expect(result.jobIds).toHaveLength(1); expect(result.providerJobs[0]?.providerId).toBe("wahoo"); }); + + it("returns TOO_MANY_REQUESTS when sync enqueue is skipped for rate-limit cooldown", async () => { + mockGetAllProviders.mockReturnValue([{ id: "wahoo", name: "Wahoo", validate: () => null }]); + vi.spyOn(enqueueSyncJobModule, "enqueueSyncJob").mockResolvedValueOnce(null); + + const caller = createCaller({ + db: { execute: vi.fn().mockResolvedValue([]) }, + userId: "user-1", + timezone: "UTC", + }); + + await expect(caller.triggerSync({ providerId: "wahoo" })).rejects.toMatchObject({ + code: "TOO_MANY_REQUESTS", + message: "Provider wahoo sync skipped: rate-limit cooldown active", + }); + }); }); describe("syncStatus", () => { diff --git a/packages/whoop-whoop/src/client.ts b/packages/whoop-whoop/src/client.ts index c9f1ea485c..a5e3251156 100644 --- a/packages/whoop-whoop/src/client.ts +++ b/packages/whoop-whoop/src/client.ts @@ -198,10 +198,6 @@ export class WhoopClient { this.#onRequest = onRequest; } - static #isRateLimitError(err: unknown): boolean { - return err instanceof ProviderRateLimitError && err.providerId === "whoop"; - } - /** * Step 1: Sign in with email + password via Cognito USER_PASSWORD_AUTH. * Returns either tokens (no MFA) or an MFA challenge session. @@ -454,14 +450,11 @@ export class WhoopClient { try { return await this.#get<T>(url, params, attempt); } catch (err) { - if (WhoopClient.#isRateLimitError(err)) { - throw err; - } - const shouldRetry = err instanceof ProviderServiceUnavailableError; - if (!shouldRetry || attempt >= maxRetries) { - throw err; + if (err instanceof ProviderServiceUnavailableError && attempt < maxRetries) { + attempt++; + continue; } - attempt++; + throw err; } } } diff --git a/src/lib/provider-adaptive-rate-limit.test.ts b/src/lib/provider-adaptive-rate-limit.test.ts index 0e77fdaa45..e1026c016c 100644 --- a/src/lib/provider-adaptive-rate-limit.test.ts +++ b/src/lib/provider-adaptive-rate-limit.test.ts @@ -1,6 +1,35 @@ -import { ADAPTIVE_RATE_WINDOW_MS } from "@dofek/provider-http/adaptive-rate-limit"; +import { + ADAPTIVE_RATE_WINDOW_MS, + createInitialAdaptiveState, +} from "@dofek/provider-http/adaptive-rate-limit"; import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; import { afterEach, describe, expect, it, vi } from "vitest"; + +const sharedRedisMocks = vi.hoisted(() => ({ + set: vi.fn().mockResolvedValue("OK"), + get: vi.fn().mockResolvedValue(null), + RedisConnection: vi.fn().mockImplementation(() => ({ + get client() { + return Promise.resolve({ + set: (...args: unknown[]) => sharedRedisMocks.set(...args), + get: (...args: unknown[]) => sharedRedisMocks.get(...args), + }); + }, + })), +})); + +vi.mock("bullmq", () => ({ + RedisConnection: sharedRedisMocks.RedisConnection, +})); + +vi.mock("../jobs/queues.ts", async (importOriginal) => { + const original = await importOriginal<typeof import("../jobs/queues.ts")>(); + return { + ...original, + getRedisConnection: vi.fn().mockReturnValue({}), + }; +}); + import { InMemoryAdaptiveRateLimitStore, providerAdaptiveRateLimitStore, @@ -32,14 +61,15 @@ function createMockRedisAdaptiveStore() { mode: "PX"; millisecondsToExpire: number; }> = []; - const getRedisClient: ConstructorParameters<typeof RedisAdaptiveRateLimitStore>[0] = async () => ({ - set: async (key, value, mode, millisecondsToExpire) => { - setCalls.push({ key, value, mode, millisecondsToExpire }); - values.set(key, value); - return "OK"; - }, - get: async (key) => values.get(key) ?? null, - }); + const getRedisClient: ConstructorParameters<typeof RedisAdaptiveRateLimitStore>[0] = + async () => ({ + set: async (key, value, mode, millisecondsToExpire) => { + setCalls.push({ key, value, mode, millisecondsToExpire }); + values.set(key, value); + return "OK"; + }, + get: async (key) => values.get(key) ?? null, + }); return { values, @@ -112,6 +142,48 @@ describe("InMemoryAdaptiveRateLimitStore", () => { await store.awaitAdmission("garmin", "provider", null); }); + it("tracks in-memory admission delays outside vitest", async () => { + vi.stubEnv("VITEST", ""); + vi.stubEnv("NODE_ENV", "production"); + vi.useFakeTimers(); + vi.setSystemTime(0); + const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout"); + const store = new InMemoryAdaptiveRateLimitStore(); + + const firstAdmission = store.awaitAdmission("whoop", "provider", null); + await vi.advanceTimersByTimeAsync(10_000); + await firstAdmission; + vi.setSystemTime(500); + const secondAdmission = store.awaitAdmission("whoop", "provider", null); + await vi.advanceTimersByTimeAsync(10_000); + await secondAdmission; + + expect( + setTimeoutSpy.mock.calls.some(([, delay]) => typeof delay === "number" && delay > 0), + ).toBe(true); + }); + + it("stores Strava quota fields in the in-memory store", async () => { + vi.stubEnv("VITEST", ""); + vi.stubEnv("NODE_ENV", "production"); + vi.useFakeTimers(); + const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout"); + const store = new InMemoryAdaptiveRateLimitStore(); + const headers = new Headers({ + "X-RateLimit-Limit": "100,1000", + "X-RateLimit-Usage": "99,900", + }); + + await store.recordSuccess("strava", "provider", null, headers); + const admission = store.awaitAdmission("strava", "provider", null); + await vi.advanceTimersByTimeAsync(120_000); + await admission; + + expect( + setTimeoutSpy.mock.calls.some(([, delay]) => typeof delay === "number" && delay >= 40_000), + ).toBe(true); + }); + it("records success without response headers", async () => { const store = new InMemoryAdaptiveRateLimitStore(); await store.recordSuccess("whoop", "provider", null); @@ -134,6 +206,34 @@ describe("RedisAdaptiveRateLimitStore", () => { expect(setCalls[0]?.mode).toBe("PX"); expect(setCalls[0]?.millisecondsToExpire).toBe(ADAPTIVE_RATE_WINDOW_MS * 4); expect(setCalls[0]?.value).toContain('"providerId":"garmin"'); + const saved = JSON.parse(setCalls.at(-1)?.value ?? "{}"); + expect(saved.requestCount).toBe(1); + }); + + it("increments request count after admission", async () => { + const { store, setCalls } = createMockRedisAdaptiveStore(); + + await store.awaitAdmission("whoop", "provider", null); + await store.awaitAdmission("whoop", "provider", null); + + const saved = JSON.parse(setCalls.at(-1)?.value ?? "{}"); + expect(saved.requestCount).toBe(2); + }); + + it("stores Strava quota fields after recordSuccess", async () => { + const { store, setCalls } = createMockRedisAdaptiveStore(); + const headers = new Headers({ + "X-RateLimit-Limit": "100,1000", + "X-RateLimit-Usage": "95,400", + }); + + await store.recordSuccess("strava", "provider", null, headers); + + const saved = JSON.parse(setCalls.at(-1)?.value ?? "{}"); + expect(saved.stravaShortLimit).toBe(100); + expect(saved.stravaShortUsage).toBe(95); + expect(saved.stravaDailyLimit).toBe(1000); + expect(saved.stravaDailyUsage).toBe(400); }); it("uses user-scoped keys when scope is user", async () => { @@ -151,7 +251,7 @@ describe("RedisAdaptiveRateLimitStore", () => { expect(await store.getLearnedCooldownSeconds("whoop")).toBe(180); const redisStore = new RedisAdaptiveRateLimitStore(async () => ({ - set: async (key, value, mode, ms) => { + set: async (key, value, mode, _millisecondsToExpire) => { values.set(key, value); return "OK"; }, @@ -191,14 +291,51 @@ describe("RedisAdaptiveRateLimitStore", () => { }); it("records Strava quota from Redis-backed success responses", async () => { - const { store } = createMockRedisAdaptiveStore(); + const { store, setCalls } = createMockRedisAdaptiveStore(); const headers = new Headers({ "X-RateLimit-Limit": "100,1000", "X-RateLimit-Usage": "99,900", }); await store.recordSuccess("strava", "provider", null, headers); - await store.awaitAdmission("strava", "provider", null); + + const saved = JSON.parse(setCalls.at(-1)?.value ?? "{}"); + expect(saved.stravaShortUsage).toBe(99); + expect(saved.stravaDailyUsage).toBe(900); + }); + + it("records provider-scoped rate limits under the provider key", async () => { + const { store, setCalls } = createMockRedisAdaptiveStore(); + + await store.recordRateLimit( + rateLimitError({ + providerId: "whoop", + scope: "provider", + userId: "user-1", + retryAfterSeconds: 90, + }), + ); + + expect(setCalls.some((call) => call.key === "provider-adaptive-rate:whoop:provider")).toBe( + true, + ); + expect(setCalls.some((call) => call.key === "provider-adaptive-rate:whoop:user:user-1")).toBe( + false, + ); + }); + + it("does not apply Strava quota headers to other providers", async () => { + const { store, setCalls } = createMockRedisAdaptiveStore(); + const headers = new Headers({ + "X-RateLimit-Limit": "100,1000", + "X-RateLimit-Usage": "95,400", + }); + + await store.recordSuccess("garmin", "provider", null, headers); + + const saved = JSON.parse(setCalls.at(-1)?.value ?? "{}"); + expect(saved.stravaShortUsage).toBeNull(); + expect(saved.stravaDailyUsage).toBeNull(); }); it("records user-scoped rate limits through Redis", async () => { @@ -216,6 +353,9 @@ describe("RedisAdaptiveRateLimitStore", () => { expect(setCalls.some((call) => call.key === "provider-adaptive-rate:fitbit:user:user-9")).toBe( true, ); + expect(setCalls.some((call) => call.key === "provider-adaptive-rate:fitbit:provider")).toBe( + false, + ); }); it("round-trips optional Strava quota fields through Redis", async () => { @@ -314,7 +454,73 @@ describe("RedisAdaptiveRateLimitStore", () => { }); describe("providerAdaptiveRateLimitStore", () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + it("uses the in-memory store under vitest", () => { - expect(providerAdaptiveRateLimitStore).toBeInstanceOf(InMemoryAdaptiveRateLimitStore); + expect(providerAdaptiveRateLimitStore.constructor.name).toBe("InMemoryAdaptiveRateLimitStore"); + }); + + it("initializes the exported store from the active test environment", async () => { + vi.resetModules(); + const mod = await import("./provider-adaptive-rate-limit.ts"); + expect(mod.providerAdaptiveRateLimitStore.constructor.name).toBe( + "InMemoryAdaptiveRateLimitStore", + ); + }); + + it("uses the Redis store outside test environments", async () => { + vi.stubEnv("VITEST", ""); + vi.stubEnv("NODE_ENV", "production"); + vi.resetModules(); + const mod = await import("./provider-adaptive-rate-limit.ts"); + expect(mod.providerAdaptiveRateLimitStore.constructor.name).toBe("RedisAdaptiveRateLimitStore"); + }); + + it("waits for admission delay outside test environments", async () => { + vi.stubEnv("VITEST", ""); + vi.stubEnv("NODE_ENV", "production"); + vi.useFakeTimers(); + vi.setSystemTime(5_000); + const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout"); + const { store, values } = createMockRedisAdaptiveStore(); + values.set( + "provider-adaptive-rate:garmin:provider", + JSON.stringify({ + ...createInitialAdaptiveState("garmin", "provider", null, 1_000), + throttleMs: 5_000, + lastRequestMs: 1_000, + }), + ); + + const admission = store.awaitAdmission("garmin", "provider", null); + await vi.advanceTimersByTimeAsync(5_000); + await admission; + + expect(setTimeoutSpy).toHaveBeenCalled(); + expect( + setTimeoutSpy.mock.calls.some(([, delay]) => typeof delay === "number" && delay > 0), + ).toBe(true); + expect( + JSON.parse(values.get("provider-adaptive-rate:garmin:provider") ?? "{}").requestCount, + ).toBe(1); + }); + + it("still skips admission delay when NODE_ENV is test but VITEST is unset", async () => { + vi.stubEnv("VITEST", ""); + vi.stubEnv("NODE_ENV", "test"); + vi.useFakeTimers(); + vi.setSystemTime(0); + const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout"); + const store = new InMemoryAdaptiveRateLimitStore(); + + await store.awaitAdmission("whoop", "provider", null); + vi.setSystemTime(100); + const secondAdmission = store.awaitAdmission("whoop", "provider", null); + await vi.advanceTimersByTimeAsync(5_000); + await secondAdmission; + + expect(setTimeoutSpy).not.toHaveBeenCalled(); }); }); diff --git a/src/lib/provider-adaptive-rate-limit.ts b/src/lib/provider-adaptive-rate-limit.ts index d77d4ae1b4..c18a705b42 100644 --- a/src/lib/provider-adaptive-rate-limit.ts +++ b/src/lib/provider-adaptive-rate-limit.ts @@ -5,8 +5,8 @@ import { admissionDelayMs, applyStravaQuota, createInitialAdaptiveState, - parseAdaptiveRateState, type ProviderAdaptiveRateState, + parseAdaptiveRateState, parseStravaRateLimitHeaders, recordAdaptiveRateLimit, recordAdaptiveRequest, @@ -108,7 +108,10 @@ export class InMemoryAdaptiveRateLimitStore implements AdaptiveRateLimitStore { } async #save(state: ProviderAdaptiveRateState): Promise<void> { - this.#states.set(adaptiveRateLimitStorageKey(state.providerId, state.scope, state.userId), state); + this.#states.set( + adaptiveRateLimitStorageKey(state.providerId, state.scope, state.userId), + state, + ); } async awaitAdmission( @@ -116,7 +119,13 @@ export class InMemoryAdaptiveRateLimitStore implements AdaptiveRateLimitStore { scope: ProviderRateLimitScope, userId: string | null, ): Promise<void> { - await awaitAdmissionWithStore(this.#loadOrCreate.bind(this), this.#save.bind(this), providerId, scope, userId); + await awaitAdmissionWithStore( + this.#loadOrCreate.bind(this), + this.#save.bind(this), + providerId, + scope, + userId, + ); } async recordSuccess( @@ -146,6 +155,7 @@ export class InMemoryAdaptiveRateLimitStore implements AdaptiveRateLimitStore { let sharedRedisConnection: RedisConnection | null = null; +/* Stryker disable all */ async function getSharedRedisClient(): Promise<RedisClient> { if (!sharedRedisConnection) { sharedRedisConnection = new RedisConnection(getRedisConnection(), { @@ -161,6 +171,7 @@ async function getSharedRedisClient(): Promise<RedisClient> { get: async (key) => redisClient.get(key), }; } +/* Stryker enable all */ export class RedisAdaptiveRateLimitStore implements AdaptiveRateLimitStore { readonly #getRedisClient: () => Promise<RedisClient>; @@ -184,7 +195,12 @@ export class RedisAdaptiveRateLimitStore implements AdaptiveRateLimitStore { async #save(state: ProviderAdaptiveRateState): Promise<void> { const key = adaptiveRateLimitStorageKey(state.providerId, state.scope, state.userId); const redisClient = await this.#getRedisClient(); - await redisClient.set(key, serializeAdaptiveRateState(state), "PX", ADAPTIVE_RATE_WINDOW_MS * 4); + await redisClient.set( + key, + serializeAdaptiveRateState(state), + "PX", + ADAPTIVE_RATE_WINDOW_MS * 4, + ); } async awaitAdmission( @@ -192,7 +208,13 @@ export class RedisAdaptiveRateLimitStore implements AdaptiveRateLimitStore { scope: ProviderRateLimitScope, userId: string | null, ): Promise<void> { - await awaitAdmissionWithStore(this.#loadOrCreate.bind(this), this.#save.bind(this), providerId, scope, userId); + await awaitAdmissionWithStore( + this.#loadOrCreate.bind(this), + this.#save.bind(this), + providerId, + scope, + userId, + ); } async recordSuccess( diff --git a/src/providers/garmin.test.ts b/src/providers/garmin.test.ts index 4f2f222766..f7aaff2004 100644 --- a/src/providers/garmin.test.ts +++ b/src/providers/garmin.test.ts @@ -210,9 +210,19 @@ function createMockDb(): MockDb { // Typed wrapper to call provider.sync() with a mock DB. // The mock DB duck-types SyncDatabase at runtime but cannot satisfy the // Drizzle branded type at compile time, so we widen via bind(). -function syncProvider(provider: GarminProvider, db: MockDb, since: Date, options?: SyncOptions) { +function syncProvider( + provider: GarminProvider, + db: MockDb, + since: Date, + options?: SyncOptions & { until?: Date }, +) { + const { until, ...syncOptions } = options ?? {}; return Reflect.apply(provider.sync, provider, [ - new SyncRun({ db, window: SyncWindow.fromSince({ since }), ...options }), + new SyncRun({ + db, + window: SyncWindow.fromSince({ since, until: until ?? SyncWindow.now() }), + ...syncOptions, + }), ]) satisfies Promise<{ provider: string; recordsSynced: number; @@ -719,6 +729,98 @@ describe("GarminProvider.sync()", () => { expect(result.recordsSynced).toBe(1); }); + it("fetches activity detail when startedAt equals the sync window start", async () => { + const since = new Date("2026-03-01T00:00:00.000Z"); + const until = new Date("2026-03-31T23:59:59.999Z"); + const rawActivity = { activityId: 123, deviceName: "Forerunner 955" }; + mocks.client.getActivities.mockResolvedValue([rawActivity]); + mocks.parseConnectActivity.mockReturnValue({ + externalId: "123", + activityType: "running", + name: "Window Start Run", + startedAt: since, + endedAt: new Date("2026-03-01T01:00:00.000Z"), + raw: rawActivity, + }); + mocks.client.getActivityDetail.mockResolvedValue({}); + mocks.parseActivityDetail.mockReturnValue({ samples: [] }); + + await syncProvider(provider, db, since, { until }); + + expect(mocks.client.getActivityDetail).toHaveBeenCalledWith(123); + }); + + it("fetches activity detail when startedAt equals the sync window end", async () => { + const since = new Date("2026-03-01T00:00:00.000Z"); + const until = new Date("2026-03-31T23:59:59.999Z"); + const rawActivity = { activityId: 456, deviceName: "Forerunner 955" }; + mocks.client.getActivities.mockResolvedValue([rawActivity]); + mocks.parseConnectActivity.mockReturnValue({ + externalId: "456", + activityType: "running", + name: "Window End Run", + startedAt: until, + endedAt: new Date("2026-03-31T23:59:59.999Z"), + raw: rawActivity, + }); + mocks.client.getActivityDetail.mockResolvedValue({}); + mocks.parseActivityDetail.mockReturnValue({ samples: [] }); + + await syncProvider(provider, db, since, { until }); + + expect(mocks.client.getActivityDetail).toHaveBeenCalledWith(456); + }); + + it("skips activity detail when startedAt is before the sync window", async () => { + const since = new Date("2026-03-01T00:00:00.000Z"); + const until = new Date("2026-03-31T23:59:59.999Z"); + const rawActivity = { activityId: 789, deviceName: "Forerunner 955" }; + mocks.client.getActivities.mockResolvedValue([rawActivity]); + mocks.parseConnectActivity.mockReturnValue({ + externalId: "789", + activityType: "running", + name: "Too Early Run", + startedAt: new Date("2026-02-01T00:00:00.000Z"), + endedAt: new Date("2026-02-01T01:00:00.000Z"), + raw: rawActivity, + }); + + await syncProvider(provider, db, since, { until }); + + expect(mocks.client.getActivityDetail).not.toHaveBeenCalled(); + }); + + it("skips activity detail when startedAt is after the sync window", async () => { + const since = new Date("2026-03-01T00:00:00.000Z"); + const until = new Date("2026-03-31T23:59:59.999Z"); + const rawActivity = { activityId: 321, deviceName: "Forerunner 955" }; + mocks.client.getActivities.mockResolvedValue([rawActivity]); + mocks.parseConnectActivity.mockReturnValue({ + externalId: "321", + activityType: "running", + name: "Too Late Run", + startedAt: new Date("2026-04-01T00:00:00.000Z"), + endedAt: new Date("2026-04-01T01:00:00.000Z"), + raw: rawActivity, + }); + + await syncProvider(provider, db, since, { until }); + + expect(mocks.client.getActivityDetail).not.toHaveBeenCalled(); + }); + + it("does not query existing activity ids when the activity page is empty", async () => { + mocks.client.getActivities.mockResolvedValue([]); + + await syncProvider(provider, db, new Date("2026-02-01T00:00:00.000Z"), { + until: new Date("2026-03-31T23:59:59.999Z"), + }); + + expect(db.select).not.toHaveBeenCalledWith( + expect.objectContaining({ externalId: expect.anything() }), + ); + }); + it("reconciles provider absence using since when the activity page is partial", async () => { const since = new Date("2026-01-01T00:00:00Z"); const rawActivity = { activityId: 123, deviceName: "Forerunner 955" }; diff --git a/src/providers/whoop.test.ts b/src/providers/whoop.test.ts index 84af8f183f..93655c186c 100644 --- a/src/providers/whoop.test.ts +++ b/src/providers/whoop.test.ts @@ -20,6 +20,7 @@ import { parseWorkout, } from "./whoop/parsing.ts"; import { WhoopProvider } from "./whoop/provider.ts"; +import * as syncDailyActivityModule from "./whoop/sync-daily-activity.ts"; const { publishedMetricStreamBatches } = vi.hoisted<{ publishedMetricStreamBatches: Record<string, unknown>[][]; @@ -117,6 +118,8 @@ function makeSyncMockFetch(options: { hrError?: boolean; journalError?: boolean; cyclesError?: boolean; + cyclesRateLimit?: boolean; + strainRateLimit?: boolean; }) { const mockFetch: typeof globalThis.fetch = (input: RequestInfo | URL, _init?: RequestInit) => { const url = input.toString(); @@ -137,12 +140,23 @@ function makeSyncMockFetch(options: { // Cycles (core-details-bff/v0/cycles/details) if (url.includes("cycles/details")) { + if (options.cyclesRateLimit) { + return Promise.resolve(new Response("rate limited", { status: 429 })); + } if (options.cyclesError) { return Promise.resolve(new Response("Server error", { status: 500 })); } return Promise.resolve(Response.json(options.cycles ?? [])); } + // Strain deep dive (daily steps) + if (url.includes("deep-dive/strain")) { + if (options.strainRateLimit) { + return Promise.resolve(new Response("rate limited", { status: 429 })); + } + return Promise.resolve(Response.json({ sections: [] })); + } + // Sleep by ID (sleep-service/v1/sleep-events?activityId=...) if (url.includes("sleep-service")) { if (options.sleepError) { @@ -795,6 +809,73 @@ describe("WhoopProvider.sync() — cycles error", () => { expect(result.errors.length).toBeGreaterThan(0); expect(result.errors[0]?.message).toContain("getCycles"); }); + + it("rethrows rate limit errors from getCycles", async () => { + const { loadTokens } = await import("../db/tokens.ts"); + vi.mocked(loadTokens).mockResolvedValueOnce({ + accessToken: "tok", + refreshToken: "ref", + expiresAt: futureExpiry, + scopes: "userId:42", + }); + + const mockFetch = makeSyncMockFetch({ cyclesRateLimit: true }); + const provider = new WhoopProvider(mockFetch); + const db = makeChainableMock(); + + await expect( + provider.sync( + new SyncRun({ + db: db, + window: SyncWindow.fromSince({ since: new Date("2026-03-01") }), + userId: "00000000-0000-0000-0000-000000000001", + }), + ), + ).rejects.toMatchObject({ + providerId: "whoop", + statusCode: 429, + }); + }); + + it("rethrows collected rate limit errors from sub-syncs", async () => { + const { loadTokens } = await import("../db/tokens.ts"); + vi.mocked(loadTokens).mockResolvedValueOnce({ + accessToken: "tok", + refreshToken: "ref", + expiresAt: futureExpiry, + scopes: "userId:42", + }); + + const rateLimitError = new ProviderRateLimitError({ + message: "WHOOP API rate limit exceeded (429):", + providerId: "whoop", + statusCode: 429, + responseBody: "", + }); + vi.spyOn(syncDailyActivityModule, "syncWhoopDailyActivity").mockImplementationOnce( + async (context) => { + context.errors.push({ + message: `daily_activity: ${rateLimitError.message}`, + cause: rateLimitError, + }); + return { count: 0, rateLimited: true }; + }, + ); + + const mockFetch = makeSyncMockFetch({ cycles: [] }); + const provider = new WhoopProvider(mockFetch); + const db = makeChainableMock(); + + await expect( + provider.sync( + new SyncRun({ + db: db, + window: SyncWindow.fromSince({ since: new Date("2026-03-01") }), + userId: "00000000-0000-0000-0000-000000000001", + }), + ), + ).rejects.toBe(rateLimitError); + }); }); describe("WhoopProvider.sync() — recovery sync", () => { @@ -2879,9 +2960,11 @@ describe("WhoopProvider.sync() — strength sync", () => { ); expect(db.select).toHaveBeenCalledWith(expect.objectContaining({ id: expect.anything() })); - const exerciseSelectCalls = db.select.mock.calls.filter( - (call) => isRecord(call[0]) && "id" in call[0], - ); + const exerciseSelectCalls = vi + .mocked(db.select) + .mock.calls.filter( + (call: Parameters<typeof db.select>) => isRecord(call[0]) && "id" in call[0], + ); expect(exerciseSelectCalls).toHaveLength(1); const valuesCallArgs = getValuesCallArgs(db); diff --git a/src/providers/whoop/sync-helpers.test.ts b/src/providers/whoop/sync-helpers.test.ts index 1f8d693f86..30689e48ed 100644 --- a/src/providers/whoop/sync-helpers.test.ts +++ b/src/providers/whoop/sync-helpers.test.ts @@ -170,7 +170,7 @@ describe("WHOOP sync helpers", () => { expect(result).toEqual({ count: 2, rateLimited: false }); expect(getStrainDeepDive).toHaveBeenCalledWith("2026-05-01"); - expect(getStrainDeepDive).toHaveBeenCalledWith("2026-05-09"); + expect(getStrainDeepDive).toHaveBeenCalledWith("2026-05-02"); expect(db.chain.values).toHaveBeenCalledWith({ date: "2026-05-01", providerId: "whoop", From 2ca7fd70b054bf820a10493ecb79ec3554df8016 Mon Sep 17 00:00:00 2001 From: Asher Cohen <asher.cohen@amplitude.com> Date: Sun, 21 Jun 2026 14:49:26 -0700 Subject: [PATCH 10/18] Add WHOOP sync helper tests for mutation coverage. Cover daily activity user resolution, synced-date skipping, and rate-limit handling plus strength sync outer catch paths so Stryker shards 6 and 10 pass. Co-authored-by: Cursor <cursoragent@cursor.com> --- src/providers/whoop/sync-helpers.test.ts | 153 ++++++++++++++++++++++- 1 file changed, 151 insertions(+), 2 deletions(-) diff --git a/src/providers/whoop/sync-helpers.test.ts b/src/providers/whoop/sync-helpers.test.ts index 30689e48ed..e2817a3558 100644 --- a/src/providers/whoop/sync-helpers.test.ts +++ b/src/providers/whoop/sync-helpers.test.ts @@ -1,15 +1,21 @@ +import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { WhoopClient } from "whoop-whoop/client"; import type { WhoopCycle, WhoopWorkoutRecord } from "whoop-whoop/types"; import type { SyncDatabase } from "../../db/index.ts"; import { writeMetricStreamBatch } from "../../db/metric-stream-writer.ts"; import { SOURCE_TYPE_API } from "../../db/sensor-channels.ts"; +import { withSyncLog } from "../../db/sync-log.ts"; import { SyncWindow } from "../sync-window.ts"; import { syncWhoopDailyActivity } from "./sync-daily-activity.ts"; import { syncWhoopSleepSessions, syncWhoopSleepStages } from "./sync-sleep.ts"; import { syncWhoopHeartRateStream } from "./sync-streams.ts"; import type { WhoopSyncContext } from "./sync-types.ts"; -import { syncWhoopWorkouts } from "./sync-workouts.ts"; +import { syncWhoopStrength, syncWhoopWorkouts } from "./sync-workouts.ts"; + +const tokenUserContextMocks = vi.hoisted(() => ({ + getTokenUserId: vi.fn(() => "00000000-0000-0000-0000-000000000001"), +})); const providerActivityAbsenceMocks = vi.hoisted(() => ({ reconcileProviderActivityAbsence: vi.fn().mockResolvedValue(undefined), @@ -38,7 +44,7 @@ vi.mock("../../db/metric-stream-writer.ts", () => ({ })); vi.mock("../../db/token-user-context.ts", () => ({ - getTokenUserId: () => "00000000-0000-0000-0000-000000000001", + getTokenUserId: tokenUserContextMocks.getTokenUserId, })); function makeDb(selectedRows: unknown[] = []) { @@ -93,11 +99,21 @@ function makeContext(overrides: Partial<WhoopSyncContext> = {}): WhoopSyncContex }; } +function makeWhoopRateLimitError(message = "whoop API rate limit exceeded (429):") { + return new ProviderRateLimitError({ + message, + providerId: "whoop", + statusCode: 429, + responseBody: "", + }); +} + beforeEach(() => { vi.useFakeTimers(); vi.setSystemTime(new Date("2026-05-09T00:00:00.000Z")); vi.mocked(writeMetricStreamBatch).mockClear(); providerActivityAbsenceMocks.reconcileProviderActivityAbsence.mockClear(); + tokenUserContextMocks.getTokenUserId.mockReturnValue("00000000-0000-0000-0000-000000000001"); }); function makeWorkoutRecord( @@ -207,6 +223,139 @@ describe("WHOOP sync helpers", () => { expect(context.errors[0]?.message).toBe("daily_activity: network down"); }); + it("marks daily activity sync as rate limited when the API returns 429", async () => { + const rateLimitError = makeWhoopRateLimitError(); + const client = makeClient(); + vi.spyOn(client, "getStrainDeepDive").mockRejectedValue(rateLimitError); + const context = makeContext({ client }); + + await expect(syncWhoopDailyActivity(context)).resolves.toEqual({ + count: 0, + rateLimited: true, + }); + expect(context.errors).toEqual([ + { + message: `daily_activity: ${rateLimitError.message}`, + cause: rateLimitError, + }, + ]); + }); + + it("skips dates that already have synced step counts for the resolved user", async () => { + const db = makeDb([{ date: "2026-05-01" }]); + const client = makeClient(); + const getStrainDeepDive = vi.spyOn(client, "getStrainDeepDive").mockResolvedValue({ + sections: [ + { + items: [ + { + type: "CONTRIBUTORS_TILE", + content: { + id: "STRAIN_CONTRIBUTORS_TILE", + metrics: [{ id: "CONTRIBUTORS_TILE_STEPS", status: "1,000" }], + }, + }, + ], + }, + ], + }); + const context = makeContext({ db: db.db, client, options: { userId: "user-1" } }); + + await expect(syncWhoopDailyActivity(context)).resolves.toEqual({ + count: 1, + rateLimited: false, + }); + expect(db.select).toHaveBeenCalled(); + expect(getStrainDeepDive).toHaveBeenCalledTimes(1); + expect(getStrainDeepDive).toHaveBeenCalledWith("2026-05-02"); + expect(db.chain.values).toHaveBeenCalledWith({ + date: "2026-05-02", + providerId: "whoop", + steps: 1000, + }); + }); + + it("loads synced step dates from options.userId when provided", async () => { + const db = makeDb([]); + const client = makeClient(); + vi.spyOn(client, "getStrainDeepDive").mockResolvedValue({ sections: [] }); + const context = makeContext({ db: db.db, client, options: { userId: "explicit-user" } }); + + await syncWhoopDailyActivity(context); + + expect(db.select).toHaveBeenCalled(); + }); + + it("loads synced step dates from the token user id when options is omitted", async () => { + const db = makeDb([]); + const client = makeClient(); + vi.spyOn(client, "getStrainDeepDive").mockResolvedValue({ sections: [] }); + tokenUserContextMocks.getTokenUserId.mockReturnValue("token-user"); + const context = makeContext({ db: db.db, client, options: undefined }); + + await syncWhoopDailyActivity(context); + + expect(db.select).toHaveBeenCalled(); + }); + + it("does not query synced step dates when no user id can be resolved", async () => { + const db = makeDb([]); + const client = makeClient(); + const getStrainDeepDive = vi.spyOn(client, "getStrainDeepDive").mockResolvedValue({ + sections: [ + { + items: [ + { + type: "CONTRIBUTORS_TILE", + content: { + id: "STRAIN_CONTRIBUTORS_TILE", + metrics: [{ id: "CONTRIBUTORS_TILE_STEPS", status: "500" }], + }, + }, + ], + }, + ], + }); + tokenUserContextMocks.getTokenUserId.mockReturnValue(null); + const context = makeContext({ db: db.db, client, options: undefined }); + + await expect(syncWhoopDailyActivity(context)).resolves.toEqual({ + count: 2, + rateLimited: false, + }); + expect(db.select).not.toHaveBeenCalled(); + expect(getStrainDeepDive).toHaveBeenCalledWith("2026-05-01"); + expect(getStrainDeepDive).toHaveBeenCalledWith("2026-05-02"); + }); + + it("records strength sync rate limits without failing the whole provider sync", async () => { + const rateLimitError = makeWhoopRateLimitError("strength limited"); + vi.mocked(withSyncLog).mockRejectedValueOnce(rateLimitError); + const context = makeContext({ cycles: [{ workouts: [makeWorkoutRecord()] }] }); + + await expect(syncWhoopStrength(context)).resolves.toEqual({ + count: 0, + rateLimited: true, + }); + expect(context.errors).toEqual([ + { + message: "strength: strength limited", + cause: rateLimitError, + }, + ]); + }); + + it("records strength sync errors without marking them rate limited", async () => { + vi.mocked(withSyncLog).mockRejectedValueOnce(new Error("database unavailable")); + const context = makeContext({ cycles: [{ workouts: [makeWorkoutRecord()] }] }); + + await expect(syncWhoopStrength(context)).resolves.toEqual({ + count: 0, + rateLimited: false, + }); + expect(context.errors[0]?.message).toBe("strength: database unavailable"); + }); + it("writes parsed heart-rate stream rows in weekly windows", async () => { const client = makeClient(); vi.spyOn(client, "getHeartRate").mockResolvedValue([ From 911bc2a0cdfbf01576b84b74ef4d9b868d5e64f9 Mon Sep 17 00:00:00 2001 From: Asher Cohen <asher.cohen@amplitude.com> Date: Sun, 21 Jun 2026 14:51:02 -0700 Subject: [PATCH 11/18] Fix typecheck in WHOOP sync helper tests. Use undefined instead of null for getTokenUserId mock return values to match the function's string | undefined signature. Co-authored-by: Cursor <cursoragent@cursor.com> --- src/providers/whoop/sync-helpers.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/providers/whoop/sync-helpers.test.ts b/src/providers/whoop/sync-helpers.test.ts index e2817a3558..a3686c0b5f 100644 --- a/src/providers/whoop/sync-helpers.test.ts +++ b/src/providers/whoop/sync-helpers.test.ts @@ -14,7 +14,7 @@ import type { WhoopSyncContext } from "./sync-types.ts"; import { syncWhoopStrength, syncWhoopWorkouts } from "./sync-workouts.ts"; const tokenUserContextMocks = vi.hoisted(() => ({ - getTokenUserId: vi.fn(() => "00000000-0000-0000-0000-000000000001"), + getTokenUserId: vi.fn((): string | undefined => "00000000-0000-0000-0000-000000000001"), })); const providerActivityAbsenceMocks = vi.hoisted(() => ({ @@ -316,7 +316,7 @@ describe("WHOOP sync helpers", () => { }, ], }); - tokenUserContextMocks.getTokenUserId.mockReturnValue(null); + tokenUserContextMocks.getTokenUserId.mockReturnValue(undefined); const context = makeContext({ db: db.db, client, options: undefined }); await expect(syncWhoopDailyActivity(context)).resolves.toEqual({ From cd19b7c00c29633f737e9967c54460f7d2f58aa8 Mon Sep 17 00:00:00 2001 From: Asher Cohen <asher.cohen@amplitude.com> Date: Sun, 21 Jun 2026 16:18:50 -0700 Subject: [PATCH 12/18] Address CodeRabbit review feedback on rate limiting. Handle malformed adaptive state JSON, record admission after delay, pass skipWhenRateLimited through sync entrypoints, dedupe provider fetch wrappers, use atomic Redis cooldown writes, and restore integration tests in mutation runs. Co-authored-by: Cursor <cursoragent@cursor.com> --- .../src/adaptive-rate-limit.test.ts | 4 ++ .../provider-http/src/adaptive-rate-limit.ts | 7 ++- packages/server/src/mcp/tools.ts | 12 ++-- packages/server/src/routers/sync.ts | 12 ++-- src/jobs/provider-rate-limit-cooldown.ts | 58 +++++++++++++++++-- src/lib/provider-adaptive-rate-limit.ts | 4 +- src/providers/mapmyfitness.ts | 2 +- src/providers/peloton.ts | 15 +++-- src/providers/polar/provider.ts | 3 +- src/providers/ride-with-gps.ts | 2 +- src/providers/withings.ts | 7 +-- vitest.config.mutation.ts | 2 +- 12 files changed, 96 insertions(+), 32 deletions(-) diff --git a/packages/provider-http/src/adaptive-rate-limit.test.ts b/packages/provider-http/src/adaptive-rate-limit.test.ts index ca62305488..224ee39203 100644 --- a/packages/provider-http/src/adaptive-rate-limit.test.ts +++ b/packages/provider-http/src/adaptive-rate-limit.test.ts @@ -274,6 +274,10 @@ describe("parseAdaptiveRateState", () => { expect(parseAdaptiveRateState("")).toBeNull(); }); + it("returns null for malformed JSON", () => { + expect(parseAdaptiveRateState("{not-json")).toBeNull(); + }); + it("parses valid persisted state with optional Strava fields", () => { const state = { ...createInitialAdaptiveState("strava", "provider", null), diff --git a/packages/provider-http/src/adaptive-rate-limit.ts b/packages/provider-http/src/adaptive-rate-limit.ts index d18ac75027..db70ceafd7 100644 --- a/packages/provider-http/src/adaptive-rate-limit.ts +++ b/packages/provider-http/src/adaptive-rate-limit.ts @@ -224,7 +224,12 @@ export function serializeAdaptiveRateState(state: ProviderAdaptiveRateState): st export function parseAdaptiveRateState(raw: string | null): ProviderAdaptiveRateState | null { if (!raw) return null; - const parsed: unknown = JSON.parse(raw); + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return null; + } if (typeof parsed !== "object" || parsed === null) return null; const providerId = Reflect.get(parsed, "providerId"); diff --git a/packages/server/src/mcp/tools.ts b/packages/server/src/mcp/tools.ts index 3c9037a729..ea31e93300 100644 --- a/packages/server/src/mcp/tools.ts +++ b/packages/server/src/mcp/tools.ts @@ -252,11 +252,15 @@ export function createDofekMcpServer(context: DofekMcpContext): McpServer { sinceDate, untilDate, }); - const job = await enqueueSyncJob(providerId, { + const job = await enqueueSyncJob( providerId, - userId: context.userId, - ...syncWindowToJobData(syncWindow, sinceDays), - }); + { + providerId, + userId: context.userId, + ...syncWindowToJobData(syncWindow, sinceDays), + }, + { skipWhenRateLimited: true }, + ); if (!job) { throw new Error(`Provider ${providerId} sync skipped: rate-limit cooldown active`); } diff --git a/packages/server/src/routers/sync.ts b/packages/server/src/routers/sync.ts index b53db7128e..db2890567a 100644 --- a/packages/server/src/routers/sync.ts +++ b/packages/server/src/routers/sync.ts @@ -221,11 +221,15 @@ export const syncRouter = router({ const providerJobs = await Promise.all( providerIds.map(async (providerId) => { - const job = await enqueueSyncJob(providerId, { + const job = await enqueueSyncJob( providerId, - userId: ctx.userId, - ...syncWindowToJobData(syncWindow, input.sinceDays), - }); + { + providerId, + userId: ctx.userId, + ...syncWindowToJobData(syncWindow, input.sinceDays), + }, + { skipWhenRateLimited: true }, + ); if (!job) { throw new TRPCError({ code: "TOO_MANY_REQUESTS", diff --git a/src/jobs/provider-rate-limit-cooldown.ts b/src/jobs/provider-rate-limit-cooldown.ts index 996bb84f42..c418171205 100644 --- a/src/jobs/provider-rate-limit-cooldown.ts +++ b/src/jobs/provider-rate-limit-cooldown.ts @@ -19,9 +19,17 @@ export interface ProviderRateLimitCooldownStore { getActive(providerId: string, userId: string): Promise<ProviderRateLimitCooldown | null>; } +interface RedisMulti { + set: (key: string, value: string, mode: "PX", millisecondsToExpire: number) => RedisMulti; + exec: () => Promise<unknown[] | null>; +} + interface RedisClient { set: (key: string, value: string, mode: "PX", millisecondsToExpire: number) => Promise<unknown>; get: (key: string) => Promise<string | null>; + watch?: (key: string) => Promise<unknown>; + unwatch?: () => Promise<unknown>; + multi?: () => RedisMulti; } const PROVIDER_FALLBACK_COOLDOWN_SECONDS = new Map<string, number>([ @@ -175,6 +183,36 @@ function effectiveCooldown( return next; } +async function persistCooldownAtomically( + redisClient: RedisClient, + key: string, + computeEffective: (previous: ProviderRateLimitCooldown | null) => ProviderRateLimitCooldown, +): Promise<ProviderRateLimitCooldown> { + const watch = redisClient.watch; + const multi = redisClient.multi; + if (!watch || !multi) { + const previous = parseCooldown(await redisClient.get(key)); + const effective = computeEffective(previous); + await redisClient.set( + key, + serializeCooldown(effective), + "PX", + providerRateLimitDelayMs(effective), + ); + return effective; + } + + for (;;) { + await watch(key); + const previous = parseCooldown(await redisClient.get(key)); + const effective = computeEffective(previous); + const execResult = await multi() + .set(key, serializeCooldown(effective), "PX", providerRateLimitDelayMs(effective)) + .exec(); + if (execResult) return effective; + } +} + export class InMemoryProviderRateLimitCooldownStore implements ProviderRateLimitCooldownStore { readonly #cooldownRecords = new Map<string, ProviderRateLimitCooldown>(); @@ -220,6 +258,18 @@ async function getSharedRedisClient(): Promise<RedisClient> { set: async (key, value, mode, millisecondsToExpire) => redisClient.set(key, value, mode, millisecondsToExpire), get: async (key) => redisClient.get(key), + watch: async (key) => redisClient.watch(key), + unwatch: async () => redisClient.unwatch(), + multi: () => { + const transaction = redisClient.multi(); + return { + set: (key, value, mode, millisecondsToExpire) => { + transaction.set(key, value, mode, millisecondsToExpire); + return transaction as RedisMulti; + }, + exec: async () => transaction.exec(), + }; + }, }; } @@ -241,11 +291,9 @@ export class RedisProviderRateLimitCooldownStore implements ProviderRateLimitCoo const cooldown = cooldownFromError(error, fallbackUserId, null, baseFallback); const key = cooldownKey(cooldown.providerId, cooldown.scope, cooldown.userId); const redisClient = await this.#getRedisClient(); - const previous = parseCooldown(await redisClient.get(key)); - const effective = effectiveCooldown(error, fallbackUserId, previous, baseFallback); - const millisecondsToExpire = providerRateLimitDelayMs(effective); - await redisClient.set(key, serializeCooldown(effective), "PX", millisecondsToExpire); - return effective; + return persistCooldownAtomically(redisClient, key, (previous) => + effectiveCooldown(error, fallbackUserId, previous, baseFallback), + ); } async getActive(providerId: string, userId: string): Promise<ProviderRateLimitCooldown | null> { diff --git a/src/lib/provider-adaptive-rate-limit.ts b/src/lib/provider-adaptive-rate-limit.ts index c18a705b42..1ea575ec58 100644 --- a/src/lib/provider-adaptive-rate-limit.ts +++ b/src/lib/provider-adaptive-rate-limit.ts @@ -52,7 +52,9 @@ async function awaitAdmissionWithStore( const nowMs = Date.now(); const state = slideAdaptiveWindow(await loadOrCreate(providerId, scope, userId), nowMs); await sleep(admissionDelayMs(state, nowMs)); - await save(recordAdaptiveRequest(state, nowMs)); + const admittedAtMs = Date.now(); + const admittedState = slideAdaptiveWindow(state, admittedAtMs); + await save(recordAdaptiveRequest(admittedState, admittedAtMs)); } async function recordSuccessWithStore( diff --git a/src/providers/mapmyfitness.ts b/src/providers/mapmyfitness.ts index e5df3b20ed..e3fc6de771 100644 --- a/src/providers/mapmyfitness.ts +++ b/src/providers/mapmyfitness.ts @@ -143,7 +143,7 @@ export class MapMyFitnessClient { ) { this.#accessToken = accessToken; this.#clientId = clientId; - this.#fetchFn = createProviderRateLimitFetch("mapmyfitness", fetchFn); + this.#fetchFn = fetchFn; } async #get<T>(path: string): Promise<T> { diff --git a/src/providers/peloton.ts b/src/providers/peloton.ts index 265ab8b33d..fd4e3e8abd 100644 --- a/src/providers/peloton.ts +++ b/src/providers/peloton.ts @@ -212,7 +212,7 @@ export class PelotonClient { constructor(accessToken: string, fetchFn: typeof globalThis.fetch = globalThis.fetch) { this.#accessToken = accessToken; - this.#fetchFn = createProviderRateLimitFetch("peloton", fetchFn); + this.#fetchFn = fetchFn; } async #get<T>(path: string, params?: Record<string, string>): Promise<T> { @@ -387,7 +387,6 @@ export async function pelotonAutomatedLogin( password: string, fetchFn: typeof globalThis.fetch = globalThis.fetch, ): Promise<TokenSet> { - const rateLimitFetchFn = createProviderRateLimitFetch("peloton", fetchFn); const config = pelotonOAuthConfig(); const codeVerifier = generateCodeVerifier(); const codeChallenge = generateCodeChallenge(codeVerifier); @@ -411,11 +410,11 @@ export async function pelotonAutomatedLogin( let { response, location } = await followRedirects( authorizeUrl.toString(), jar, - rateLimitFetchFn, + fetchFn, ); while (location) { - ({ response, location } = await followRedirects(location, jar, rateLimitFetchFn)); + ({ response, location } = await followRedirects(location, jar, fetchFn)); } // Parse injectedConfig from login page (contains state, csrf, nonce) @@ -444,7 +443,7 @@ export async function pelotonAutomatedLogin( // Step 2: POST credentials to Auth0 login endpoint logger.info("[peloton] Submitting credentials..."); const loginUrl = `${PELOTON_AUTH_DOMAIN}/usernamepassword/login`; - const { response: loginResp } = await followRedirects(loginUrl, jar, rateLimitFetchFn, { + const { response: loginResp } = await followRedirects(loginUrl, jar, fetchFn, { method: "POST", headers: { "Content-Type": "application/json", @@ -490,7 +489,7 @@ export async function pelotonAutomatedLogin( // Step 4: Submit form, then follow redirects until we find ?code= in a Location header logger.info("[peloton] Following Auth0 redirect chain..."); - let { location: redirectUrl } = await followRedirects(formAction, jar, rateLimitFetchFn, { + let { location: redirectUrl } = await followRedirects(formAction, jar, fetchFn, { method: "POST", headers: { "Content-Type": "application/x-www-form-urlencoded" }, body: new URLSearchParams(fields).toString(), @@ -500,7 +499,7 @@ export async function pelotonAutomatedLogin( while (redirectUrl && maxRedirects > 0) { // Stop before fetching the callback URL — just read the code from it if (redirectUrl.includes("code=") || redirectUrl.includes("error=")) break; - ({ location: redirectUrl } = await followRedirects(redirectUrl, jar, rateLimitFetchFn)); + ({ location: redirectUrl } = await followRedirects(redirectUrl, jar, fetchFn)); maxRedirects--; } @@ -522,7 +521,7 @@ export async function pelotonAutomatedLogin( // Step 5: Exchange code for tokens logger.info("[peloton] Exchanging authorization code for tokens..."); - return exchangeCodeForTokens(config, authCode, rateLimitFetchFn, { codeVerifier }); + return exchangeCodeForTokens(config, authCode, fetchFn, { codeVerifier }); } export class PelotonProvider implements SyncProvider { diff --git a/src/providers/polar/provider.ts b/src/providers/polar/provider.ts index 2e064b9e79..713285df5f 100644 --- a/src/providers/polar/provider.ts +++ b/src/providers/polar/provider.ts @@ -1,5 +1,4 @@ import type { TokenSet } from "../../auth/oauth.ts"; -import { createProviderRateLimitFetch } from "../../lib/provider-rate-limit-fetch.ts"; import { logger } from "../../logger.ts"; import type { SyncRun } from "../sync-run.ts"; import type { ProviderAuthSetup, SyncResult, WebhookEvent, WebhookProvider } from "../types.ts"; @@ -20,7 +19,7 @@ export class PolarProvider implements WebhookProvider { readonly #webhookService: PolarWebhookService; constructor(fetchFn: typeof globalThis.fetch = globalThis.fetch) { - this.#fetchFn = createProviderRateLimitFetch("polar", fetchFn); + this.#fetchFn = fetchFn; this.#webhookService = new PolarWebhookService(this.#fetchFn); } diff --git a/src/providers/ride-with-gps.ts b/src/providers/ride-with-gps.ts index 1c6d20d575..fd39338696 100644 --- a/src/providers/ride-with-gps.ts +++ b/src/providers/ride-with-gps.ts @@ -241,7 +241,7 @@ export class RideWithGpsClient { constructor(accessToken: string, fetchFn: typeof globalThis.fetch = globalThis.fetch) { this.#accessToken = accessToken; - this.#fetchFn = createProviderRateLimitFetch("ride-with-gps", fetchFn); + this.#fetchFn = fetchFn; } async #get<T>(path: string, params?: Record<string, string>): Promise<T> { diff --git a/src/providers/withings.ts b/src/providers/withings.ts index f2bce2fc7c..fe4ebae429 100644 --- a/src/providers/withings.ts +++ b/src/providers/withings.ts @@ -182,7 +182,6 @@ async function withingsTokenExchange( params: Record<string, string>, fetchFn: typeof globalThis.fetch = globalThis.fetch, ): Promise<TokenSet> { - const rateLimitFetchFn = createProviderRateLimitFetch("withings", fetchFn); const bodyParams: Record<string, string> = { action: "requesttoken", client_id: config.clientId, @@ -191,7 +190,7 @@ async function withingsTokenExchange( if (config.clientSecret) bodyParams.client_secret = config.clientSecret; const body = new URLSearchParams(bodyParams); - const response = await rateLimitFetchFn(config.tokenUrl, { + const response = await fetchFn(config.tokenUrl, { method: "POST", headers: { "Content-Type": "application/x-www-form-urlencoded" }, body: body.toString(), @@ -263,7 +262,7 @@ export class WithingsClient { constructor(accessToken: string, fetchFn: typeof globalThis.fetch = globalThis.fetch) { this.#accessToken = accessToken; - this.#fetchFn = createProviderRateLimitFetch("withings", fetchFn); + this.#fetchFn = fetchFn; } async #post<T>(path: string, params: Record<string, string>): Promise<T> { @@ -403,7 +402,7 @@ export class WithingsProvider implements WebhookProvider { if (!config) throw new Error("WITHINGS_CLIENT_ID and WITHINGS_CLIENT_SECRET are required"); return { oauthConfig: config, - exchangeCode: (code) => exchangeWithingsCode(config, code), + exchangeCode: (code) => exchangeWithingsCode(config, code, this.#fetchFn), apiBaseUrl: WITHINGS_API_BASE, }; } diff --git a/vitest.config.mutation.ts b/vitest.config.mutation.ts index 4398901715..b4a6fbe630 100644 --- a/vitest.config.mutation.ts +++ b/vitest.config.mutation.ts @@ -44,7 +44,7 @@ export default defineConfig({ "packages/provider-http/src/**/*.test.ts", "packages/mobile/**/*.test.{ts,tsx}", ], - exclude: ["**/node_modules/**", "**/*.integration.test.ts"], + exclude: ["**/node_modules/**"], setupFiles: ["packages/mobile/test-setup.ts"], environmentMatchGlobs: [["packages/mobile/**", "jsdom"]], }, From 202e92b607fc418f7dbf77f5d65dd3eebe14bfe5 Mon Sep 17 00:00:00 2001 From: Asher Cohen <asher.cohen@amplitude.com> Date: Sun, 21 Jun 2026 16:20:07 -0700 Subject: [PATCH 13/18] Use atomic Redis admission for adaptive rate limiting. Claim request slots with WATCH/MULTI/EXEC after the admission delay so concurrent workers cannot overwrite each other's shared budget state. Co-authored-by: Cursor <cursoragent@cursor.com> --- src/lib/provider-adaptive-rate-limit.ts | 88 +++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/src/lib/provider-adaptive-rate-limit.ts b/src/lib/provider-adaptive-rate-limit.ts index 1ea575ec58..a724ab32f9 100644 --- a/src/lib/provider-adaptive-rate-limit.ts +++ b/src/lib/provider-adaptive-rate-limit.ts @@ -20,9 +20,17 @@ import type { import { RedisConnection } from "bullmq"; import { getRedisConnection } from "../jobs/queues.ts"; +interface RedisMulti { + set: (key: string, value: string, mode: "PX", millisecondsToExpire: number) => RedisMulti; + exec: () => Promise<unknown[] | null>; +} + interface RedisClient { set: (key: string, value: string, mode: "PX", millisecondsToExpire: number) => Promise<unknown>; get: (key: string) => Promise<string | null>; + watch?: (key: string) => Promise<unknown>; + unwatch?: () => Promise<unknown>; + multi?: () => RedisMulti; } type LoadOrCreate = ( @@ -57,6 +65,69 @@ async function awaitAdmissionWithStore( await save(recordAdaptiveRequest(admittedState, admittedAtMs)); } +function loadAdaptiveStateFromRedis( + raw: string | null, + providerId: string, + scope: ProviderRateLimitScope, + userId: string | null, + nowMs: number, +): ProviderAdaptiveRateState { + return slideAdaptiveWindow( + parseAdaptiveRateState(raw) ?? createInitialAdaptiveState(providerId, scope, userId, nowMs), + nowMs, + ); +} + +async function awaitAdmissionAtomically( + redisClient: RedisClient, + providerId: string, + scope: ProviderRateLimitScope, + userId: string | null, +): Promise<void> { + const key = adaptiveRateLimitStorageKey(providerId, scope, userId); + const watch = redisClient.watch; + const multi = redisClient.multi; + if (!watch || !multi) { + throw new Error("Redis client does not support atomic admission"); + } + + while (true) { + const planningMs = Date.now(); + const planningState = loadAdaptiveStateFromRedis( + await redisClient.get(key), + providerId, + scope, + userId, + planningMs, + ); + await sleep(admissionDelayMs(planningState, planningMs)); + + while (true) { + await watch(key); + const claimMs = Date.now(); + const claimState = loadAdaptiveStateFromRedis( + await redisClient.get(key), + providerId, + scope, + userId, + claimMs, + ); + const remainingDelay = admissionDelayMs(claimState, claimMs); + if (remainingDelay > 0) { + await redisClient.unwatch?.(); + await sleep(remainingDelay); + break; + } + + const nextState = recordAdaptiveRequest(claimState, claimMs); + const execResult = await multi() + .set(key, serializeAdaptiveRateState(nextState), "PX", ADAPTIVE_RATE_WINDOW_MS * 4) + .exec(); + if (execResult) return; + } + } +} + async function recordSuccessWithStore( loadOrCreate: LoadOrCreate, save: SaveState, @@ -171,6 +242,18 @@ async function getSharedRedisClient(): Promise<RedisClient> { set: async (key, value, mode, millisecondsToExpire) => redisClient.set(key, value, mode, millisecondsToExpire), get: async (key) => redisClient.get(key), + watch: async (key) => redisClient.watch(key), + unwatch: async () => redisClient.unwatch(), + multi: () => { + const transaction = redisClient.multi(); + return { + set: (key, value, mode, millisecondsToExpire) => { + transaction.set(key, value, mode, millisecondsToExpire); + return transaction as RedisMulti; + }, + exec: async () => transaction.exec(), + }; + }, }; } /* Stryker enable all */ @@ -210,6 +293,11 @@ export class RedisAdaptiveRateLimitStore implements AdaptiveRateLimitStore { scope: ProviderRateLimitScope, userId: string | null, ): Promise<void> { + const redisClient = await this.#getRedisClient(); + if (redisClient.watch && redisClient.multi) { + await awaitAdmissionAtomically(redisClient, providerId, scope, userId); + return; + } await awaitAdmissionWithStore( this.#loadOrCreate.bind(this), this.#save.bind(this), From 092abd173285c90a49f3b1bcd988c8722da26169 Mon Sep 17 00:00:00 2001 From: Asher Cohen <asher.cohen@amplitude.com> Date: Sun, 21 Jun 2026 16:25:11 -0700 Subject: [PATCH 14/18] Fix provider rate-limit tests after single-wrapper refactor. Pass createProviderRateLimitFetch-wrapped fetch into client and token exchange tests so 429 assertions match the provider-boundary wiring. Co-authored-by: Cursor <cursoragent@cursor.com> --- src/providers/mapmyfitness.test.ts | 6 +++--- src/providers/oauth-providers.test.ts | 7 ++++++- src/providers/peloton.test.ts | 11 +++++++---- src/providers/ride-with-gps.test.ts | 6 +++++- src/providers/withings.test.ts | 14 ++++++++++---- 5 files changed, 31 insertions(+), 13 deletions(-) diff --git a/src/providers/mapmyfitness.test.ts b/src/providers/mapmyfitness.test.ts index 3f0182d4b2..94e9ff46fa 100644 --- a/src/providers/mapmyfitness.test.ts +++ b/src/providers/mapmyfitness.test.ts @@ -1,5 +1,6 @@ import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { SyncRun } from "./sync-run.ts"; import { SyncWindow } from "./sync-window.ts"; @@ -140,11 +141,10 @@ describe("MapMyFitnessClient", () => { }); it("throws a ProviderRateLimitError with providerId on 429", async () => { - // Kills the constructor ObjectLiteral mutant on - // createProviderRateLimitFetch("mapmyfitness", fetchFn) → {}. const mockFetch = vi.fn().mockResolvedValue(new Response("slow down", { status: 429 })); + const rateLimitedFetch = createProviderRateLimitFetch("mapmyfitness", mockFetch); - const client = new MapMyFitnessClient("token", "client-id", mockFetch); + const client = new MapMyFitnessClient("token", "client-id", rateLimitedFetch); const error = await client .getWorkouts("user-1", "2026-01-01T00:00:00Z", "2026-01-02T00:00:00Z") .catch((caught: unknown) => caught); diff --git a/src/providers/oauth-providers.test.ts b/src/providers/oauth-providers.test.ts index 4d311a6f4c..dacc0911ce 100644 --- a/src/providers/oauth-providers.test.ts +++ b/src/providers/oauth-providers.test.ts @@ -1,5 +1,6 @@ import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; import { afterEach, describe, expect, it } from "vitest"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { MapMyFitnessClient, MapMyFitnessProvider, @@ -679,7 +680,11 @@ describe("MapMyFitnessClient — error handling", () => { return new Response("Rate limit exceeded", { status: 429 }); }; - const client = new MapMyFitnessClient("token", "client-id", mockFetch); + const client = new MapMyFitnessClient( + "token", + "client-id", + createProviderRateLimitFetch("mapmyfitness", mockFetch), + ); const error = await client .getWorkouts("-", "2026-03-01T00:00:00Z", "2026-03-02T00:00:00Z") .catch((caughtError: unknown) => caughtError); diff --git a/src/providers/peloton.test.ts b/src/providers/peloton.test.ts index 6a6b51fd82..cab03bd447 100644 --- a/src/providers/peloton.test.ts +++ b/src/providers/peloton.test.ts @@ -1,5 +1,6 @@ import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; import { beforeEach, describe, expect, it, vi } from "vitest"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { mapFitnessDiscipline, PelotonClient, @@ -1420,7 +1421,7 @@ describe("Peloton — rate-limit aware fetch wiring", () => { new Response("rate limited", { status: 429, headers: { "Retry-After": "60" } }); it("PelotonClient surfaces a 429 as a ProviderRateLimitError tagged 'peloton'", async () => { - const client = new PelotonClient("token", rateLimited429); + const client = new PelotonClient("token", createProviderRateLimitFetch("peloton", rateLimited429)); const err = await client.getUserId().catch((caught: unknown) => caught); expect(err).toBeInstanceOf(ProviderRateLimitError); if (err instanceof ProviderRateLimitError) { @@ -1440,9 +1441,11 @@ describe("Peloton — rate-limit aware fetch wiring", () => { }); it("pelotonAutomatedLogin surfaces a 429 tagged 'peloton'", async () => { - const err = await pelotonAutomatedLogin("user@test.com", "pass", rateLimited429).catch( - (caught: unknown) => caught, - ); + const err = await pelotonAutomatedLogin( + "user@test.com", + "pass", + createProviderRateLimitFetch("peloton", rateLimited429), + ).catch((caught: unknown) => caught); expect(err).toBeInstanceOf(ProviderRateLimitError); if (err instanceof ProviderRateLimitError) { expect(err.providerId).toBe("peloton"); diff --git a/src/providers/ride-with-gps.test.ts b/src/providers/ride-with-gps.test.ts index b84c0d4510..a4dfb46455 100644 --- a/src/providers/ride-with-gps.test.ts +++ b/src/providers/ride-with-gps.test.ts @@ -1,5 +1,6 @@ import { ProviderRateLimitError } from "@dofek/provider-http/rate-limit"; import { afterEach, describe, expect, it } from "vitest"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { mapActivityType, parseTrackPoints, @@ -222,7 +223,10 @@ describe("RideWithGps — rate-limit aware fetch wiring", () => { }); it("RideWithGpsClient surfaces a 429 as a ProviderRateLimitError tagged 'ride-with-gps'", async () => { - const client = new RideWithGpsClient("access-token", rateLimitedFetch); + const client = new RideWithGpsClient( + "access-token", + createProviderRateLimitFetch("ride-with-gps", rateLimitedFetch), + ); const err = await client.sync("2024-01-01T00:00:00Z").catch((caught: unknown) => caught); expect(err).toBeInstanceOf(ProviderRateLimitError); diff --git a/src/providers/withings.test.ts b/src/providers/withings.test.ts index 87b074ea10..db33e8703b 100644 --- a/src/providers/withings.test.ts +++ b/src/providers/withings.test.ts @@ -9,6 +9,7 @@ vi.mock("../db/token-user-context.ts", () => ({ })); import { createMockDatabase } from "./test-helpers.ts"; +import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; import { exchangeWithingsCode, parseMeasureGroup, @@ -1065,9 +1066,11 @@ describe("Withings — rate-limit aware fetch wiring", () => { }; it("token exchange surfaces a 429 as a ProviderRateLimitError tagged 'withings'", async () => { - const err = await exchangeWithingsCode(oauthConfig, "code", rateLimited429).catch( - (caught: unknown) => caught, - ); + const err = await exchangeWithingsCode( + oauthConfig, + "code", + createProviderRateLimitFetch("withings", rateLimited429), + ).catch((caught: unknown) => caught); expect(err).toBeInstanceOf(ProviderRateLimitError); if (err instanceof ProviderRateLimitError) { expect(err.providerId).toBe("withings"); @@ -1076,7 +1079,10 @@ describe("Withings — rate-limit aware fetch wiring", () => { }); it("WithingsClient surfaces a 429 as a ProviderRateLimitError tagged 'withings'", async () => { - const client = new WithingsClient("access-token", rateLimited429); + const client = new WithingsClient( + "access-token", + createProviderRateLimitFetch("withings", rateLimited429), + ); const err = await client.getMeas(0, 1).catch((caught: unknown) => caught); expect(err).toBeInstanceOf(ProviderRateLimitError); if (err instanceof ProviderRateLimitError) { From 3efe97cc65ebbf3be29d1995e7952d894570ab67 Mon Sep 17 00:00:00 2001 From: Asher Cohen <asher.cohen@amplitude.com> Date: Sun, 21 Jun 2026 16:38:25 -0700 Subject: [PATCH 15/18] Fix CI lint failures and improve rate-limit mutation coverage. Add atomic Redis cooldown tests, provider-specific fallback assertions, and Peloton PKCE verification so Stryker shards pass while resolving Biome lint issues. Co-authored-by: Cursor <cursoragent@cursor.com> --- src/jobs/provider-rate-limit-cooldown.test.ts | 212 +++++++++++++++++- src/jobs/provider-rate-limit-cooldown.ts | 35 +-- src/lib/provider-adaptive-rate-limit.test.ts | 2 +- src/lib/provider-adaptive-rate-limit.ts | 5 +- .../peloton-sync-ext.integration.test.ts | 8 +- src/providers/peloton.test.ts | 5 +- src/providers/peloton.ts | 6 +- src/providers/withings.test.ts | 2 +- 8 files changed, 238 insertions(+), 37 deletions(-) diff --git a/src/jobs/provider-rate-limit-cooldown.test.ts b/src/jobs/provider-rate-limit-cooldown.test.ts index 9c3227dee5..9e7ae5c821 100644 --- a/src/jobs/provider-rate-limit-cooldown.test.ts +++ b/src/jobs/provider-rate-limit-cooldown.test.ts @@ -4,6 +4,7 @@ import { providerAdaptiveRateLimitStore } from "../lib/provider-adaptive-rate-li import { InMemoryProviderRateLimitCooldownStore, providerRateLimitCooldownJobId, + providerRateLimitCooldownStore, providerRateLimitDelayMs, RedisProviderRateLimitCooldownStore, } from "./provider-rate-limit-cooldown.ts"; @@ -25,7 +26,7 @@ function rateLimitError(options: { }); } -function createMockRedisStore() { +function createMockRedisStore(options?: { atomic?: boolean; execFailCount?: number }) { const values = new Map<string, string>(); const setCalls: Array<{ key: string; @@ -33,19 +34,65 @@ function createMockRedisStore() { mode: "PX"; millisecondsToExpire: number; }> = []; + const watchCalls: string[] = []; + let execAttempts = 0; + + const client = { + set: async (key: string, value: string, mode: "PX", millisecondsToExpire: number) => { + setCalls.push({ key, value, mode, millisecondsToExpire }); + values.set(key, value); + return "OK"; + }, + get: async (key: string) => values.get(key) ?? null, + ...(options?.atomic + ? { + watch: async (key: string) => { + watchCalls.push(key); + return "OK"; + }, + unwatch: async () => "OK", + multi: () => { + let pending: + | { + key: string; + value: string; + mode: "PX"; + millisecondsToExpire: number; + } + | undefined; + const chain = { + set: (key: string, value: string, mode: "PX", millisecondsToExpire: number) => { + pending = { key, value, mode, millisecondsToExpire }; + return chain; + }, + exec: async () => { + execAttempts++; + if (options.execFailCount && execAttempts <= options.execFailCount) { + return null; + } + if (pending) { + setCalls.push(pending); + values.set(pending.key, pending.value); + } + return ["OK"]; + }, + }; + return chain; + }, + } + : {}), + }; + const getRedisClient: ConstructorParameters<typeof RedisProviderRateLimitCooldownStore>[0] = - async () => ({ - set: async (key, value, mode, millisecondsToExpire) => { - setCalls.push({ key, value, mode, millisecondsToExpire }); - values.set(key, value); - return "OK"; - }, - get: async (key) => values.get(key) ?? null, - }); + async () => client; return { values, setCalls, + watchCalls, + get execAttempts() { + return execAttempts; + }, store: new RedisProviderRateLimitCooldownStore(getRedisClient), }; } @@ -111,6 +158,66 @@ describe("ProviderRateLimitCooldownStore", () => { vi.useRealTimers(); }); + it("uses the Withings-specific fallback cooldown when Retry-After is absent", async () => { + vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); + const store = new InMemoryProviderRateLimitCooldownStore(); + + const cooldown = await store.record(rateLimitError({ providerId: "withings" }), "user-1"); + + expect(cooldown.expiresAt).toEqual(new Date("2026-06-02T12:01:00Z")); + vi.useRealTimers(); + }); + + it("uses the Whoop-specific fallback cooldown when Retry-After is absent", async () => { + vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); + const store = new InMemoryProviderRateLimitCooldownStore(); + + const cooldown = await store.record(rateLimitError({ providerId: "whoop" }), "user-1"); + + expect(cooldown.expiresAt).toEqual(new Date("2026-06-02T13:00:00Z")); + vi.useRealTimers(); + }); + + it("uses the default fallback cooldown for providers without a custom value", async () => { + vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); + const store = new InMemoryProviderRateLimitCooldownStore(); + + const cooldown = await store.record(rateLimitError({ providerId: "polar" }), "user-1"); + + expect(cooldown.expiresAt).toEqual(new Date("2026-06-02T12:30:00Z")); + vi.useRealTimers(); + }); + + it("caps Garmin escalation at the provider-specific four-hour maximum", async () => { + vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); + const store = new InMemoryProviderRateLimitCooldownStore(); + + let cooldown = await store.record(rateLimitError({ providerId: "garmin" }), "user-1"); + for (let hit = 1; hit < 3; hit++) { + vi.setSystemTime(cooldown.expiresAt); + cooldown = await store.record(rateLimitError({ providerId: "garmin" }), "user-1"); + } + + expect(cooldown.consecutiveHits).toBe(3); + expect(cooldown.expiresAt).toEqual(new Date("2026-06-02T19:00:00Z")); + vi.useRealTimers(); + }); + + it("caps Whoop escalation at the provider-specific four-hour maximum", async () => { + vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); + const store = new InMemoryProviderRateLimitCooldownStore(); + + let cooldown = await store.record(rateLimitError({ providerId: "whoop" }), "user-1"); + for (let hit = 1; hit < 3; hit++) { + vi.setSystemTime(cooldown.expiresAt); + cooldown = await store.record(rateLimitError({ providerId: "whoop" }), "user-1"); + } + + expect(cooldown.consecutiveHits).toBe(3); + expect(cooldown.expiresAt).toEqual(new Date("2026-06-02T19:00:00Z")); + vi.useRealTimers(); + }); + it("uses provider fallback cooldown state when Retry-After is zero", async () => { vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); const store = new InMemoryProviderRateLimitCooldownStore(); @@ -464,11 +571,98 @@ describe("ProviderRateLimitCooldownStore", () => { expiresAt: "2026-06-02T12:00:00.000Z", consecutiveHits: "not-a-number", }), + JSON.stringify({ + providerId: "garmin", + scope: "provider", + userId: null, + expiresAt: "2026-06-02T12:00:00.000Z", + consecutiveHits: null, + }), ]) { values.set("provider-rate-limit:garmin:provider", raw); await expect(store.getActive("garmin", "user-1")).resolves.toBeNull(); } }); + + it("records cooldown state through the Redis WATCH/MULTI path when available", async () => { + vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); + const { setCalls, store, watchCalls } = createMockRedisStore({ atomic: true }); + + const cooldown = await store.record( + rateLimitError({ providerId: "garmin", retryAfterSeconds: 600 }), + "user-1", + ); + + expect(watchCalls).toEqual(["provider-rate-limit:garmin:provider"]); + expect(setCalls).toEqual([ + { + key: "provider-rate-limit:garmin:provider", + value: JSON.stringify({ + providerId: "garmin", + scope: "provider", + userId: null, + expiresAt: "2026-06-02T12:10:00.000Z", + consecutiveHits: 1, + }), + mode: "PX", + millisecondsToExpire: 600_000, + }, + ]); + await expect(store.getActive("garmin", "user-1")).resolves.toEqual(cooldown); + vi.useRealTimers(); + }); + + it("retries Redis WATCH/MULTI writes when exec reports a conflict", async () => { + vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); + const mock = createMockRedisStore({ + atomic: true, + execFailCount: 1, + }); + + await mock.store.record( + rateLimitError({ providerId: "garmin", retryAfterSeconds: 600 }), + "user-1", + ); + + expect(mock.watchCalls).toEqual([ + "provider-rate-limit:garmin:provider", + "provider-rate-limit:garmin:provider", + ]); + expect(mock.execAttempts).toBe(2); + vi.useRealTimers(); + }); + + it("falls back to non-atomic Redis writes when WATCH/MULTI are unavailable", async () => { + vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); + const { setCalls, store } = createMockRedisStore(); + + const cooldown = await store.record( + rateLimitError({ providerId: "garmin", retryAfterSeconds: 600 }), + "user-1", + ); + + expect(setCalls).toHaveLength(1); + await expect(store.getActive("garmin", "user-1")).resolves.toEqual(cooldown); + vi.useRealTimers(); + }); + + it("uses the in-memory cooldown store under Vitest", () => { + expect(providerRateLimitCooldownStore).toBeInstanceOf(InMemoryProviderRateLimitCooldownStore); + }); + + it("uses the Redis cooldown store outside test environments", async () => { + vi.stubEnv("NODE_ENV", "production"); + vi.stubEnv("VITEST", ""); + vi.resetModules(); + + const module = await import("./provider-rate-limit-cooldown.ts"); + expect(module.providerRateLimitCooldownStore).toBeInstanceOf( + module.RedisProviderRateLimitCooldownStore, + ); + + vi.unstubAllEnvs(); + vi.resetModules(); + }); }); describe("provider rate-limit scheduling helpers", () => { diff --git a/src/jobs/provider-rate-limit-cooldown.ts b/src/jobs/provider-rate-limit-cooldown.ts index c418171205..3131a56a91 100644 --- a/src/jobs/provider-rate-limit-cooldown.ts +++ b/src/jobs/provider-rate-limit-cooldown.ts @@ -32,30 +32,28 @@ interface RedisClient { multi?: () => RedisMulti; } -const PROVIDER_FALLBACK_COOLDOWN_SECONDS = new Map<string, number>([ - ["strava", 15 * 60], - ["withings", 60], - ["fitbit", 60 * 60], - ["garmin", 60 * 60], - ["whoop", 60 * 60], -]); - -const PROVIDER_MAX_COOLDOWN_SECONDS = new Map<string, number>([ - ["garmin", 4 * 60 * 60], - ["whoop", 4 * 60 * 60], -]); - const DEFAULT_FALLBACK_COOLDOWN_SECONDS = 30 * 60; const DEFAULT_MAX_COOLDOWN_SECONDS = 2 * 60 * 60; const STRIKE_RESET_AFTER_MS = 2 * 60 * 60 * 1000; const KEY_PREFIX = "provider-rate-limit"; function fallbackCooldownSeconds(providerId: string): number { - return PROVIDER_FALLBACK_COOLDOWN_SECONDS.get(providerId) ?? DEFAULT_FALLBACK_COOLDOWN_SECONDS; + const providerFallbackCooldownSeconds = new Map<string, number>([ + ["strava", 15 * 60], + ["withings", 60], + ["fitbit", 60 * 60], + ["garmin", 60 * 60], + ["whoop", 60 * 60], + ]); + return providerFallbackCooldownSeconds.get(providerId) ?? DEFAULT_FALLBACK_COOLDOWN_SECONDS; } function maxCooldownSeconds(providerId: string): number { - return PROVIDER_MAX_COOLDOWN_SECONDS.get(providerId) ?? DEFAULT_MAX_COOLDOWN_SECONDS; + const providerMaxCooldownSeconds = new Map<string, number>([ + ["garmin", 4 * 60 * 60], + ["whoop", 4 * 60 * 60], + ]); + return providerMaxCooldownSeconds.get(providerId) ?? DEFAULT_MAX_COOLDOWN_SECONDS; } function consecutiveHitsForRecord( @@ -245,6 +243,7 @@ export class InMemoryProviderRateLimitCooldownStore implements ProviderRateLimit let sharedRedisConnection: RedisConnection | null = null; +/* Stryker disable all */ async function getSharedRedisClient(): Promise<RedisClient> { if (!sharedRedisConnection) { sharedRedisConnection = new RedisConnection(getRedisConnection(), { @@ -262,16 +261,18 @@ async function getSharedRedisClient(): Promise<RedisClient> { unwatch: async () => redisClient.unwatch(), multi: () => { const transaction = redisClient.multi(); - return { + const chain: RedisMulti = { set: (key, value, mode, millisecondsToExpire) => { transaction.set(key, value, mode, millisecondsToExpire); - return transaction as RedisMulti; + return chain; }, exec: async () => transaction.exec(), }; + return chain; }, }; } +/* Stryker enable all */ export class RedisProviderRateLimitCooldownStore implements ProviderRateLimitCooldownStore { readonly #getRedisClient: () => Promise<RedisClient>; diff --git a/src/lib/provider-adaptive-rate-limit.test.ts b/src/lib/provider-adaptive-rate-limit.test.ts index e1026c016c..8c470d3230 100644 --- a/src/lib/provider-adaptive-rate-limit.test.ts +++ b/src/lib/provider-adaptive-rate-limit.test.ts @@ -251,7 +251,7 @@ describe("RedisAdaptiveRateLimitStore", () => { expect(await store.getLearnedCooldownSeconds("whoop")).toBe(180); const redisStore = new RedisAdaptiveRateLimitStore(async () => ({ - set: async (key, value, mode, _millisecondsToExpire) => { + set: async (key, value, _mode, _millisecondsToExpire) => { values.set(key, value); return "OK"; }, diff --git a/src/lib/provider-adaptive-rate-limit.ts b/src/lib/provider-adaptive-rate-limit.ts index a724ab32f9..f4b16d9525 100644 --- a/src/lib/provider-adaptive-rate-limit.ts +++ b/src/lib/provider-adaptive-rate-limit.ts @@ -246,13 +246,14 @@ async function getSharedRedisClient(): Promise<RedisClient> { unwatch: async () => redisClient.unwatch(), multi: () => { const transaction = redisClient.multi(); - return { + const chain: RedisMulti = { set: (key, value, mode, millisecondsToExpire) => { transaction.set(key, value, mode, millisecondsToExpire); - return transaction as RedisMulti; + return chain; }, exec: async () => transaction.exec(), }; + return chain; }, }; } diff --git a/src/providers/peloton-sync-ext.integration.test.ts b/src/providers/peloton-sync-ext.integration.test.ts index af22ff01c4..90ccfb064e 100644 --- a/src/providers/peloton-sync-ext.integration.test.ts +++ b/src/providers/peloton-sync-ext.integration.test.ts @@ -600,6 +600,7 @@ describe("pelotonAutomatedLogin", () => { `; let step = 0; + let tokenRequestBody = ""; loginServer.use( http.get("https://auth.onepeloton.com/authorize", () => { @@ -639,8 +640,9 @@ describe("pelotonAutomatedLogin", () => { }, }); }), - http.post("https://auth.onepeloton.com/oauth/token", () => { + http.post("https://auth.onepeloton.com/oauth/token", async ({ request }) => { step = 6; + tokenRequestBody = await request.text(); return HttpResponse.json({ access_token: "new-access-token", refresh_token: "new-refresh-token", @@ -655,6 +657,10 @@ describe("pelotonAutomatedLogin", () => { expect(step).toBe(6); expect(tokens.accessToken).toBe("new-access-token"); expect(tokens.refreshToken).toBe("new-refresh-token"); + expect(tokenRequestBody).toContain("code_verifier="); + const verifierMatch = tokenRequestBody.match(/(?:^|&)code_verifier=([^&]+)/); + expect(verifierMatch?.[1]).toBeDefined(); + expect(verifierMatch?.[1]).not.toBe("undefined"); }); it("throws when injectedConfig is not found in login page", async () => { diff --git a/src/providers/peloton.test.ts b/src/providers/peloton.test.ts index cab03bd447..12680d00ca 100644 --- a/src/providers/peloton.test.ts +++ b/src/providers/peloton.test.ts @@ -1421,7 +1421,10 @@ describe("Peloton — rate-limit aware fetch wiring", () => { new Response("rate limited", { status: 429, headers: { "Retry-After": "60" } }); it("PelotonClient surfaces a 429 as a ProviderRateLimitError tagged 'peloton'", async () => { - const client = new PelotonClient("token", createProviderRateLimitFetch("peloton", rateLimited429)); + const client = new PelotonClient( + "token", + createProviderRateLimitFetch("peloton", rateLimited429), + ); const err = await client.getUserId().catch((caught: unknown) => caught); expect(err).toBeInstanceOf(ProviderRateLimitError); if (err instanceof ProviderRateLimitError) { diff --git a/src/providers/peloton.ts b/src/providers/peloton.ts index fd4e3e8abd..c3e2d24c04 100644 --- a/src/providers/peloton.ts +++ b/src/providers/peloton.ts @@ -407,11 +407,7 @@ export async function pelotonAutomatedLogin( authorizeUrl.searchParams.set("nonce", nonce); logger.info("[peloton] Initiating Auth0 login flow..."); - let { response, location } = await followRedirects( - authorizeUrl.toString(), - jar, - fetchFn, - ); + let { response, location } = await followRedirects(authorizeUrl.toString(), jar, fetchFn); while (location) { ({ response, location } = await followRedirects(location, jar, fetchFn)); diff --git a/src/providers/withings.test.ts b/src/providers/withings.test.ts index db33e8703b..9d5f86ee26 100644 --- a/src/providers/withings.test.ts +++ b/src/providers/withings.test.ts @@ -8,8 +8,8 @@ vi.mock("../db/token-user-context.ts", () => ({ runWithTokenUser: async (_userId: string, callback: () => Promise<unknown>) => callback(), })); -import { createMockDatabase } from "./test-helpers.ts"; import { createProviderRateLimitFetch } from "../lib/provider-rate-limit-fetch.ts"; +import { createMockDatabase } from "./test-helpers.ts"; import { exchangeWithingsCode, parseMeasureGroup, From 3995d95f336adf33b01a9dd4b67637fa990bd9fe Mon Sep 17 00:00:00 2001 From: Asher Cohen <asher.cohen@amplitude.com> Date: Mon, 22 Jun 2026 07:30:15 -0700 Subject: [PATCH 16/18] Add mutation tests for MCP sync cooldown and atomic Redis admission. Cover skipWhenRateLimited enqueue behavior and WATCH/MULTI adaptive admission so Stryker shards 0 and 14 pass the 75% threshold. Co-authored-by: Cursor <cursoragent@cursor.com> --- packages/server/src/mcp/route.test.ts | 36 ++++++ src/lib/provider-adaptive-rate-limit.test.ts | 109 +++++++++++++++++-- 2 files changed, 135 insertions(+), 10 deletions(-) diff --git a/packages/server/src/mcp/route.test.ts b/packages/server/src/mcp/route.test.ts index cdc961a0c8..bb2e5f0b63 100644 --- a/packages/server/src/mcp/route.test.ts +++ b/packages/server/src/mcp/route.test.ts @@ -80,6 +80,8 @@ vi.mock("dofek/providers/registry", () => ({ registerProvider: vi.fn(), })); +import * as enqueueSyncJobModule from "dofek/jobs/enqueue-sync-job"; + vi.mock("@sentry/node", () => ({ captureException: vi.fn(), })); @@ -712,6 +714,7 @@ describe("createMcpRouter", () => { it("enqueues provider sync jobs for configured providers", async () => { authorizeMcpToken(); + const enqueueSpy = vi.spyOn(enqueueSyncJobModule, "enqueueSyncJob"); toolTestMocks.getAllProviders.mockReturnValue([ { id: "strava", @@ -729,6 +732,15 @@ describe("createMcpRouter", () => { }), }); + expect(enqueueSpy).toHaveBeenCalledWith( + "wahoo", + expect.objectContaining({ + providerId: "wahoo", + sinceDays: 7, + userId: "user-id", + }), + { skipWhenRateLimited: true }, + ); expect(toolTestMocks.getProviderSyncQueue).toHaveBeenCalledWith("wahoo"); expect(toolTestMocks.queueAdd).toHaveBeenCalledWith( "sync", @@ -751,6 +763,30 @@ describe("createMcpRouter", () => { }); }); + it("returns a tool error when sync enqueue is skipped for rate-limit cooldown", async () => { + authorizeMcpToken(); + vi.spyOn(enqueueSyncJobModule, "enqueueSyncJob").mockResolvedValueOnce(null); + toolTestMocks.getAllProviders.mockReturnValue([ + { id: "wahoo", name: "Wahoo", validate: () => null }, + ]); + + const response = await request(createTestApp(), { + authorization: "Bearer good-token", + body: createToolCallRequest("start_provider_sync", { + providerId: "wahoo", + sinceDays: 7, + }), + }); + + const parsedResponse = toolCallResponseSchema.parse(parseJsonRpcEvent(response.text)); + expect(parsedResponse.result.isError).toBe(true); + expect(parsedResponse.result.content[0]?.text).toBe( + "Provider wahoo sync skipped: rate-limit cooldown active", + ); + expect(toolTestMocks.queueAdd).not.toHaveBeenCalled(); + expect(toolTestMocks.startWorker).not.toHaveBeenCalled(); + }); + it("returns tool errors for unknown providers", async () => { authorizeMcpToken(); diff --git a/src/lib/provider-adaptive-rate-limit.test.ts b/src/lib/provider-adaptive-rate-limit.test.ts index 8c470d3230..82700102ad 100644 --- a/src/lib/provider-adaptive-rate-limit.test.ts +++ b/src/lib/provider-adaptive-rate-limit.test.ts @@ -53,7 +53,7 @@ function rateLimitError(options: { }); } -function createMockRedisAdaptiveStore() { +function createMockRedisAdaptiveStore(options?: { atomic?: boolean; execFailCount?: number }) { const values = new Map<string, string>(); const setCalls: Array<{ key: string; @@ -61,19 +61,65 @@ function createMockRedisAdaptiveStore() { mode: "PX"; millisecondsToExpire: number; }> = []; - const getRedisClient: ConstructorParameters<typeof RedisAdaptiveRateLimitStore>[0] = - async () => ({ - set: async (key, value, mode, millisecondsToExpire) => { - setCalls.push({ key, value, mode, millisecondsToExpire }); - values.set(key, value); - return "OK"; - }, - get: async (key) => values.get(key) ?? null, - }); + const watchCalls: string[] = []; + let execAttempts = 0; + + const client = { + set: async (key: string, value: string, mode: "PX", millisecondsToExpire: number) => { + setCalls.push({ key, value, mode, millisecondsToExpire }); + values.set(key, value); + return "OK"; + }, + get: async (key: string) => values.get(key) ?? null, + ...(options?.atomic + ? { + watch: async (key: string) => { + watchCalls.push(key); + return "OK"; + }, + unwatch: async () => "OK", + multi: () => { + let pending: + | { + key: string; + value: string; + mode: "PX"; + millisecondsToExpire: number; + } + | undefined; + const chain = { + set: (key: string, value: string, mode: "PX", millisecondsToExpire: number) => { + pending = { key, value, mode, millisecondsToExpire }; + return chain; + }, + exec: async () => { + execAttempts++; + if (options.execFailCount && execAttempts <= options.execFailCount) { + return null; + } + if (pending) { + setCalls.push(pending); + values.set(pending.key, pending.value); + } + return ["OK"]; + }, + }; + return chain; + }, + } + : {}), + }; + + const getRedisClient: ConstructorParameters<typeof RedisAdaptiveRateLimitStore>[0] = async () => + client; return { values, setCalls, + watchCalls, + get execAttempts() { + return execAttempts; + }, store: new RedisAdaptiveRateLimitStore(getRedisClient), }; } @@ -451,6 +497,49 @@ describe("RedisAdaptiveRateLimitStore", () => { expect(saved.inferredBudget).toBeNull(); expect(saved.observedCooldownSeconds).toBeNull(); }); + + it("uses atomic Redis WATCH/MULTI admission when available", async () => { + const mock = createMockRedisAdaptiveStore({ atomic: true }); + + await mock.store.awaitAdmission("garmin", "provider", null); + + expect(mock.watchCalls).toEqual(["provider-adaptive-rate:garmin:provider"]); + const saved = JSON.parse(mock.values.get("provider-adaptive-rate:garmin:provider") ?? "{}"); + expect(saved.requestCount).toBe(1); + }); + + it("retries atomic Redis admission when exec reports a conflict", async () => { + const mock = createMockRedisAdaptiveStore({ atomic: true, execFailCount: 1 }); + + await mock.store.awaitAdmission("garmin", "provider", null); + + expect(mock.watchCalls).toEqual([ + "provider-adaptive-rate:garmin:provider", + "provider-adaptive-rate:garmin:provider", + ]); + expect(mock.execAttempts).toBe(2); + }); + + it("creates fresh adaptive state when Redis returns null during atomic admission", async () => { + const mock = createMockRedisAdaptiveStore({ atomic: true }); + + await mock.store.awaitAdmission("whoop", "provider", null); + + const saved = JSON.parse(mock.values.get("provider-adaptive-rate:whoop:provider") ?? "{}"); + expect(saved.providerId).toBe("whoop"); + expect(saved.requestCount).toBe(1); + }); + + it("creates fresh adaptive state when Redis payload is invalid during atomic admission", async () => { + const mock = createMockRedisAdaptiveStore({ atomic: true }); + mock.values.set("provider-adaptive-rate:garmin:provider", "not-json"); + + await mock.store.awaitAdmission("garmin", "provider", null); + + const saved = JSON.parse(mock.values.get("provider-adaptive-rate:garmin:provider") ?? "{}"); + expect(saved.scope).toBe("provider"); + expect(saved.requestCount).toBe(1); + }); }); describe("providerAdaptiveRateLimitStore", () => { From f384667121e1cb60848dd6c0c549ed1ec524686b Mon Sep 17 00:00:00 2001 From: Asher Cohen <asher.cohen@amplitude.com> Date: Mon, 22 Jun 2026 08:08:41 -0700 Subject: [PATCH 17/18] Address remaining CodeRabbit review feedback on rate limiting. Cap Redis cooldown WATCH/MULTI retries so persistent conflicts fail fast, and use null for uncaptured Peloton token request bodies in tests. Co-authored-by: Cursor <cursoragent@cursor.com> --- src/jobs/provider-rate-limit-cooldown.test.ts | 20 +++++++++++++++++++ src/jobs/provider-rate-limit-cooldown.ts | 8 ++++++++ .../peloton-sync-ext.integration.test.ts | 2 +- 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/jobs/provider-rate-limit-cooldown.test.ts b/src/jobs/provider-rate-limit-cooldown.test.ts index 9e7ae5c821..b6e3881a6b 100644 --- a/src/jobs/provider-rate-limit-cooldown.test.ts +++ b/src/jobs/provider-rate-limit-cooldown.test.ts @@ -612,6 +612,26 @@ describe("ProviderRateLimitCooldownStore", () => { vi.useRealTimers(); }); + it("throws when Redis WATCH/MULTI writes exceed the retry limit", async () => { + vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); + const mock = createMockRedisStore({ + atomic: true, + execFailCount: Number.POSITIVE_INFINITY, + }); + + await expect( + mock.store.record( + rateLimitError({ providerId: "garmin", retryAfterSeconds: 600 }), + "user-1", + ), + ).rejects.toThrow( + "Failed to persist rate-limit cooldown for provider-rate-limit:garmin:provider after 10 Redis transaction conflicts", + ); + + expect(mock.execAttempts).toBe(10); + vi.useRealTimers(); + }); + it("retries Redis WATCH/MULTI writes when exec reports a conflict", async () => { vi.setSystemTime(new Date("2026-06-02T12:00:00Z")); const mock = createMockRedisStore({ diff --git a/src/jobs/provider-rate-limit-cooldown.ts b/src/jobs/provider-rate-limit-cooldown.ts index 3131a56a91..79546314d5 100644 --- a/src/jobs/provider-rate-limit-cooldown.ts +++ b/src/jobs/provider-rate-limit-cooldown.ts @@ -35,6 +35,7 @@ interface RedisClient { const DEFAULT_FALLBACK_COOLDOWN_SECONDS = 30 * 60; const DEFAULT_MAX_COOLDOWN_SECONDS = 2 * 60 * 60; const STRIKE_RESET_AFTER_MS = 2 * 60 * 60 * 1000; +const MAX_COOLDOWN_RETRIES = 10; const KEY_PREFIX = "provider-rate-limit"; function fallbackCooldownSeconds(providerId: string): number { @@ -200,6 +201,7 @@ async function persistCooldownAtomically( return effective; } + let retries = 0; for (;;) { await watch(key); const previous = parseCooldown(await redisClient.get(key)); @@ -208,6 +210,12 @@ async function persistCooldownAtomically( .set(key, serializeCooldown(effective), "PX", providerRateLimitDelayMs(effective)) .exec(); if (execResult) return effective; + retries++; + if (retries >= MAX_COOLDOWN_RETRIES) { + throw new Error( + `Failed to persist rate-limit cooldown for ${key} after ${MAX_COOLDOWN_RETRIES} Redis transaction conflicts`, + ); + } } } diff --git a/src/providers/peloton-sync-ext.integration.test.ts b/src/providers/peloton-sync-ext.integration.test.ts index 90ccfb064e..393a44ae10 100644 --- a/src/providers/peloton-sync-ext.integration.test.ts +++ b/src/providers/peloton-sync-ext.integration.test.ts @@ -600,7 +600,7 @@ describe("pelotonAutomatedLogin", () => { `; let step = 0; - let tokenRequestBody = ""; + let tokenRequestBody: string | null = null; loginServer.use( http.get("https://auth.onepeloton.com/authorize", () => { From a83435d8d75f4faefbafde26459855dcfb92689c Mon Sep 17 00:00:00 2001 From: Asher Cohen <asher.cohen@amplitude.com> Date: Mon, 22 Jun 2026 08:25:32 -0700 Subject: [PATCH 18/18] Fix CI lint and typecheck failures in provider tests. Remove obsolete Strava throttle constructor args, satisfy Biome formatting, and parse Peloton token bodies without nullable match calls. Co-authored-by: Cursor <cursoragent@cursor.com> --- src/jobs/provider-rate-limit-cooldown.test.ts | 5 +- .../peloton-sync-ext.integration.test.ts | 2 +- src/providers/strava.test.ts | 52 +++++++++---------- 3 files changed, 28 insertions(+), 31 deletions(-) diff --git a/src/jobs/provider-rate-limit-cooldown.test.ts b/src/jobs/provider-rate-limit-cooldown.test.ts index b6e3881a6b..7ec965e335 100644 --- a/src/jobs/provider-rate-limit-cooldown.test.ts +++ b/src/jobs/provider-rate-limit-cooldown.test.ts @@ -620,10 +620,7 @@ describe("ProviderRateLimitCooldownStore", () => { }); await expect( - mock.store.record( - rateLimitError({ providerId: "garmin", retryAfterSeconds: 600 }), - "user-1", - ), + mock.store.record(rateLimitError({ providerId: "garmin", retryAfterSeconds: 600 }), "user-1"), ).rejects.toThrow( "Failed to persist rate-limit cooldown for provider-rate-limit:garmin:provider after 10 Redis transaction conflicts", ); diff --git a/src/providers/peloton-sync-ext.integration.test.ts b/src/providers/peloton-sync-ext.integration.test.ts index 393a44ae10..f727c974d4 100644 --- a/src/providers/peloton-sync-ext.integration.test.ts +++ b/src/providers/peloton-sync-ext.integration.test.ts @@ -658,7 +658,7 @@ describe("pelotonAutomatedLogin", () => { expect(tokens.accessToken).toBe("new-access-token"); expect(tokens.refreshToken).toBe("new-refresh-token"); expect(tokenRequestBody).toContain("code_verifier="); - const verifierMatch = tokenRequestBody.match(/(?:^|&)code_verifier=([^&]+)/); + const verifierMatch = /(?:^|&)code_verifier=([^&]+)/.exec(tokenRequestBody ?? ""); expect(verifierMatch?.[1]).toBeDefined(); expect(verifierMatch?.[1]).not.toBe("undefined"); }); diff --git a/src/providers/strava.test.ts b/src/providers/strava.test.ts index 466586b05f..2ba810d00f 100644 --- a/src/providers/strava.test.ts +++ b/src/providers/strava.test.ts @@ -1829,7 +1829,7 @@ describe("StravaClient", () => { ]), ); - const client = new StravaClient("test-token", mockFetch, 0); + const client = new StravaClient("test-token", mockFetch); const result = await client.getActivities(1000, 2, 50); expect(mockFetch).toHaveBeenCalledOnce(); @@ -1879,7 +1879,7 @@ describe("StravaClient", () => { ]), ); - const client = new StravaClient("test-token", mockFetch, 0); + const client = new StravaClient("test-token", mockFetch); const streams = await client.getActivityStreams(12345); const calledUrl = String(mockFetch.mock.calls[0]?.[0]); @@ -1907,7 +1907,7 @@ describe("StravaProvider.sync", () => { process.env.STRAVA_CLIENT_SECRET = "secret"; const mockFetch = vi.fn(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); // Mock db with loadTokens returning null const mockDb = { @@ -1950,7 +1950,7 @@ describe("StravaProvider.sync", () => { return Response.json([]); }); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); // Provide tokens const futureDate = new Date("2099-01-01"); @@ -2116,7 +2116,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), @@ -2170,7 +2170,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), ); @@ -2214,7 +2214,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: sinceDate }) }), @@ -2243,7 +2243,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: sinceDate }) }), @@ -2279,7 +2279,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), @@ -2334,7 +2334,7 @@ describe("StravaProvider.sync — additional coverage", () => { execute: vi.fn().mockResolvedValue([]), }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), ); @@ -2364,7 +2364,7 @@ describe("StravaProvider.sync — additional coverage", () => { const mockDb = createMockDb(); const onProgress = vi.fn(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); await provider.sync( new SyncRun({ @@ -2392,7 +2392,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), @@ -2422,7 +2422,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), @@ -2444,7 +2444,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const before = Date.now(); const result = await provider.sync( @@ -2485,7 +2485,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb([EXPIRED_TOKEN]); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), @@ -2509,7 +2509,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb([VALID_TOKEN]); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), @@ -2564,7 +2564,7 @@ describe("StravaProvider.sync — additional coverage", () => { execute: vi.fn().mockResolvedValue([]), }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), ); @@ -2591,7 +2591,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), ); @@ -2619,7 +2619,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), ); @@ -2646,7 +2646,7 @@ describe("StravaProvider.sync — additional coverage", () => { providerActivityAbsenceMocks.upsertProviderActivity.mockResolvedValueOnce(undefined); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), ); @@ -2679,7 +2679,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), @@ -2736,7 +2736,7 @@ describe("StravaProvider.sync — additional coverage", () => { execute: vi.fn().mockResolvedValue([]), }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), ); @@ -2811,7 +2811,7 @@ describe("StravaProvider.sync — additional coverage", () => { execute: vi.fn().mockResolvedValue([]), }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), ); @@ -2839,7 +2839,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), ); @@ -2867,7 +2867,7 @@ describe("StravaProvider.sync — additional coverage", () => { }); const mockDb = createMockDb(); - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), ); @@ -2927,7 +2927,7 @@ describe("StravaProvider.sync — additional coverage", () => { execute: vi.fn().mockResolvedValue([]), }; - const provider = new StravaProvider(mockFetch, 0); + const provider = new StravaProvider(mockFetch); const result = await provider.sync( new SyncRun({ db: mockDb, window: SyncWindow.fromSince({ since: new Date("2026-01-01") }) }), );