diff --git a/packages/mobile/app/providers/file-import-button.tsx b/packages/mobile/app/providers/file-import-button.tsx new file mode 100644 index 0000000000..a48408e1c8 --- /dev/null +++ b/packages/mobile/app/providers/file-import-button.tsx @@ -0,0 +1,28 @@ +import { ActivityIndicator, Text, TouchableOpacity } from "react-native"; +import { colors } from "../../theme"; +import { styles } from "./styles.ts"; + +export function FileImportButton({ + disabled, + loading, + onPress, +}: { + disabled: boolean; + loading: boolean; + onPress: () => void; +}) { + return ( + + {loading ? ( + + ) : ( + Import file + )} + + ); +} diff --git a/packages/mobile/app/providers/file-import-provider-card.tsx b/packages/mobile/app/providers/file-import-provider-card.tsx new file mode 100644 index 0000000000..60526ca2d1 --- /dev/null +++ b/packages/mobile/app/providers/file-import-provider-card.tsx @@ -0,0 +1,44 @@ +import type { ProviderStats } from "@dofek/providers/provider-stats"; +import type { ImportProviderId } from "../../lib/share-import"; +import { getFileImportProviderConfig } from "./file-import-providers.ts"; +import { type Provider, ProviderCard } from "./provider-card.tsx"; + +export function FileImportProviderCard({ + provider, + stats, + syncing, + importing, + syncProgress, + onSync, + onFullSync, + onConnect, + onImportProvider, + onPress, +}: { + provider: Provider; + stats: ProviderStats | undefined; + syncing: boolean; + importing?: boolean; + syncProgress: { percentage?: number; message?: string } | undefined; + onSync: () => void; + onFullSync: () => void; + onConnect: () => void; + onImportProvider: (providerId: ImportProviderId) => void; + onPress: () => void; +}) { + const providerConfig = getFileImportProviderConfig(provider.id); + return ( + onImportProvider(providerConfig.providerId) : undefined} + onPress={onPress} + /> + ); +} diff --git a/packages/mobile/app/providers/file-import-providers.ts b/packages/mobile/app/providers/file-import-providers.ts new file mode 100644 index 0000000000..8b7d01b58e --- /dev/null +++ b/packages/mobile/app/providers/file-import-providers.ts @@ -0,0 +1,64 @@ +import type { ImportProviderId } from "../../lib/share-import"; + +type FileImportProviderConfig = { + providerId: ImportProviderId; + documentTypes: string[]; + selectionErrorMessage: string; +}; + +const csvDocumentTypes = [ + "text/csv", + "text/comma-separated-values", + "application/csv", + "text/plain", +]; + +const fileImportProviderConfigs = { + "apple-health": { + providerId: "apple-health", + documentTypes: [ + "application/zip", + "application/x-zip-compressed", + "application/xml", + "text/xml", + ], + selectionErrorMessage: "Unable to select Apple Health export", + }, + "strong-csv": { + providerId: "strong-csv", + documentTypes: csvDocumentTypes, + selectionErrorMessage: "Unable to select Strong export", + }, + "cronometer-csv": { + providerId: "cronometer-csv", + documentTypes: csvDocumentTypes, + selectionErrorMessage: "Unable to select Cronometer export", + }, + "garmin-dump": { + providerId: "garmin-dump", + documentTypes: ["application/zip", "application/x-zip-compressed"], + selectionErrorMessage: "Unable to select Garmin export", + }, + "kaya-export": { + providerId: "kaya-export", + documentTypes: csvDocumentTypes, + selectionErrorMessage: "Unable to select Kaya export", + }, +} satisfies Partial>; + +export function getFileImportProviderConfig( + providerId: string, +): FileImportProviderConfig | undefined { + switch (providerId) { + case "apple-health": + case "apple_health": + return fileImportProviderConfigs["apple-health"]; + case "strong-csv": + case "cronometer-csv": + case "garmin-dump": + case "kaya-export": + return fileImportProviderConfigs[providerId]; + default: + return undefined; + } +} diff --git a/packages/mobile/app/providers/index.stories.tsx b/packages/mobile/app/providers/index.stories.tsx index 86d49d82b4..5a2fa231de 100644 --- a/packages/mobile/app/providers/index.stories.tsx +++ b/packages/mobile/app/providers/index.stories.tsx @@ -106,6 +106,22 @@ export const ImportOnly: ProviderCardStory = { }, }; +export const GarminDumpImport: ProviderCardStory = { + args: { + provider: { + id: "garmin-dump", + label: "Garmin Dump", + enabled: false, + authStatus: "connected", + authType: "none", + lastSyncAt: null, + importOnly: true, + pushOnly: false, + }, + onImport: () => {}, + }, +}; + export const AppleHealthImportOnly: ProviderCardStory = { args: { provider: { diff --git a/packages/mobile/app/providers/index.test.tsx b/packages/mobile/app/providers/index.test.tsx index 9bd0fe0d4c..bd7e169397 100644 --- a/packages/mobile/app/providers/index.test.tsx +++ b/packages/mobile/app/providers/index.test.tsx @@ -10,11 +10,12 @@ const mockReplace = vi.fn(); const mockUseLocalSearchParams = vi.fn().mockReturnValue({}); const mockSyncMutateAsync = vi.fn(); const mockImportSharedFile = vi.fn(); +const mockGetDocumentAsync = vi.fn(); vi.mock("react-native", () => ({ View: ({ children, ...props }: { children?: React.ReactNode } & Record) => { const { - style: _s, + style, contentContainerStyle: _cs, activeOpacity: _ao, numberOfLines: _nl, @@ -25,6 +26,7 @@ vi.mock("react-native", () => ({ "div", { ...(testID ? { "data-testid": testID } : {}), + ...(testID && style ? { "data-style": JSON.stringify(style) } : {}), ...rest, }, children, @@ -145,10 +147,14 @@ vi.mock("../../theme", () => ({ spacing: { xs: 4, sm: 8, md: 16, lg: 24, xl: 32 }, })); +const mockAuthState = vi.hoisted((): { sessionToken: string | null } => ({ + sessionToken: "test-token", +})); + vi.mock("../../lib/auth-context", () => ({ useAuth: () => ({ serverUrl: "https://test.example.com", - sessionToken: "test-token", + sessionToken: mockAuthState.sessionToken, }), })); @@ -189,6 +195,10 @@ vi.mock("expo-web-browser", () => ({ openBrowserAsync: vi.fn().mockResolvedValue({ type: "cancel" }), })); +vi.mock("expo-document-picker", () => ({ + getDocumentAsync: (...args: unknown[]) => mockGetDocumentAsync(...args), +})); + vi.mock("../../lib/share-import", () => ({ importSharedFile: (...args: unknown[]) => mockImportSharedFile(...args), })); @@ -242,12 +252,14 @@ const mockStatsQuery = vi.fn(); const mockLogsQuery = vi.fn(); const mockDataHealthQuery = vi.fn(); const mockActiveSyncsQuery = vi.fn(); +const mockActiveImportsQuery = vi.fn(); const mockInvalidate = vi.fn(); const mockInvalidateProviders = vi.fn(); const mockInvalidateProviderStats = vi.fn(); const mockInvalidateLogs = vi.fn(); const mockInvalidateDataHealth = vi.fn(); const mockInvalidateActiveSyncs = vi.fn(); +const mockInvalidateActiveImports = vi.fn(); const mockSyncStatusFetch = vi.fn(); const mockCredentialSignIn = vi.fn(); const mockGarminSignIn = vi.fn(); @@ -273,6 +285,7 @@ vi.mock("../../lib/trpc", () => ({ dataHealth: { useQuery: (...args: unknown[]) => mockDataHealthQuery(...args) }, triggerSync: { useMutation: () => ({ mutateAsync: mockSyncMutateAsync }) }, activeSyncs: { useQuery: (...args: unknown[]) => mockActiveSyncsQuery(...args) }, + activeImports: { useQuery: (...args: unknown[]) => mockActiveImportsQuery(...args) }, }, credentialAuth: { signIn: { useMutation: () => ({ mutateAsync: mockCredentialSignIn }) }, @@ -304,6 +317,7 @@ vi.mock("../../lib/trpc", () => ({ logs: { invalidate: mockInvalidateLogs }, dataHealth: { invalidate: mockInvalidateDataHealth }, activeSyncs: { invalidate: mockInvalidateActiveSyncs }, + activeImports: { invalidate: mockInvalidateActiveImports }, syncStatus: { fetch: mockSyncStatusFetch }, }, }), @@ -366,6 +380,7 @@ function setupDefaultMocks() { mockLogsQuery.mockReturnValue({ data: [], isLoading: false, error: null }); mockDataHealthQuery.mockReturnValue({ data: undefined, isLoading: false, error: null }); mockActiveSyncsQuery.mockReturnValue({ data: [] }); + mockActiveImportsQuery.mockReturnValue({ data: [], error: null }); } function makeProvider( @@ -651,7 +666,119 @@ describe("ProviderCard", () => { expect(screen.getByText("Wahoo")).toBeTruthy(); }); + it("renders the shared file import button alongside sync for providers that support both", async () => { + const { ProviderCard } = await import("./provider-card.tsx"); + const importFile = vi.fn(); + render( + , + ); + + expect(screen.getByText("Sync")).toBeTruthy(); + fireEvent.click(screen.getByText("Import file")); + + expect(importFile).toHaveBeenCalledOnce(); + }); + + it("wires provider ids through the shared file import provider card", async () => { + const { FileImportProviderCard } = await import("./file-import-provider-card.tsx"); + const importProvider = vi.fn(); + render( + , + ); + + fireEvent.click(screen.getByText("Import file")); + + expect(importProvider).toHaveBeenCalledWith("apple-health"); + }); + + it("renders the shared provider summary on file import provider cards", async () => { + const { FileImportProviderCard } = await import("./file-import-provider-card.tsx"); + render( + , + ); + + expect(screen.getByText("206,539")).toBeTruthy(); + expect(screen.getByText("records")).toBeTruthy(); + expect(screen.getByText("Activities")).toBeTruthy(); + expect(screen.getByText("352")).toBeTruthy(); + expect(screen.getByText("Metric Stream")).toBeTruthy(); + expect(screen.getByText("205,367")).toBeTruthy(); + expect(screen.getByText("Daily Metrics")).toBeTruthy(); + expect(screen.getByText("229")).toBeTruthy(); + expect(screen.getByText("Sleep")).toBeTruthy(); + expect(screen.getByText("155")).toBeTruthy(); + expect(screen.getByText("Body")).toBeTruthy(); + expect(screen.getByText("43")).toBeTruthy(); + expect(screen.getByText("Events")).toBeTruthy(); + expect(screen.getByText("392")).toBeTruthy(); + }); + describe("import-only providers", () => { + it("keeps the sync action separate from an active file import", async () => { + const { ProviderCard } = await import("./provider-card.tsx"); + render( + , + ); + + expect(screen.getByText("Sync")).toBeTruthy(); + expect(screen.queryByText("Import file")).toBeNull(); + expect(screen.getByText("Importing file")).toBeTruthy(); + }); + it("does not render Sync button for import-only providers", async () => { const { ProviderCard } = await import("./provider-card.tsx"); render( @@ -718,17 +845,20 @@ describe("ProvidersScreen", () => { mockUseLocalSearchParams.mockReturnValue({}); mockSyncMutateAsync.mockReset(); mockImportSharedFile.mockReset(); + mockGetDocumentAsync.mockReset(); mockProvidersQuery.mockReset(); mockStatsQuery.mockReset(); mockLogsQuery.mockReset(); mockDataHealthQuery.mockReset(); mockActiveSyncsQuery.mockReset(); + mockActiveImportsQuery.mockReset(); mockInvalidate.mockReset(); mockInvalidateProviders.mockReset(); mockInvalidateProviderStats.mockReset(); mockInvalidateLogs.mockReset(); mockInvalidateDataHealth.mockReset(); mockInvalidateActiveSyncs.mockReset(); + mockInvalidateActiveImports.mockReset(); mockSyncStatusFetch.mockReset(); mockUseRefresh.mockClear(); mockCredentialSignIn.mockReset(); @@ -737,6 +867,7 @@ describe("ProvidersScreen", () => { mockWhoopVerifyCode.mockReset(); mockWhoopSaveTokens.mockReset(); mockFileDelete.mockReset(); + mockAuthState.sessionToken = "test-token"; mockFileBytes.mockClear(); mockIsHealthKitAvailable.mockReset().mockReturnValue(true); mockGetRequestStatus.mockReset().mockResolvedValue("unnecessary"); @@ -1212,6 +1343,263 @@ describe("ProvidersScreen", () => { }); }); + it("deletes a shared file and surfaces auth errors when importing without a session", async () => { + mockAuthState.sessionToken = null; + mockUseLocalSearchParams.mockReturnValue({ + sharedFile: "file:///tmp/Strong%20Export.csv", + }); + + await renderProvidersScreen(); + + await waitFor(() => { + expect(mockFileDelete).toHaveBeenCalledWith("file:///tmp/Strong%20Export.csv"); + }); + expect(mockImportSharedFile).not.toHaveBeenCalled(); + expect(screen.getByText("Sign in before importing a file")).toBeTruthy(); + }); + + it("selects and imports a Garmin dump ZIP from the Garmin card", async () => { + mockProvidersQuery.mockReturnValue({ + data: [ + { + id: "garmin-dump", + name: "Garmin Dump", + authType: "none", + authorized: true, + importOnly: true, + lastSyncedAt: null, + }, + ], + isLoading: false, + error: null, + }); + mockGetDocumentAsync.mockResolvedValue({ + canceled: false, + assets: [ + { + uri: "file:///tmp/garmin-export.zip", + name: "garmin-export.zip", + mimeType: "application/zip", + }, + ], + }); + mockImportSharedFile.mockResolvedValue({ providerId: "garmin-dump", jobId: "job-garmin" }); + + await renderProvidersScreen(); + + const garminCard = within(screen.getByTestId("provider-card-garmin-dump")); + fireEvent.click(garminCard.getByText("Import file")); + + await waitFor(() => { + expect(mockGetDocumentAsync).toHaveBeenCalledWith({ + copyToCacheDirectory: true, + multiple: false, + type: ["application/zip", "application/x-zip-compressed"], + }); + expect(mockImportSharedFile).toHaveBeenCalledWith( + expect.objectContaining({ + fileUri: "file:///tmp/garmin-export.zip", + providerId: "garmin-dump", + serverUrl: "https://test.example.com", + sessionToken: "test-token", + }), + expect.objectContaining({ readBlob: expect.any(Function) }), + ); + }); + }); + + it("selects and imports a Strong CSV from the Strong card", async () => { + mockProvidersQuery.mockReturnValue({ + data: [importOnlyProvider], + isLoading: false, + error: null, + }); + mockGetDocumentAsync.mockResolvedValue({ + canceled: false, + assets: [ + { + uri: "file:///tmp/strong-export.csv", + name: "strong-export.csv", + mimeType: "text/csv", + }, + ], + }); + mockImportSharedFile.mockResolvedValue({ providerId: "strong-csv", jobId: "job-strong" }); + + await renderProvidersScreen(); + + const strongCard = within(screen.getByTestId("provider-card-strong-csv")); + fireEvent.click(strongCard.getByText("Import file")); + + await waitFor(() => { + expect(mockGetDocumentAsync).toHaveBeenCalledWith({ + copyToCacheDirectory: true, + multiple: false, + type: ["text/csv", "text/comma-separated-values", "application/csv", "text/plain"], + }); + expect(mockImportSharedFile).toHaveBeenCalledWith( + expect.objectContaining({ + fileUri: "file:///tmp/strong-export.csv", + providerId: "strong-csv", + serverUrl: "https://test.example.com", + sessionToken: "test-token", + }), + expect.objectContaining({ readBlob: expect.any(Function) }), + ); + }); + }); + + it("selects and imports a Cronometer CSV from the Cronometer card", async () => { + mockProvidersQuery.mockReturnValue({ + data: [ + { + ...importOnlyProvider, + id: "cronometer-csv", + name: "Cronometer", + }, + ], + isLoading: false, + error: null, + }); + mockGetDocumentAsync.mockResolvedValue({ + canceled: false, + assets: [ + { + uri: "file:///tmp/cronometer-export.csv", + name: "cronometer-export.csv", + mimeType: "text/csv", + }, + ], + }); + mockImportSharedFile.mockResolvedValue({ + providerId: "cronometer-csv", + jobId: "job-cronometer", + }); + + await renderProvidersScreen(); + + const cronometerCard = within(screen.getByTestId("provider-card-cronometer-csv")); + fireEvent.click(cronometerCard.getByText("Import file")); + + await waitFor(() => { + expect(mockGetDocumentAsync).toHaveBeenCalledWith({ + copyToCacheDirectory: true, + multiple: false, + type: ["text/csv", "text/comma-separated-values", "application/csv", "text/plain"], + }); + expect(mockImportSharedFile).toHaveBeenCalledWith( + expect.objectContaining({ + fileUri: "file:///tmp/cronometer-export.csv", + providerId: "cronometer-csv", + serverUrl: "https://test.example.com", + sessionToken: "test-token", + }), + expect.objectContaining({ readBlob: expect.any(Function) }), + ); + }); + }); + + it("selects and imports a Kaya CSV from the Kaya card", async () => { + mockProvidersQuery.mockReturnValue({ + data: [ + { + ...importOnlyProvider, + id: "kaya-export", + name: "Kaya", + }, + ], + isLoading: false, + error: null, + }); + mockGetDocumentAsync.mockResolvedValue({ + canceled: false, + assets: [ + { + uri: "file:///tmp/kaya-export.csv", + name: "kaya-export.csv", + mimeType: "text/csv", + }, + ], + }); + mockImportSharedFile.mockResolvedValue({ + providerId: "kaya-export", + jobId: "job-kaya", + }); + + await renderProvidersScreen(); + + const kayaCard = within(screen.getByTestId("provider-card-kaya-export")); + fireEvent.click(kayaCard.getByText("Import file")); + + await waitFor(() => { + expect(mockGetDocumentAsync).toHaveBeenCalledWith({ + copyToCacheDirectory: true, + multiple: false, + type: ["text/csv", "text/comma-separated-values", "application/csv", "text/plain"], + }); + expect(mockImportSharedFile).toHaveBeenCalledWith( + expect.objectContaining({ + fileUri: "file:///tmp/kaya-export.csv", + providerId: "kaya-export", + serverUrl: "https://test.example.com", + sessionToken: "test-token", + }), + expect.objectContaining({ readBlob: expect.any(Function) }), + ); + }); + }); + + it("shows resumable progress for an in-flight Garmin import", async () => { + mockProvidersQuery.mockReturnValue({ + data: [ + { + id: "garmin-dump", + name: "Garmin Dump", + authType: "none", + authorized: true, + importOnly: true, + lastSyncedAt: null, + }, + ], + isLoading: false, + error: null, + }); + mockActiveImportsQuery.mockReturnValue({ + data: [ + { + jobId: "job-garmin", + providerId: "garmin-dump", + progress: 64, + message: "Importing activities", + }, + ], + error: null, + }); + + await renderProvidersScreen(); + + const garminCard = within(screen.getByTestId("provider-card-garmin-dump")); + expect(garminCard.getByText("Importing activities")).toBeTruthy(); + expect(garminCard.getByText("Loading...")).toBeTruthy(); + expect( + screen.getByTestId("provider-card-garmin-dump-progress-fill").getAttribute("data-style"), + ).toContain('"width":"64%"'); + expect(garminCard.queryByText("Import only")).toBeNull(); + }); + + it("shows the server error when active import progress cannot load", async () => { + mockActiveImportsQuery.mockReturnValue({ + data: undefined, + error: new Error("Unable to check import progress because the queue service is unavailable."), + }); + + await renderProvidersScreen(); + + expect( + screen.getByText("Unable to check import progress because the queue service is unavailable."), + ).toBeTruthy(); + }); + it("shows shared import progress while providers are still loading", async () => { mockProvidersQuery.mockReturnValue({ data: undefined, @@ -1612,6 +2000,70 @@ describe("ProvidersScreen", () => { expect(screen.getByText("Apple Health")).toBeTruthy(); }); + it("shows resumable progress for an in-flight Apple Health import", async () => { + mockActiveImportsQuery.mockReturnValue({ + data: [ + { + jobId: "job-apple-health", + providerId: "apple_health", + progress: 42, + message: "Importing Apple Health data", + }, + ], + error: null, + }); + + await renderProvidersScreen(); + + const appleCard = within(screen.getByTestId("provider-card-apple_health")); + expect(appleCard.getByText("Importing Apple Health data")).toBeTruthy(); + expect(appleCard.getByText("Loading...")).toBeTruthy(); + expect( + screen.getByTestId("provider-card-apple_health-progress-fill").getAttribute("data-style"), + ).toContain('"width":"42%"'); + }); + + it("selects and imports an Apple Health export from the Apple Health card", async () => { + mockGetDocumentAsync.mockResolvedValue({ + canceled: false, + assets: [ + { + uri: "file:///tmp/apple-health-export.zip", + name: "apple-health-export.zip", + mimeType: "application/zip", + }, + ], + }); + mockImportSharedFile.mockResolvedValue({ providerId: "apple-health", jobId: "job-apple" }); + + await renderProvidersScreen(); + + await waitFor(() => { + const appleCard = within(screen.getByTestId("provider-card-apple_health")); + expect(appleCard.getByText("Import file")).toBeTruthy(); + }); + + const appleCard = within(screen.getByTestId("provider-card-apple_health")); + fireEvent.click(appleCard.getByText("Import file")); + + await waitFor(() => { + expect(mockGetDocumentAsync).toHaveBeenCalledWith({ + copyToCacheDirectory: true, + multiple: false, + type: ["application/zip", "application/x-zip-compressed", "application/xml", "text/xml"], + }); + expect(mockImportSharedFile).toHaveBeenCalledWith( + expect.objectContaining({ + fileUri: "file:///tmp/apple-health-export.zip", + providerId: "apple-health", + serverUrl: "https://test.example.com", + sessionToken: "test-token", + }), + expect.objectContaining({ readBlob: expect.any(Function) }), + ); + }); + }); + it("triggers HealthKit sync with syncRangeDays: 7 when Sync is clicked", async () => { await renderProvidersScreen(); diff --git a/packages/mobile/app/providers/index.tsx b/packages/mobile/app/providers/index.tsx index 5aee58ca8c..fc764756fa 100644 --- a/packages/mobile/app/providers/index.tsx +++ b/packages/mobile/app/providers/index.tsx @@ -1,5 +1,6 @@ import { formatDateYmd } from "@dofek/format/format"; import type { ProviderStats } from "@dofek/providers/provider-stats"; +import * as DocumentPicker from "expo-document-picker"; import { File as ExpoFile } from "expo-file-system"; import { useLocalSearchParams, useRouter } from "expo-router"; import * as WebBrowser from "expo-web-browser"; @@ -17,13 +18,19 @@ import { getQueryErrorMessage, QueryStatePanel } from "../../components/QuerySta import { useAppleHealthProviderModel } from "../../lib/apple-health-provider"; import { useAuth } from "../../lib/auth-context"; import { syncDofekFoodToHealthKit } from "../../lib/health-kit-food-writeback"; -import { importSharedFile, type ShareImportProgress } from "../../lib/share-import"; +import { + type ImportProviderId, + importSharedFile, + type ShareImportProgress, +} from "../../lib/share-import"; import { captureException } from "../../lib/telemetry"; import { trpc } from "../../lib/trpc"; import { useRefresh } from "../../lib/useRefresh"; import { deleteDietarySamples, writeDietarySamples } from "../../modules/health-kit"; import { colors } from "../../theme"; import { CredentialAuthModal, GarminAuthModal, WhoopAuthModal } from "./auth-modals.tsx"; +import { FileImportProviderCard } from "./file-import-provider-card.tsx"; +import { getFileImportProviderConfig } from "./file-import-providers.ts"; import { importProviderLabel, type Provider, @@ -70,6 +77,10 @@ export default function ProvidersScreen() { const syncMutation = trpc.sync.triggerSync.useMutation(); const trpcUtils = trpc.useUtils(); const activeSyncs = trpc.sync.activeSyncs.useQuery(undefined, { staleTime: 0 }); + const activeImports = trpc.sync.activeImports.useQuery(undefined, { + staleTime: 0, + refetchInterval: (query) => ((query.state.data?.length ?? 0) > 0 ? 1000 : false), + }); // Auth modal state const [credentialAuthProvider, setCredentialAuthProvider] = useState<{ @@ -274,17 +285,16 @@ export default function ProvidersScreen() { } }, [activeSyncs.data, pollJob]); - useEffect(() => { - if (!sharedFileUri) return; - if (!sessionToken) return; - if (importedSharedUris.current.has(sharedFileUri)) return; - importedSharedUris.current.add(sharedFileUri); - - void (async () => { + const importFile = useCallback( + async (fileUri: string, providerId?: ImportProviderId) => { try { + if (!sessionToken) { + throw new Error("Sign in before importing a file"); + } await importSharedFile( { - fileUri: sharedFileUri, + fileUri, + providerId, serverUrl, sessionToken, onProgress: setSharedImportState, @@ -293,21 +303,58 @@ export default function ProvidersScreen() { ); trpcUtils.invalidate(); } catch (error: unknown) { - captureException(error, { context: "share-import", fileUri: sharedFileUri }); + captureException(error, { context: "share-import", fileUri, providerId }); setSharedImportState({ status: "error", progress: 0, message: error instanceof Error ? error.message : "Import failed", + providerId, }); } finally { try { - deleteSharedFile(sharedFileUri); + deleteSharedFile(fileUri); } catch (error: unknown) { - captureException(error, { context: "share-import-cleanup", fileUri: sharedFileUri }); + captureException(error, { context: "share-import-cleanup", fileUri }); } } - })(); - }, [sharedFileUri, serverUrl, sessionToken, trpcUtils]); + }, + [serverUrl, sessionToken, trpcUtils], + ); + + useEffect(() => { + if (!sharedFileUri) return; + if (importedSharedUris.current.has(sharedFileUri)) return; + importedSharedUris.current.add(sharedFileUri); + + void importFile(sharedFileUri); + }, [importFile, sharedFileUri]); + + const handleFileImportProvider = useCallback( + async (providerId: ImportProviderId) => { + const providerConfig = getFileImportProviderConfig(providerId); + if (!providerConfig) return; + try { + const result = await DocumentPicker.getDocumentAsync({ + copyToCacheDirectory: true, + multiple: false, + type: providerConfig.documentTypes, + }); + if (result.canceled) return; + const asset = result.assets[0]; + if (!asset) return; + await importFile(asset.uri, providerConfig.providerId); + } catch (error: unknown) { + captureException(error, { context: "file-import-document-picker", providerId }); + setSharedImportState({ + status: "error", + progress: 0, + message: error instanceof Error ? error.message : providerConfig.selectionErrorMessage, + providerId: providerConfig.providerId, + }); + } + }, + [importFile], + ); const handleSyncProvider = useCallback( async (providerId: string, fullSync = false) => { @@ -490,12 +537,37 @@ export default function ProvidersScreen() { trpcUtils.sync.logs.invalidate({ limit: 50 }), trpcUtils.sync.dataHealth.invalidate(), trpcUtils.sync.activeSyncs.invalidate(), + trpcUtils.sync.activeImports.invalidate(), ]).then(() => undefined), }); const isLoading = providers.isLoading; const enabledProviders = providerList.filter((p) => p.enabled); const appleHealthProvider = appleHealth.model.toProviderCard(); + const activeImportRows = activeImports.error ? [] : (activeImports.data ?? []); + const activeImportByProvider = new Map( + activeImportRows.map((activeImport) => [activeImport.providerId, activeImport]), + ); + const localImportIsActive = + sharedImportState !== null && + sharedImportState.status !== "done" && + sharedImportState.status !== "error"; + const appleHealthActiveImport = activeImportByProvider.get("apple_health"); + const appleHealthActiveImportProgress = appleHealthActiveImport + ? { + percentage: appleHealthActiveImport.progress, + message: appleHealthActiveImport.message, + } + : undefined; + const appleHealthLocalImportProgress = + localImportIsActive && sharedImportState.providerId === "apple-health" + ? { + percentage: sharedImportState.progress, + message: sharedImportState.message, + } + : undefined; + const appleHealthImportProgress = + appleHealthLocalImportProgress ?? appleHealthActiveImportProgress; return ( Import from Share - Export a CSV, XML, or ZIP file from Strong, Cronometer, or Apple Health and share it to - Dofek. + Export a CSV, XML, or ZIP file from Strong, Cronometer, Apple Health, or Garmin and share + it to Dofek. + {activeImports.error ? ( + + ) : null} {sharedImportState ? ( @@ -581,18 +661,21 @@ export default function ProvidersScreen() { loading={dataHealth.isLoading} /> Data Sources - handleHealthKitSync()} onFullSync={() => handleHealthKitSync(true)} onConnect={handleHealthKitConnect} + onImportProvider={handleFileImportProvider} onPress={() => router.push("/providers/apple_health")} /> {appleHealth.model.shouldShowPermissionBanner() && ( @@ -623,19 +706,51 @@ export default function ProvidersScreen() { {isLoading && !providers.error ? ( ) : null} - {providerList.map((provider) => ( - handleSyncProvider(provider.id)} - onFullSync={() => handleSyncProvider(provider.id, true)} - onConnect={() => handleConnect(provider)} - onPress={() => router.push(`/providers/${provider.id}`)} - /> - ))} + {providerList.map((provider) => { + const fileImportProviderConfig = getFileImportProviderConfig(provider.id); + const activeImport = activeImportByProvider.get(provider.id); + const activeImportProgress = activeImport + ? { + percentage: activeImport.progress, + message: activeImport.message, + } + : undefined; + const localImportProgress = + localImportIsActive && sharedImportState.providerId === provider.id + ? { + percentage: sharedImportState.progress, + message: sharedImportState.message, + } + : undefined; + const importProgress = localImportProgress ?? activeImportProgress; + return fileImportProviderConfig ? ( + handleSyncProvider(provider.id)} + onFullSync={() => handleSyncProvider(provider.id, true)} + onConnect={() => handleConnect(provider)} + onImportProvider={handleFileImportProvider} + onPress={() => router.push(`/providers/${provider.id}`)} + /> + ) : ( + handleSyncProvider(provider.id)} + onFullSync={() => handleSyncProvider(provider.id, true)} + onConnect={() => handleConnect(provider)} + onPress={() => router.push(`/providers/${provider.id}`)} + /> + ); + })} {/* Sync History */} Sync History diff --git a/packages/mobile/app/providers/provider-card.tsx b/packages/mobile/app/providers/provider-card.tsx index 724d1844aa..086781db0f 100644 --- a/packages/mobile/app/providers/provider-card.tsx +++ b/packages/mobile/app/providers/provider-card.tsx @@ -5,6 +5,7 @@ import { ProviderLogo } from "../../components/ProviderLogo"; import { ProviderStatsBreakdown } from "../../components/ProviderStatsBreakdown"; import { useAuth } from "../../lib/auth-context"; import { colors } from "../../theme"; +import { FileImportButton } from "./file-import-button.tsx"; import { styles } from "./styles.ts"; export type AuthStatus = "connected" | "not_connected" | "expired"; @@ -68,6 +69,7 @@ const IMPORT_PROVIDER_LABELS: Readonly> = { "strong-csv": "Strong", "cronometer-csv": "Cronometer", "garmin-dump": "Garmin Dump", + "kaya-export": "Kaya", }; export function importProviderLabel(providerId: string | undefined): string { @@ -78,25 +80,31 @@ export function ProviderCard({ provider, stats, syncing, + importing = false, syncProgress, onSync, onFullSync, onConnect, + onImport, onPress, }: { provider: Provider; stats: ProviderStats | undefined; syncing: boolean; + importing?: boolean; syncProgress: { percentage?: number; message?: string } | undefined; onSync: () => void; onFullSync: () => void; onConnect: () => void; + onImport?: () => void; onPress: () => void; }) { const { serverUrl } = useAuth(); const dotColor = statusDotColor(provider.authStatus); const lastSyncRelative = provider.lastSyncAt ? formatRelativeTime(provider.lastSyncAt) : null; const canRunManualSync = !provider.importOnly && !provider.pushOnly; + const canImport = onImport !== undefined; + const showingProgress = (syncing || importing) && syncProgress !== undefined; return ( @@ -106,27 +114,37 @@ export function ProviderCard({ {provider.label} - {canRunManualSync && ( - - {syncing ? ( - - ) : ( - {providerActionLabel(provider.authStatus)} - )} - + {(canRunManualSync || canImport) && ( + + {canRunManualSync ? ( + + {syncing ? ( + + ) : ( + + {providerActionLabel(provider.authStatus)} + + )} + + ) : null} + {canImport ? ( + + ) : null} + )} - {syncing && syncProgress ? ( + {showingProgress ? ( {syncProgress.percentage != null && ( ) : ( - {!syncing && syncProgress?.message ? ( + {!showingProgress && syncProgress?.message ? ( {syncProgress.message} ) : ( diff --git a/packages/mobile/app/providers/styles.ts b/packages/mobile/app/providers/styles.ts index 63e8230de5..afd0e29f91 100644 --- a/packages/mobile/app/providers/styles.ts +++ b/packages/mobile/app/providers/styles.ts @@ -134,6 +134,8 @@ export const styles = StyleSheet.create({ flexDirection: "row", alignItems: "center", gap: 8, + flex: 1, + minWidth: 0, }, statusDot: { width: 8, @@ -145,6 +147,11 @@ export const styles = StyleSheet.create({ fontWeight: "700", color: colors.text, }, + cardActions: { + flexDirection: "row", + alignItems: "center", + gap: 8, + }, cardMeta: { marginTop: 6, gap: 2, diff --git a/packages/mobile/components/ProviderStatsBreakdown.tsx b/packages/mobile/components/ProviderStatsBreakdown.tsx index 8b5cfa96db..daa68193b1 100644 --- a/packages/mobile/components/ProviderStatsBreakdown.tsx +++ b/packages/mobile/components/ProviderStatsBreakdown.tsx @@ -1,8 +1,4 @@ -import { - type ProviderStats, - providerStatsBreakdown, - providerStatsTotal, -} from "@dofek/providers/provider-stats"; +import { type ProviderStats, providerStatsBreakdown } from "@dofek/providers/provider-stats"; import { StyleSheet, Text, View } from "react-native"; import { colors } from "../theme"; @@ -26,7 +22,7 @@ export function ProviderStatsBreakdown({ stats: ProviderStats; variant?: "compact" | "full"; }) { - const total = providerStatsTotal(stats); + const total = stats.totalRecords ?? 0; const breakdown = providerStatsBreakdown(stats); if (total === 0) return null; @@ -48,10 +44,18 @@ export function ProviderStatsBreakdown({ } return ( - - {breakdown.map((b) => ( - - ))} + + + {total.toLocaleString()} + records + + {breakdown.length > 1 && ( + + {breakdown.map((b) => ( + + ))} + + )} ); } @@ -82,15 +86,27 @@ const styles = StyleSheet.create({ flexWrap: "wrap", gap: 8, }, - compactRow: { - flexDirection: "row", - flexWrap: "wrap", + compactContainer: { gap: 8, marginTop: 10, paddingTop: 10, borderTopWidth: StyleSheet.hairlineWidth, borderTopColor: colors.surfaceSecondary, }, + compactTotalRow: { + flexDirection: "row", + alignItems: "baseline", + gap: 4, + }, + compactTotalCount: { + fontSize: 16, + fontWeight: "700", + color: colors.text, + }, + compactTotalLabel: { + fontSize: 12, + color: colors.textSecondary, + }, statBadge: { flexDirection: "row", alignItems: "center", diff --git a/packages/mobile/lib/share-import.test.ts b/packages/mobile/lib/share-import.test.ts index 6144bda59d..d074add659 100644 --- a/packages/mobile/lib/share-import.test.ts +++ b/packages/mobile/lib/share-import.test.ts @@ -41,9 +41,174 @@ describe("inferImportProviderFromFile", () => { }); expect(provider).toBe("apple-health"); }); + + it("detects Garmin dump before generic zip handling", () => { + const provider = inferImportProviderFromFile({ + fileName: "garmin-export.zip", + fileExtension: ".zip", + mimeType: "application/zip", + csvHeaderLine: "", + }); + expect(provider).toBe("garmin-dump"); + }); + + it("detects Kaya CSV by filename", () => { + const provider = inferImportProviderFromFile({ + fileName: "kaya-export.csv", + fileExtension: ".csv", + mimeType: "text/csv", + csvHeaderLine: "Date,Climb,Grade", + }); + expect(provider).toBe("kaya-export"); + }); }); describe("importSharedFile", () => { + it("uploads an explicitly selected Garmin dump ZIP", async () => { + const fetchImpl = vi.fn(); + const customReadBlob = vi + .fn() + .mockResolvedValue(new Blob(["garmin-export"], { type: "application/zip" })); + + fetchImpl + .mockResolvedValueOnce( + new Response(JSON.stringify({ status: "processing", jobId: "job-garmin" }), { + status: 200, + headers: { "content-type": "application/json" }, + }), + ) + .mockResolvedValueOnce( + new Response( + JSON.stringify({ status: "processing", progress: 64, message: "Importing activities" }), + { + status: 200, + headers: { "content-type": "application/json" }, + }, + ), + ) + .mockResolvedValueOnce( + new Response(JSON.stringify({ status: "done", progress: 100 }), { + status: 200, + headers: { "content-type": "application/json" }, + }), + ); + + const progressMessages: string[] = []; + const result = await importSharedFile( + { + fileUri: "file:///tmp/export.zip", + providerId: "garmin-dump", + serverUrl: "https://example.com", + sessionToken: "session-token", + onProgress: (progress) => progressMessages.push(progress.message), + }, + { + fetchImpl, + readBlob: customReadBlob, + sleep: async () => {}, + now: () => 123, + }, + ); + + expect(result).toEqual({ providerId: "garmin-dump", jobId: "job-garmin" }); + expect(fetchImpl).toHaveBeenNthCalledWith( + 1, + "https://example.com/api/upload/garmin-dump", + expect.objectContaining({ + headers: expect.objectContaining({ + Authorization: "Bearer session-token", + "Content-Type": "application/octet-stream", + "x-upload-id": expect.stringMatching(/^share-123-/), + "x-chunk-index": "0", + "x-chunk-total": "1", + "x-file-ext": ".zip", + }), + }), + ); + expect(fetchImpl).toHaveBeenNthCalledWith( + 2, + "https://example.com/api/upload/garmin-dump/status/job-garmin", + expect.anything(), + ); + expect(progressMessages).toContain("Importing activities"); + }); + + it("splits large Garmin dumps into shared upload chunks", async () => { + const fetchImpl = vi.fn(); + const chunkSize = 50 * 1024 * 1024; + const largeBlob = new Blob(["garmin-export"], { type: "application/zip" }); + Object.defineProperty(largeBlob, "size", { value: chunkSize + 1 }); + const slice = vi + .spyOn(largeBlob, "slice") + .mockImplementation(() => new Blob(["chunk"], { type: "application/octet-stream" })); + const customReadBlob = vi.fn().mockResolvedValue(largeBlob); + + fetchImpl + .mockResolvedValueOnce( + new Response(JSON.stringify({ status: "uploading", jobId: "large-upload" }), { + status: 200, + headers: { "content-type": "application/json" }, + }), + ) + .mockResolvedValueOnce( + new Response(JSON.stringify({ status: "assembling", jobId: "large-upload" }), { + status: 200, + headers: { "content-type": "application/json" }, + }), + ) + .mockResolvedValueOnce( + new Response(JSON.stringify({ status: "done", progress: 100 }), { + status: 200, + headers: { "content-type": "application/json" }, + }), + ); + + const result = await importSharedFile( + { + fileUri: "file:///tmp/garmin-export.zip", + providerId: "garmin-dump", + serverUrl: "https://example.com", + sessionToken: "session-token", + }, + { + fetchImpl, + readBlob: customReadBlob, + sleep: async () => {}, + now: () => 999, + }, + ); + + expect(result).toEqual({ providerId: "garmin-dump", jobId: "large-upload" }); + expect(slice).toHaveBeenNthCalledWith(1, 0, chunkSize); + expect(slice).toHaveBeenNthCalledWith(2, chunkSize, chunkSize + 1); + const firstChunkHeaders = new Headers(fetchImpl.mock.calls[0]?.[1]?.headers); + const secondChunkHeaders = new Headers(fetchImpl.mock.calls[1]?.[1]?.headers); + expect(firstChunkHeaders.get("x-upload-id")).toMatch(/^share-999-/); + expect(secondChunkHeaders.get("x-upload-id")).toBe(firstChunkHeaders.get("x-upload-id")); + expect(fetchImpl).toHaveBeenNthCalledWith( + 1, + "https://example.com/api/upload/garmin-dump", + expect.objectContaining({ + headers: expect.objectContaining({ + "x-chunk-index": "0", + "x-chunk-total": "2", + "x-file-ext": ".zip", + }), + }), + ); + expect(fetchImpl).toHaveBeenNthCalledWith( + 2, + "https://example.com/api/upload/garmin-dump", + expect.objectContaining({ + headers: expect.objectContaining({ + "x-chunk-index": "1", + "x-chunk-total": "2", + "x-file-ext": ".zip", + }), + }), + ); + }); + it("uploads a Strong CSV and polls until done", async () => { const fetchImpl = vi.fn(); const fileBody = "Date,Workout Name,Duration,Exercise Name\n2026-03-10,Leg Day,00:45:00,Squat"; @@ -87,6 +252,7 @@ describe("importSharedFile", () => { { fetchImpl, sleep: async () => {}, + now: () => 456, }, ); @@ -101,7 +267,65 @@ describe("importSharedFile", () => { 2, "https://example.com/api/upload/strong-csv?units=kg", expect.objectContaining({ - headers: expect.objectContaining({ Authorization: "Bearer session-token" }), + headers: expect.objectContaining({ + Authorization: "Bearer session-token", + "Content-Type": "application/octet-stream", + "x-upload-id": expect.stringMatching(/^share-456-/), + "x-chunk-index": "0", + "x-chunk-total": "1", + "x-file-ext": ".csv", + }), + }), + ); + }); + + it("uploads an explicitly selected Kaya CSV through the shared chunked path", async () => { + const fetchImpl = vi.fn(); + const fileBody = "Date,Climb,Grade\n2026-03-10,Test Boulder,V5"; + const customReadBlob = vi.fn().mockResolvedValue(new Blob([fileBody], { type: "text/csv" })); + + fetchImpl + .mockResolvedValueOnce( + new Response(JSON.stringify({ status: "processing", jobId: "job-kaya" }), { + status: 200, + headers: { "content-type": "application/json" }, + }), + ) + .mockResolvedValueOnce( + new Response(JSON.stringify({ status: "done", progress: 100 }), { + status: 200, + headers: { "content-type": "application/json" }, + }), + ); + + const result = await importSharedFile( + { + fileUri: "file:///tmp/kaya-export.csv", + providerId: "kaya-export", + serverUrl: "https://example.com", + sessionToken: "session-token", + }, + { + fetchImpl, + readBlob: customReadBlob, + sleep: async () => {}, + now: () => 567, + }, + ); + + expect(result).toEqual({ providerId: "kaya-export", jobId: "job-kaya" }); + expect(fetchImpl).toHaveBeenNthCalledWith( + 1, + "https://example.com/api/upload/kaya-export", + expect.objectContaining({ + headers: expect.objectContaining({ + Authorization: "Bearer session-token", + "Content-Type": "application/octet-stream", + "x-upload-id": expect.stringMatching(/^share-567-/), + "x-chunk-index": "0", + "x-chunk-total": "1", + "x-file-ext": ".csv", + }), }), ); }); diff --git a/packages/mobile/lib/share-import.ts b/packages/mobile/lib/share-import.ts index 09f72a0808..7c645dc981 100644 --- a/packages/mobile/lib/share-import.ts +++ b/packages/mobile/lib/share-import.ts @@ -1,7 +1,13 @@ import { z } from "zod"; import { captureException } from "./telemetry"; -const importProviderIds = ["apple-health", "strong-csv", "cronometer-csv"] as const; +const importProviderIds = [ + "apple-health", + "strong-csv", + "cronometer-csv", + "garmin-dump", + "kaya-export", +] as const; export type ImportProviderId = (typeof importProviderIds)[number]; @@ -21,6 +27,7 @@ export interface ShareImportProgress { export interface ImportSharedFileArgs { fileUri: string; + providerId?: ImportProviderId; serverUrl: string; sessionToken: string; onProgress?: (progress: ShareImportProgress) => void; @@ -99,6 +106,16 @@ function getUploadTarget(serverUrl: string, providerId: ImportProviderId): Uploa uploadUrl: `${baseUrl}/api/upload/cronometer-csv`, statusUrl: `${baseUrl}/api/upload/cronometer-csv/status`, }; + case "garmin-dump": + return { + uploadUrl: `${baseUrl}/api/upload/garmin-dump`, + statusUrl: `${baseUrl}/api/upload/garmin-dump/status`, + }; + case "kaya-export": + return { + uploadUrl: `${baseUrl}/api/upload/kaya-export`, + statusUrl: `${baseUrl}/api/upload/kaya-export/status`, + }; } } @@ -138,13 +155,6 @@ async function parseJsonResponse(response: Response, source: string): Promise void, @@ -263,6 +299,7 @@ async function uploadAppleHealthFile( const chunkSize = 50 * 1024 * 1024; const totalChunks = Math.max(1, Math.ceil(blob.size / chunkSize)); const uploadId = `share-${(now ?? Date.now)()}-${Math.random().toString(36).slice(2, 8)}`; + const uploadFileExtension = getChunkedUploadFileExtension(providerId, fileExtension); let jobId: string | null = null; for (let chunkIndex = 0; chunkIndex < totalChunks; chunkIndex++) { @@ -276,7 +313,7 @@ async function uploadAppleHealthFile( totalChunks > 1 ? `Uploading chunk ${chunkIndex + 1} of ${totalChunks}...` : "Uploading file...", - providerId: "apple-health", + providerId, }); const response = await fetchImpl(target.uploadUrl, { @@ -287,7 +324,7 @@ async function uploadAppleHealthFile( "x-upload-id": uploadId, "x-chunk-index": String(chunkIndex), "x-chunk-total": String(totalChunks), - "x-file-ext": fileExtension === ".xml" ? ".xml" : ".zip", + "x-file-ext": uploadFileExtension, }, body: chunk, }); @@ -302,35 +339,6 @@ async function uploadAppleHealthFile( return jobId; } -async function uploadSingleFile( - fetchImpl: typeof fetch, - target: UploadTarget, - blob: Blob, - providerId: ImportProviderId, - fileExtension: string, - sessionToken: string, - onProgress?: (progress: ShareImportProgress) => void, -): Promise { - onProgress?.({ - status: "uploading", - progress: 25, - message: "Uploading file...", - providerId, - }); - - const response = await fetchImpl(target.uploadUrl, { - method: "POST", - headers: { - Authorization: `Bearer ${sessionToken}`, - "Content-Type": getContentTypeForUpload(providerId, fileExtension), - }, - body: blob, - }); - - const result = await parseUploadResponse(response); - return result.jobId; -} - async function pollImportStatus( fetchImpl: typeof fetch, sleep: (milliseconds: number) => Promise, @@ -405,6 +413,7 @@ export async function importSharedFile( status: "reading", progress: 0, message: `Preparing ${fileName}...`, + providerId: args.providerId, }); try { @@ -417,38 +426,30 @@ export async function importSharedFile( ? getCsvHeaderLine(await readBlobText(blob, args.fileUri, fetchImpl)) : ""; - const providerId = inferImportProviderFromFile({ - fileName, - fileExtension, - mimeType, - csvHeaderLine, - }); + const providerId = + args.providerId ?? + inferImportProviderFromFile({ + fileName, + fileExtension, + mimeType, + csvHeaderLine, + }); if (!providerId) { throw new Error("Unsupported shared file type"); } const target = getUploadTarget(args.serverUrl, providerId); - const jobId = - providerId === "apple-health" - ? await uploadAppleHealthFile( - fetchImpl, - target, - blob, - fileExtension, - args.sessionToken, - args.onProgress, - deps.now, - ) - : await uploadSingleFile( - fetchImpl, - target, - blob, - providerId, - fileExtension, - args.sessionToken, - args.onProgress, - ); + const jobId = await uploadChunkedFile( + fetchImpl, + target, + blob, + providerId, + fileExtension, + args.sessionToken, + args.onProgress, + deps.now, + ); args.onProgress?.({ status: "processing", diff --git a/packages/mobile/package.json b/packages/mobile/package.json index de9add7531..8952813cb7 100644 --- a/packages/mobile/package.json +++ b/packages/mobile/package.json @@ -62,6 +62,7 @@ "expo-build-properties": "57.0.3", "expo-camera": "57.0.1", "expo-dev-client": "~57.0.5", + "expo-document-picker": "~57.0.0", "expo-file-system": "57.0.0", "expo-haptics": "57.0.0", "expo-linking": "57.0.2", diff --git a/packages/providers-meta/src/provider-stats.test.ts b/packages/providers-meta/src/provider-stats.test.ts index c769225450..b3b234a412 100644 --- a/packages/providers-meta/src/provider-stats.test.ts +++ b/packages/providers-meta/src/provider-stats.test.ts @@ -78,6 +78,10 @@ describe("providerStatsTotal", () => { it("returns the single non-zero value for sparse stats", () => { expect(providerStatsTotal(SPARSE_STATS)).toBe(42); }); + + it("uses the API-provided total when present", () => { + expect(providerStatsTotal({ ...SPARSE_STATS, totalRecords: 100 })).toBe(100); + }); }); describe("providerStatsBreakdown", () => { diff --git a/packages/providers-meta/src/provider-stats.ts b/packages/providers-meta/src/provider-stats.ts index 0b7a20e659..a14a42dca3 100644 --- a/packages/providers-meta/src/provider-stats.ts +++ b/packages/providers-meta/src/provider-stats.ts @@ -1,5 +1,6 @@ /** Per-provider record counts across all data types. */ export interface ProviderStats { + totalRecords?: number; activities: number; dailyMetrics: number; sleepSessions: number; @@ -15,7 +16,7 @@ export interface ProviderStats { /** Ordered mapping of stat keys to human-readable labels, used for display. */ export const DATA_TYPE_LABELS: ReadonlyArray<{ - key: keyof ProviderStats; + key: Exclude; label: string; }> = [ { key: "activities", label: "Activities" }, @@ -33,6 +34,10 @@ export const DATA_TYPE_LABELS: ReadonlyArray<{ /** Sum of all record counts for a provider. */ export function providerStatsTotal(stats: ProviderStats): number { + if (stats.totalRecords !== undefined) { + return stats.totalRecords; + } + let total = 0; for (const { key } of DATA_TYPE_LABELS) { total += stats[key]; diff --git a/packages/server/src/index.integration.test.ts b/packages/server/src/index.integration.test.ts index eb046b37f8..d160222997 100644 --- a/packages/server/src/index.integration.test.ts +++ b/packages/server/src/index.integration.test.ts @@ -474,8 +474,8 @@ describe("tRPC API", () => { if (i === 0) { expect(data.status).toBe("uploading"); } else { - // Last chunk responds immediately with "assembling" status - expect(data.status).toBe("assembling"); + // Last chunk returns after the assembled file is enqueued + expect(data.status).toBe("processing"); // jobId is the upload ID for seamless polling expect(data.jobId).toBe(uploadId); } diff --git a/packages/server/src/lib/server-utils.test.ts b/packages/server/src/lib/server-utils.test.ts index 46036d29dd..11f3ef865f 100644 --- a/packages/server/src/lib/server-utils.test.ts +++ b/packages/server/src/lib/server-utils.test.ts @@ -45,7 +45,7 @@ describe("streamToFile", () => { url: "/test", }); - await streamToFile(req, filePath); + await expect(streamToFile(req, filePath)).resolves.toBe(body.length); const contents = await readFile(filePath, "utf-8"); expect(contents).toBe("hello world"); }); diff --git a/packages/server/src/lib/server-utils.ts b/packages/server/src/lib/server-utils.ts index 9fefd863fa..575260e1af 100644 --- a/packages/server/src/lib/server-utils.ts +++ b/packages/server/src/lib/server-utils.ts @@ -15,7 +15,7 @@ export function streamToFile( req: Readable, filePath: string, maxBytes = MAX_UPLOAD_BYTES, -): Promise { +): Promise { return new Promise((resolve, reject) => { let bytesReceived = 0; const ws = createWriteStream(filePath); @@ -28,7 +28,7 @@ export function streamToFile( }); req.pipe(ws); - ws.on("finish", resolve); + ws.on("finish", () => resolve(bytesReceived)); ws.on("error", reject); req.on("error", reject); }); diff --git a/packages/server/src/lib/upload-state-store-redis.test.ts b/packages/server/src/lib/upload-state-store-redis.test.ts index 24347abdbb..5fd43336fc 100644 --- a/packages/server/src/lib/upload-state-store-redis.test.ts +++ b/packages/server/src/lib/upload-state-store-redis.test.ts @@ -5,6 +5,7 @@ const originalEnv = { ...process.env }; function createFakeRedisClient() { const values = new Map(); const sets = new Map>(); + const hashes = new Map>(); const ttlSeconds = new Map(); return { @@ -21,6 +22,7 @@ function createFakeRedisClient() { for (const key of keys) { if (values.delete(key)) deleted++; if (sets.delete(key)) deleted++; + if (hashes.delete(key)) deleted++; ttlSeconds.delete(key); } return deleted; @@ -42,9 +44,35 @@ function createFakeRedisClient() { ttlSeconds.set(key, seconds); return 1; }, + async eval(_script: string, _keyCount: number, ...args: Array) { + const chunkBytesKey = String(args[0]); + const chunksKey = String(args[1]); + const chunkIndex = String(args[2]); + const byteCount = Number(args[3]); + const maxBytes = Number(args[4]); + const ttlSecondsValue = Number(args[6]); + const chunkBytes = hashes.get(chunkBytesKey) ?? new Map(); + const previousBytes = chunkBytes.get(chunkIndex) ?? 0; + let currentBytes = 0; + for (const value of chunkBytes.values()) { + currentBytes += value; + } + const nextBytes = currentBytes - previousBytes + byteCount; + const chunks = sets.get(chunksKey) ?? new Set(); + if (nextBytes > maxBytes) { + return [0, chunks.size, currentBytes]; + } + chunkBytes.set(chunkIndex, byteCount); + hashes.set(chunkBytesKey, chunkBytes); + chunks.add(chunkIndex); + sets.set(chunksKey, chunks); + ttlSeconds.set(chunksKey, ttlSecondsValue); + return [1, chunks.size, nextBytes]; + }, }, values, sets, + hashes, ttlSeconds, }; } @@ -86,6 +114,27 @@ describe("RedisUploadStateStore", () => { userId: "user-1", }); expect(fakeRedis.ttlSeconds.get("upload-chunks:upload-1")).toBe(1800); + expect(fakeRedis.ttlSeconds.get("upload-chunk-bytes:upload-1")).toBe(1800); + }); + + it("tracks chunk byte totals atomically in Redis", async () => { + const fakeRedis = createFakeRedisClient(); + const module = await import("./upload-state-store.ts"); + const store = new module.RedisUploadStateStore(async () => fakeRedis.client); + + await store.saveUploadSession("upload-1", { total: 3, dir: "/tmp/upload-1", userId: "user-1" }); + + await expect(store.recordReceivedChunk("upload-1", 0, 4, 10)).resolves.toEqual({ + accepted: true, + receivedCount: 1, + receivedBytes: 4, + }); + await expect(store.recordReceivedChunk("upload-1", 1, 7, 10)).resolves.toEqual({ + accepted: false, + receivedCount: 1, + receivedBytes: 4, + }); + await expect(store.getReceivedChunks("upload-1")).resolves.toEqual([0]); }); it("cleans up invalid upload payloads", async () => { diff --git a/packages/server/src/lib/upload-state-store.test.ts b/packages/server/src/lib/upload-state-store.test.ts index 7e12797394..d3365243e6 100644 --- a/packages/server/src/lib/upload-state-store.test.ts +++ b/packages/server/src/lib/upload-state-store.test.ts @@ -27,6 +27,26 @@ describe("InMemoryUploadStateStore", () => { expect(await store.getReceivedChunks("upload-1")).toEqual([0, 2]); }); + it("tracks chunk byte totals and rejects oversized uploads", async () => { + await store.saveUploadSession("upload-1", { + total: 3, + dir: "/tmp/upload-1", + userId: "user-1", + }); + + await expect(store.recordReceivedChunk("upload-1", 0, 4, 10)).resolves.toEqual({ + accepted: true, + receivedCount: 1, + receivedBytes: 4, + }); + await expect(store.recordReceivedChunk("upload-1", 1, 7, 10)).resolves.toEqual({ + accepted: false, + receivedCount: 1, + receivedBytes: 4, + }); + await expect(store.getReceivedChunks("upload-1")).resolves.toEqual([0]); + }); + it("stores and deletes upload statuses", async () => { const status: UploadStatus = { status: "uploading", diff --git a/packages/server/src/lib/upload-state-store.ts b/packages/server/src/lib/upload-state-store.ts index d87bce8d89..48a4d91b2c 100644 --- a/packages/server/src/lib/upload-state-store.ts +++ b/packages/server/src/lib/upload-state-store.ts @@ -2,6 +2,8 @@ import { RedisConnection } from "bullmq"; import { getRedisConnection } from "dofek/jobs/queues"; import { z } from "zod"; +// cspell:words HGET HVALS HSET ipairs + export const UPLOAD_SESSION_TTL_MS = 30 * 60 * 1000; export const UPLOAD_STATUS_TTL_MS = 10 * 60 * 1000; @@ -23,6 +25,12 @@ const uploadStatusSchema = z.object({ export type UploadStatus = z.infer; +export interface RecordedChunk { + accepted: boolean; + receivedCount: number; + receivedBytes: number; +} + interface RedisClient { set(key: string, value: string, mode: "PX", millisecondsToExpire: number): Promise<"OK" | null>; get(key: string): Promise; @@ -31,6 +39,7 @@ interface RedisClient { scard(key: string): Promise; smembers(key: string): Promise; expire(key: string, seconds: number): Promise; + eval(script: string, keyCount: number, ...args: Array): Promise; } export interface UploadStateStore { @@ -38,6 +47,13 @@ export interface UploadStateStore { getUploadSession(uploadId: string): Promise; deleteUploadSession(uploadId: string): Promise; addReceivedChunk(uploadId: string, chunkIndex: number, timeToLiveMs?: number): Promise; + recordReceivedChunk( + uploadId: string, + chunkIndex: number, + byteCount: number, + maxBytes: number, + timeToLiveMs?: number, + ): Promise; getReceivedChunkCount(uploadId: string): Promise; getReceivedChunks(uploadId: string): Promise; saveUploadStatus(uploadId: string, status: UploadStatus, timeToLiveMs?: number): Promise; @@ -53,6 +69,10 @@ function uploadChunksKey(uploadId: string): string { return `upload-chunks:${uploadId}`; } +function uploadChunkBytesKey(uploadId: string): string { + return `upload-chunk-bytes:${uploadId}`; +} + function uploadStatusKey(uploadId: string): string { return `upload-status:${uploadId}`; } @@ -64,6 +84,7 @@ function ttlSeconds(timeToLiveMs: number): number { export class InMemoryUploadStateStore implements UploadStateStore { #sessions = new Map(); #chunkSets = new Map; expiresAt: number }>(); + #chunkBytes = new Map; expiresAt: number }>(); #statuses = new Map(); async saveUploadSession( @@ -81,6 +102,7 @@ export class InMemoryUploadStateStore implements UploadStateStore { async deleteUploadSession(uploadId: string): Promise { this.#sessions.delete(uploadId); this.#chunkSets.delete(uploadId); + this.#chunkBytes.delete(uploadId); } async addReceivedChunk( @@ -95,6 +117,43 @@ export class InMemoryUploadStateStore implements UploadStateStore { return next.size; } + async recordReceivedChunk( + uploadId: string, + chunkIndex: number, + byteCount: number, + maxBytes: number, + timeToLiveMs = UPLOAD_SESSION_TTL_MS, + ): Promise { + const existingBytes = this.#getFresh(this.#chunkBytes, uploadId); + const chunkBytes = existingBytes?.value ?? new Map(); + const previousBytes = chunkBytes.get(chunkIndex) ?? 0; + let receivedBytes = 0; + for (const value of chunkBytes.values()) { + receivedBytes += value; + } + const nextReceivedBytes = receivedBytes - previousBytes + byteCount; + const existingChunks = this.#getFresh(this.#chunkSets, uploadId); + const chunks = existingChunks?.value ?? new Set(); + if (nextReceivedBytes > maxBytes) { + return { + accepted: false, + receivedCount: chunks.size, + receivedBytes, + }; + } + + chunkBytes.set(chunkIndex, byteCount); + chunks.add(chunkIndex); + const expiresAt = Date.now() + timeToLiveMs; + this.#chunkBytes.set(uploadId, { value: chunkBytes, expiresAt }); + this.#chunkSets.set(uploadId, { value: chunks, expiresAt }); + return { + accepted: true, + receivedCount: chunks.size, + receivedBytes: nextReceivedBytes, + }; + } + async getReceivedChunkCount(uploadId: string): Promise { return this.#getFresh(this.#chunkSets, uploadId)?.value.size ?? 0; } @@ -149,6 +208,7 @@ export class RedisUploadStateStore implements UploadStateStore { const client = await this.#getRedisClient(); await client.set(uploadSessionKey(uploadId), JSON.stringify(session), "PX", timeToLiveMs); await client.expire(uploadChunksKey(uploadId), ttlSeconds(timeToLiveMs)); + await client.expire(uploadChunkBytesKey(uploadId), ttlSeconds(timeToLiveMs)); } async getUploadSession(uploadId: string): Promise { @@ -172,7 +232,11 @@ export class RedisUploadStateStore implements UploadStateStore { async deleteUploadSession(uploadId: string): Promise { const client = await this.#getRedisClient(); - await client.del(uploadSessionKey(uploadId), uploadChunksKey(uploadId)); + await client.del( + uploadSessionKey(uploadId), + uploadChunksKey(uploadId), + uploadChunkBytesKey(uploadId), + ); } async addReceivedChunk( @@ -187,6 +251,57 @@ export class RedisUploadStateStore implements UploadStateStore { return client.scard(key); } + async recordReceivedChunk( + uploadId: string, + chunkIndex: number, + byteCount: number, + maxBytes: number, + timeToLiveMs = UPLOAD_SESSION_TTL_MS, + ): Promise { + const client = await this.#getRedisClient(); + const result = await client.eval( + ` + local chunk_bytes_key = KEYS[1] + local chunks_key = KEYS[2] + local chunk_index = ARGV[1] + local byte_count = tonumber(ARGV[2]) + local max_bytes = tonumber(ARGV[3]) + local ttl_ms = tonumber(ARGV[4]) + local ttl_seconds = tonumber(ARGV[5]) + local previous_bytes = tonumber(redis.call("HGET", chunk_bytes_key, chunk_index) or "0") + local current_bytes = 0 + local all_bytes = redis.call("HVALS", chunk_bytes_key) + for _, value in ipairs(all_bytes) do + current_bytes = current_bytes + tonumber(value) + end + local next_bytes = current_bytes - previous_bytes + byte_count + local received_count = redis.call("SCARD", chunks_key) + if next_bytes > max_bytes then + return {0, received_count, current_bytes} + end + redis.call("HSET", chunk_bytes_key, chunk_index, byte_count) + redis.call("SADD", chunks_key, chunk_index) + redis.call("PEXPIRE", chunk_bytes_key, ttl_ms) + redis.call("EXPIRE", chunks_key, ttl_seconds) + return {1, redis.call("SCARD", chunks_key), next_bytes} + `, + 2, + uploadChunkBytesKey(uploadId), + uploadChunksKey(uploadId), + String(chunkIndex), + byteCount, + maxBytes, + timeToLiveMs, + ttlSeconds(timeToLiveMs), + ); + const parsed = z.tuple([z.number(), z.number(), z.number()]).parse(result); + return { + accepted: parsed[0] === 1, + receivedCount: parsed[1], + receivedBytes: parsed[2], + }; + } + async getReceivedChunkCount(uploadId: string): Promise { const client = await this.#getRedisClient(); return client.scard(uploadChunksKey(uploadId)); @@ -256,6 +371,7 @@ async function getSharedRedisClient(): Promise { scard: async (key) => redisClient.scard(key), smembers: async (key) => redisClient.smembers(key), expire: async (key, seconds) => redisClient.expire(key, seconds), + eval: async (script, keyCount, ...args) => redisClient.eval(script, keyCount, ...args), }; } diff --git a/packages/server/src/repositories/sync-repository.test.ts b/packages/server/src/repositories/sync-repository.test.ts index 083bfac3d5..22eef2e072 100644 --- a/packages/server/src/repositories/sync-repository.test.ts +++ b/packages/server/src/repositories/sync-repository.test.ts @@ -207,6 +207,7 @@ describe("SyncRepository", () => { expect(result).toHaveLength(1); expect(result[0]).toEqual({ providerId: "wahoo", + totalRecords: 168, activities: 5, dailyMetrics: 30, sleepSessions: 0, diff --git a/packages/server/src/repositories/sync-repository.ts b/packages/server/src/repositories/sync-repository.ts index 8ebc94567a..8a1d972cc4 100644 --- a/packages/server/src/repositories/sync-repository.ts +++ b/packages/server/src/repositories/sync-repository.ts @@ -122,6 +122,7 @@ export interface LatestError { export interface ProviderStatRow { providerId: string; + totalRecords: number; activities: number; dailyMetrics: number; sleepSessions: number; @@ -338,6 +339,18 @@ export class SyncRepository { return rows.map((row) => ({ providerId: row.provider_id, + totalRecords: + row.activities + + row.daily_metrics + + row.sleep_sessions + + row.body_measurements + + row.food_entries + + row.health_events + + row.metric_stream + + row.nutrition_daily + + row.lab_panels + + row.lab_results + + row.journal_entries, activities: row.activities, dailyMetrics: row.daily_metrics, sleepSessions: row.sleep_sessions, diff --git a/packages/server/src/routers/sync.test.ts b/packages/server/src/routers/sync.test.ts index 659608e364..c4da5c0366 100644 --- a/packages/server/src/routers/sync.test.ts +++ b/packages/server/src/routers/sync.test.ts @@ -1755,6 +1755,128 @@ describe("syncRouter", () => { }); }); + describe("activeImports", () => { + it("returns progress for current-user import jobs", async () => { + mockImportQueueGetJobs.mockResolvedValueOnce([ + { + id: "job-garmin", + data: { + userId: "user-1", + importType: "garmin-dump", + filePath: "/tmp/garmin.zip", + since: "1970-01-01T00:00:00.000Z", + }, + progress: { percentage: 64, message: "Importing activities" }, + getState: vi.fn().mockResolvedValue("active"), + }, + { + id: "job-other-user", + data: { + userId: "user-2", + importType: "garmin-dump", + filePath: "/tmp/other.zip", + since: "1970-01-01T00:00:00.000Z", + }, + progress: 20, + getState: vi.fn().mockResolvedValue("active"), + }, + ]); + + const caller = createCaller({ + db: { execute: vi.fn().mockResolvedValue([]) }, + userId: "user-1", + timezone: "UTC", + }); + + const result = await caller.activeImports(); + + expect(mockImportQueueGetJobs).toHaveBeenCalledWith(["active", "waiting", "delayed"]); + expect(result).toEqual([ + { + jobId: "job-garmin", + providerId: "garmin-dump", + progress: 64, + message: "Importing activities", + }, + ]); + }); + + it("uses queued defaults and skips unknown import types", async () => { + mockImportQueueGetJobs.mockResolvedValueOnce([ + { + id: "job-waiting", + data: { + userId: "user-1", + importType: "garmin-dump", + filePath: "/tmp/garmin.zip", + since: "1970-01-01T00:00:00.000Z", + }, + progress: undefined, + getState: vi.fn().mockResolvedValue("waiting"), + }, + { + id: "job-unknown", + data: { userId: "user-1", importType: "unknown" }, + progress: 10, + getState: vi.fn().mockResolvedValue("active"), + }, + ]); + + const caller = createCaller({ + db: { execute: vi.fn().mockResolvedValue([]) }, + userId: "user-1", + timezone: "UTC", + }); + + await expect(caller.activeImports()).resolves.toEqual([ + { + jobId: "job-waiting", + providerId: "garmin-dump", + progress: 0, + message: "Waiting to import...", + }, + ]); + }); + + it("uses a stable fallback jobId when an import job has no id", async () => { + mockImportQueueGetJobs.mockResolvedValueOnce([ + { + id: undefined, + data: { + userId: "user-1", + importType: "garmin-dump", + filePath: "/tmp/garmin.zip", + since: "1970-01-01T00:00:00.000Z", + }, + progress: 20, + getState: vi.fn().mockResolvedValue("active"), + }, + ]); + + const caller = createCaller({ + db: { execute: vi.fn().mockResolvedValue([]) }, + userId: "user-1", + timezone: "UTC", + }); + + const result = await caller.activeImports(); + expect(result[0]?.jobId).toBe("job-garmin-dump"); + }); + + it("surfaces an actionable error when the import queue is unavailable", async () => { + mockImportQueueGetJobs.mockRejectedValueOnce(new Error("Redis connection refused")); + const caller = createCaller({ + db: { execute: vi.fn().mockResolvedValue([]) }, + userId: "user-1", + timezone: "UTC", + }); + + await expect(caller.activeImports()).rejects.toThrow( + "Unable to check import progress because the queue service is unavailable.", + ); + }); + }); + describe("providerStats", () => { it("maps ClickHouse rows to provider stats", async () => { const execute = vi.fn().mockResolvedValue([]); @@ -1789,6 +1911,7 @@ describe("syncRouter", () => { expect(result).toEqual([ { providerId: "wahoo", + totalRecords: 148, activities: 10, dailyMetrics: 5, sleepSessions: 3, diff --git a/packages/server/src/routers/sync.ts b/packages/server/src/routers/sync.ts index ee64cca7aa..882dcd2430 100644 --- a/packages/server/src/routers/sync.ts +++ b/packages/server/src/routers/sync.ts @@ -8,6 +8,7 @@ import { getImportQueue, getProviderSyncQueue, IMPORT_QUEUE, + type ImportJobData, providerSyncQueueName, type SyncJobData, } from "dofek/jobs/queues"; @@ -111,6 +112,7 @@ export const logsInput = z.object({ limit: z.number().default(100) }); const providerStatsOutputSchema = z.array( z.object({ providerId: z.string(), + totalRecords: z.number().int().nonnegative(), activities: z.number().int().nonnegative(), dailyMetrics: z.number().int().nonnegative(), sleepSessions: z.number().int().nonnegative(), @@ -125,6 +127,23 @@ const providerStatsOutputSchema = z.array( }), ); +const activeImportsOutputSchema = z.array( + z.object({ + jobId: z.string(), + providerId: z.string(), + progress: z.number(), + message: z.string(), + }), +); + +const importJobProgressSchema = z.union([ + z.number(), + z.object({ + percentage: z.number().optional(), + message: z.string().optional(), + }), +]); + const syncJobDataSchema = z.object({ userId: z.string(), providerId: z.string().optional(), @@ -740,6 +759,50 @@ export const syncRouter = router({ return results; }), + /** Check for active file imports belonging to the current user. */ + activeImports: protectedProcedure.output(activeImportsOutputSchema).query(async ({ ctx }) => { + let jobs: Job[]; + try { + jobs = await getImportQueue().getJobs(["active", "waiting", "delayed"]); + } catch (error: unknown) { + captureException(error); + throw new TRPCError({ + code: "INTERNAL_SERVER_ERROR", + message: "Unable to check import progress because the queue service is unavailable.", + }); + } + + const results: z.infer = []; + for (const job of jobs) { + if (!isJobDataForUser(job.data, ctx.userId)) continue; + const importType = importTypeFromJobData(job.data); + if (!importType) continue; + const providerId = providerIdForImportType(importType); + if (!providerId) continue; + + const state = await job.getState(); + const parsedProgress = importJobProgressSchema.safeParse(job.progress); + const progressValue = parsedProgress.success ? parsedProgress.data : undefined; + const progress = + typeof progressValue === "number" ? progressValue : (progressValue?.percentage ?? 0); + const message = + typeof progressValue === "object" && progressValue.message + ? progressValue.message + : state === "waiting" || state === "delayed" + ? "Waiting to import..." + : "Processing import..."; + + results.push({ + jobId: String(job.id ?? `job-${providerId}`), + providerId, + progress, + message, + }); + } + + return results; + }), + /** Get sync log history */ logs: cachedProtectedQuery({ maxAge: CacheTTL.SHORT }) .input(logsInput) diff --git a/packages/server/src/routes/upload.integration.test.ts b/packages/server/src/routes/upload.integration.test.ts index 0a23c23546..a904409a81 100644 --- a/packages/server/src/routes/upload.integration.test.ts +++ b/packages/server/src/routes/upload.integration.test.ts @@ -184,7 +184,7 @@ describe("upload integration (real file I/O)", () => { expect(JSON.parse(res.body).status).toBe("uploading"); } - // Final chunk — responds immediately with "assembling", assembly happens in background + // Final chunk — assembles and enqueues before returning const res = await post(app, "/api/upload/apple-health", { headers: { "Content-Type": "application/octet-stream", @@ -196,10 +196,8 @@ describe("upload integration (real file I/O)", () => { }); expect(res.status).toBe(200); - expect(JSON.parse(res.body).status).toBe("assembling"); - - // Wait for background assembly + enqueue to complete - await vi.waitFor(() => expect(queue.recorded).toHaveLength(1), { timeout: 2000 }); + expect(JSON.parse(res.body).status).toBe("processing"); + expect(queue.recorded).toHaveLength(1); // Assembled file should contain all chunks concatenated in order const filePath = queue.recorded[0].data.filePath; @@ -233,10 +231,8 @@ describe("upload integration (real file I/O)", () => { }); expect(res.status).toBe(200); - expect(JSON.parse(res.body).status).toBe("assembling"); - - // Wait for background assembly + enqueue to complete - await vi.waitFor(() => expect(queue.recorded).toHaveLength(1), { timeout: 2000 }); + expect(JSON.parse(res.body).status).toBe("processing"); + expect(queue.recorded).toHaveLength(1); // assembleChunks sorts by filename (chunk-000000, chunk-000001), // so the result should be in correct order regardless of upload order diff --git a/packages/server/src/routes/upload.test.ts b/packages/server/src/routes/upload.test.ts index 064980c33c..2ec5d9fbe8 100644 --- a/packages/server/src/routes/upload.test.ts +++ b/packages/server/src/routes/upload.test.ts @@ -1,10 +1,15 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; vi.mock("../lib/server-utils.ts", () => ({ - streamToFile: vi.fn(() => Promise.resolve()), + MAX_UPLOAD_BYTES: 2 * 1024 * 1024 * 1024, + streamToFile: vi.fn(() => Promise.resolve(1)), assembleChunks: vi.fn(() => Promise.resolve()), })); +vi.mock("@sentry/node", () => ({ + captureException: vi.fn(), +})); + vi.mock("../lib/start-worker.ts", () => ({ startWorker: vi.fn(), })); @@ -16,12 +21,16 @@ vi.mock("../logger.ts", () => ({ const mockRm = vi .fn<(path: string, options?: object) => Promise>() .mockResolvedValue(undefined); +const mockRename = vi + .fn<(oldPath: string, newPath: string) => Promise>() + .mockResolvedValue(undefined); const mockUnlink = vi.fn<(path: string) => Promise>().mockResolvedValue(undefined); vi.mock("node:fs/promises", async (importOriginal) => { const actual = await importOriginal(); return { ...actual, rm: (...args: [string, object?]) => mockRm(...args), + rename: (oldPath: string, newPath: string) => mockRename(oldPath, newPath), unlink: (path: string) => mockUnlink(path), }; }); @@ -55,6 +64,7 @@ function mockQueue() { progress: 0, failedReason: null, returnvalue: null, + data: { userId: "user-1" }, }; return { add: vi.fn(() => Promise.resolve(mockJob)), @@ -204,6 +214,10 @@ describe("createUploadRouter", () => { "upload-assembly-err", "upload-rm-assembly-err", "upload-assemble-dir", + "strong-chunked", + "garmin-chunked", + "garmin-too-large", + "garmin-too-many-chunks", ]) { await uploadStateStore.deleteUploadSession(uploadId); await uploadStateStore.deleteUploadStatus(uploadId); @@ -482,6 +496,57 @@ describe("createUploadRouter", () => { undefined, ); }); + + it("accepts chunked CSV uploads through the shared upload flow", async () => { + const { app, queue } = createTestApp(); + + const first = await request(app, "post", "/api/upload/strong-csv?units=lbs", { + headers: { + "Content-Type": "application/octet-stream", + "x-upload-id": "strong-chunked", + "x-chunk-index": "0", + "x-chunk-total": "2", + "x-file-ext": ".csv", + }, + body: Buffer.from("date,exercise\n"), + }); + const second = await request(app, "post", "/api/upload/strong-csv?units=lbs", { + headers: { + "Content-Type": "application/octet-stream", + "x-upload-id": "strong-chunked", + "x-chunk-index": "1", + "x-chunk-total": "2", + "x-file-ext": ".csv", + }, + body: Buffer.from("2026-01-01,squat"), + }); + + expect(first.status).toBe(200); + expect(JSON.parse(first.body)).toMatchObject({ + status: "uploading", + jobId: "strong-chunked", + }); + expect(second.status).toBe(200); + expect(JSON.parse(second.body)).toMatchObject({ + status: "processing", + jobId: "strong-chunked", + }); + await vi.waitFor(() => { + expect(queue.add).toHaveBeenCalledWith( + "strong-csv", + expect.objectContaining({ + importType: "strong-csv", + userId: "user-1", + weightUnit: "lbs", + }), + { jobId: "strong-chunked" }, + ); + }); + expect(assembleChunks).toHaveBeenCalledWith( + expect.not.stringContaining("strong-chunked"), + expect.not.stringContaining("strong-chunked"), + ); + }); }); describe("POST /api/upload/apple-health (chunked)", () => { @@ -578,6 +643,7 @@ describe("createUploadRouter", () => { it("returns failed status for failed job", async () => { const { app, queue } = createTestApp(); const mockJob = { + data: { userId: "user-1" }, getState: vi.fn(() => Promise.resolve("failed")), progress: 0, failedReason: "Out of memory", @@ -594,6 +660,7 @@ describe("createUploadRouter", () => { it("returns processing status for active job", async () => { const { app, queue } = createTestApp(); const mockJob = { + data: { userId: "user-1" }, getState: vi.fn(() => Promise.resolve("active")), progress: { percentage: 50, message: "Parsing records..." }, failedReason: null, @@ -611,6 +678,7 @@ describe("createUploadRouter", () => { it("handles numeric progress", async () => { const { app, queue } = createTestApp(); const mockJob = { + data: { userId: "user-1" }, getState: vi.fn(() => Promise.resolve("active")), progress: 75, failedReason: null, @@ -626,6 +694,7 @@ describe("createUploadRouter", () => { it("handles undefined progress", async () => { const { app, queue } = createTestApp(); const mockJob = { + data: { userId: "user-1" }, getState: vi.fn(() => Promise.resolve("waiting")), progress: null, failedReason: null, @@ -897,7 +966,7 @@ describe("createUploadRouter", () => { }); describe("POST /api/upload/apple-health (chunked - all chunks)", () => { - it("responds immediately with assembling status when all chunks received", async () => { + it("responds with processing status after enqueue when all chunks are received", async () => { const { app } = createTestApp(); // Send chunk 0 of 2 @@ -911,7 +980,7 @@ describe("createUploadRouter", () => { body: Buffer.from("chunk-0"), }); - // Send chunk 1 of 2 (final) — should respond immediately without blocking on assembly + // Send chunk 1 of 2 (final) — should enqueue before returning the job ID const res = await request(app, "post", "/api/upload/apple-health", { headers: { "Content-Type": "application/octet-stream", @@ -924,7 +993,7 @@ describe("createUploadRouter", () => { expect(res.status).toBe(200); const data = JSON.parse(res.body); - expect(data.status).toBe("assembling"); + expect(data.status).toBe("processing"); expect(data.jobId).toBe("upload-full"); }); @@ -1007,14 +1076,11 @@ describe("createUploadRouter", () => { body: Buffer.from("chunk-1"), }); - // Wait for background assembly error + rm error to propagate - await vi.waitFor(() => { - expect(vi.mocked(logger.warn)).toHaveBeenCalledWith( - "Failed to clean up chunk dir %s: %s", - expect.any(String), - expect.any(Error), - ); - }); + expect(vi.mocked(logger.warn)).toHaveBeenCalledWith( + "Failed to clean up chunk dir %s: %s", + expect.any(String), + expect.any(Error), + ); }); it("warns when rm fails during stale upload cleanup", async () => { @@ -1048,7 +1114,7 @@ describe("createUploadRouter", () => { vi.useRealTimers(); }); - it("sets error status when background assembly fails", async () => { + it("sets error status when assembly fails", async () => { vi.mocked(assembleChunks).mockRejectedValueOnce(new Error("disk full")); const { app } = createTestApp(); @@ -1063,7 +1129,7 @@ describe("createUploadRouter", () => { body: Buffer.from("chunk-0"), }); - // Send chunk 1 of 2 (final) — responds immediately + // Send chunk 1 of 2 (final) const res = await request(app, "post", "/api/upload/apple-health", { headers: { "Content-Type": "application/octet-stream", @@ -1074,11 +1140,8 @@ describe("createUploadRouter", () => { body: Buffer.from("chunk-1"), }); - expect(res.status).toBe(200); - expect(JSON.parse(res.body).status).toBe("assembling"); - - // Let background assembly error propagate through microtasks - await new Promise((r) => setTimeout(r, 10)); + expect(res.status).toBe(500); + expect(JSON.parse(res.body)).toEqual({ error: "Failed to assemble uploaded file" }); // Verify error status is set — poll the status endpoint const statusRes = await request( @@ -1115,14 +1178,11 @@ describe("createUploadRouter", () => { body: Buffer.from("chunk-1"), }); - // Wait for background assembly + enqueue to complete. - await vi.waitFor(() => { - expect(queue.add).toHaveBeenCalledWith( - "apple-health", - expect.objectContaining({ importType: "apple-health" }), - { jobId: "upload-jobid-test" }, - ); - }); + expect(queue.add).toHaveBeenCalledWith( + "apple-health", + expect.objectContaining({ importType: "apple-health" }), + { jobId: "upload-jobid-test" }, + ); }); }); @@ -1176,7 +1236,8 @@ describe("createUploadRouter", () => { }); const assembledPath = vi.mocked(assembleChunks).mock.calls[0][1]; expect(assembledPath.startsWith(EXPECTED_JOB_FILES_DIR)).toBe(true); - expect(assembledPath).toContain("apple-health-upload-assemble-dir"); + expect(assembledPath).toContain("apple-health-"); + expect(assembledPath).not.toContain("upload-assemble-dir"); }); it("writes Strong CSV upload to JOB_FILES_DIR", async () => { @@ -1271,9 +1332,7 @@ describe("createUploadRouter", () => { expect.stringContaining("garmin-dump-"), 2 * 1024 * 1024 * 1024, ); - expect(vi.mocked(streamToFile).mock.calls[0][1]).toMatch( - /garmin-dump-job-\d+-[a-z0-9]{4}\.zip$/, - ); + expect(vi.mocked(streamToFile).mock.calls[0][1]).toMatch(/garmin-dump-[a-f0-9-]+\.zip$/); expect(queue.add).toHaveBeenCalledWith( "garmin-dump", expect.objectContaining({ importType: "garmin-dump", userId: "user-1" }), @@ -1356,6 +1415,103 @@ describe("createUploadRouter", () => { expect(streamToFile).toHaveBeenCalled(); }); + it("accepts chunked ZIP uploads through the shared upload flow", async () => { + const { app, queue } = createTestApp(); + + await request(app, "post", "/api/upload/garmin-dump", { + headers: { + "Content-Type": "application/octet-stream", + "x-upload-id": "garmin-chunked", + "x-chunk-index": "0", + "x-chunk-total": "2", + "x-file-ext": ".zip", + }, + body: Buffer.from("PK-1"), + }); + const res = await request(app, "post", "/api/upload/garmin-dump", { + headers: { + "Content-Type": "application/octet-stream", + "x-upload-id": "garmin-chunked", + "x-chunk-index": "1", + "x-chunk-total": "2", + "x-file-ext": ".zip", + }, + body: Buffer.from("PK-2"), + }); + + expect(res.status).toBe(200); + expect(JSON.parse(res.body)).toMatchObject({ + status: "processing", + jobId: "garmin-chunked", + }); + await vi.waitFor(() => { + expect(queue.add).toHaveBeenCalledWith( + "garmin-dump", + expect.objectContaining({ importType: "garmin-dump", userId: "user-1" }), + { jobId: "garmin-chunked" }, + ); + }); + expect(assembleChunks).toHaveBeenCalledWith( + expect.not.stringContaining("garmin-chunked"), + expect.not.stringContaining("garmin-chunked"), + ); + }); + + it("rejects chunked uploads that exceed the route total size limit", async () => { + const { app, queue } = createTestApp(); + const garminLimitBytes = 2 * 1024 * 1024 * 1024; + vi.mocked(streamToFile).mockResolvedValueOnce(garminLimitBytes).mockResolvedValueOnce(1); + + const first = await request(app, "post", "/api/upload/garmin-dump", { + headers: { + "Content-Type": "application/octet-stream", + "x-upload-id": "garmin-too-large", + "x-chunk-index": "0", + "x-chunk-total": "2", + "x-file-ext": ".zip", + }, + body: Buffer.from("PK-1"), + }); + const second = await request(app, "post", "/api/upload/garmin-dump", { + headers: { + "Content-Type": "application/octet-stream", + "x-upload-id": "garmin-too-large", + "x-chunk-index": "1", + "x-chunk-total": "2", + "x-file-ext": ".zip", + }, + body: Buffer.from("PK-2"), + }); + + expect(first.status).toBe(200); + expect(second.status).toBe(413); + expect(JSON.parse(second.body)).toEqual({ error: "Upload exceeds maximum size" }); + expect(queue.add).not.toHaveBeenCalled(); + expect(mockRm).toHaveBeenCalledWith(expect.not.stringContaining("garmin-too-large"), { + recursive: true, + force: true, + }); + }); + + it("rejects chunked uploads with too many chunks", async () => { + const { app } = createTestApp(); + + const res = await request(app, "post", "/api/upload/garmin-dump", { + headers: { + "Content-Type": "application/octet-stream", + "x-upload-id": "garmin-too-many-chunks", + "x-chunk-index": "0", + "x-chunk-total": "1001", + "x-file-ext": ".zip", + }, + body: Buffer.from("PK-1"), + }); + + expect(res.status).toBe(400); + expect(JSON.parse(res.body)).toEqual({ error: "Invalid chunk headers" }); + expect(streamToFile).not.toHaveBeenCalled(); + }); + it("cleans up the temp file and returns upload failure when streaming fails", async () => { vi.mocked(streamToFile).mockRejectedValueOnce(new Error("disk full")); const { app } = createTestApp(); diff --git a/packages/server/src/routes/upload.ts b/packages/server/src/routes/upload.ts index d2cada64c2..46ba36e064 100644 --- a/packages/server/src/routes/upload.ts +++ b/packages/server/src/routes/upload.ts @@ -1,7 +1,9 @@ +import { randomUUID } from "node:crypto"; import { mkdirSync } from "node:fs"; -import { mkdir, rm, unlink } from "node:fs/promises"; +import { mkdir, rename, rm, unlink } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { captureException } from "@sentry/node"; import type { Queue } from "bullmq"; import type { Database } from "dofek/db"; import type { ImportJobData } from "dofek/jobs/queues"; @@ -9,7 +11,7 @@ import { type Request, type Response, Router } from "express"; import rateLimit from "express-rate-limit"; import { getSessionIdFromRequest } from "../auth/cookies.ts"; import { validateSession } from "../auth/session.ts"; -import { assembleChunks, streamToFile } from "../lib/server-utils.ts"; +import { assembleChunks, MAX_UPLOAD_BYTES, streamToFile } from "../lib/server-utils.ts"; import { startWorker } from "../lib/start-worker.ts"; import { getUploadStateStore, @@ -28,12 +30,13 @@ const JOB_FILES_DIR = process.env.JOB_FILES_DIR || join(tmpdir(), "dofek-job-fil mkdirSync(JOB_FILES_DIR, { recursive: true }); const IN_PROGRESS_UPLOAD_STATUS_TTL_MS = UPLOAD_SESSION_TTL_MS + UPLOAD_STATUS_TTL_MS; const GARMIN_DUMP_MAX_UPLOAD_BYTES = 2 * 1024 * 1024 * 1024; +const MAX_UPLOAD_CHUNKS = 1000; const uploadStateStore = getUploadStateStore(); const uploadRateLimiter = rateLimit({ windowMs: 15 * 60 * 1000, - limit: 120, + limit: 600, standardHeaders: "draft-7", legacyHeaders: false, message: "Too many upload requests; please try again later", @@ -195,497 +198,476 @@ async function authenticate(req: Request, res: Response, db: Database): Promise< return session.userId; } -export function createUploadRouter(deps: UploadRouteDeps): Router { - const router = Router(); - const { importQueue, db } = deps; - // Poll job status — checks BullMQ first, falls back to upload-phase status - router.get<{ jobId: string }>( - "/apple-health/status/:jobId", - uploadStatusRateLimiter, - async (req, res) => { - const userId = await authenticate(req, res, db); - if (!userId) return; - - const uploadStatus = await uploadStateStore.getUploadStatus(req.params.jobId); - if (uploadStatus) { - if (uploadStatus.userId !== userId) { - res.status(403).json({ error: "Forbidden" }); - return; - } - if (uploadStatus.expiresAt && uploadStatus.expiresAt <= Date.now()) { - res.json(stripExpiry(await expireStaleUpload(req.params.jobId, userId))); - return; - } - res.json(stripExpiry(uploadStatus)); - return; - } - - const jobId = req.params.jobId; - - const status = await getImportJobStatus(importQueue, jobId); - if (!status) { - res.status(404).json({ error: "Unknown job" }); - return; - } - if (status.userId && status.userId !== userId) { - res.status(403).json({ error: "Forbidden" }); - return; - } - res.json(status); - }, - ); +interface FileImportRouteConfig { + routeId: string; + importType: ImportJobData["importType"]; + fileNamePrefix: string; + defaultFileExtension: string; + allowedFileExtensions: readonly string[]; + allowedContentTypes: readonly string[]; + unsupportedContentTypeMessage: string; + invalidFileExtensionMessage?: string; + maxUploadBytes?: number; + getSince: (request: Request) => Date; + getJobOptions?: (request: Request) => { weightUnit?: "kg" | "lbs" }; + inferFileExtension?: (input: { contentType: string | undefined }) => string | undefined; +} - // Chunked upload endpoint - router.post("/apple-health", async (req, res) => { - const userId = await authenticate(req, res, db); - if (!userId) return; +const csvContentTypes = ["text/csv", "application/octet-stream", "text/plain"] as const; + +const uploadRouteConfigs: FileImportRouteConfig[] = [ + { + routeId: "apple-health", + importType: "apple-health", + fileNamePrefix: "apple-health", + defaultFileExtension: ".zip", + allowedFileExtensions: [".zip", ".xml"], + allowedContentTypes: [ + "application/octet-stream", + "application/zip", + "application/xml", + "text/xml", + ], + unsupportedContentTypeMessage: + "Unsupported Content-Type. Expected application/octet-stream, application/zip, application/xml, or text/xml", + getSince: (request) => + request.query.fullSync === "true" + ? new Date(0) + : new Date(Date.now() - 7 * 24 * 60 * 60 * 1000), + inferFileExtension: ({ contentType }) => + contentType?.includes("xml") === true ? ".xml" : undefined, + }, + { + routeId: "strong-csv", + importType: "strong-csv", + fileNamePrefix: "strong-csv", + defaultFileExtension: ".csv", + allowedFileExtensions: [".csv"], + allowedContentTypes: csvContentTypes, + unsupportedContentTypeMessage: + "Unsupported Content-Type. Expected text/csv, application/octet-stream, or text/plain", + getSince: () => new Date(0), + getJobOptions: (request) => ({ weightUnit: request.query.units === "lbs" ? "lbs" : "kg" }), + }, + { + routeId: "cronometer-csv", + importType: "cronometer-csv", + fileNamePrefix: "cronometer-csv", + defaultFileExtension: ".csv", + allowedFileExtensions: [".csv"], + allowedContentTypes: csvContentTypes, + unsupportedContentTypeMessage: + "Unsupported Content-Type. Expected text/csv, application/octet-stream, or text/plain", + getSince: () => new Date(0), + }, + { + routeId: "kaya-export", + importType: "kaya-export", + fileNamePrefix: "kaya-export", + defaultFileExtension: ".csv", + allowedFileExtensions: [".csv"], + allowedContentTypes: csvContentTypes, + unsupportedContentTypeMessage: "Kaya imports require a CSV export", + invalidFileExtensionMessage: "Kaya imports require a CSV export", + getSince: () => new Date(0), + }, + { + routeId: "zos-app", + importType: "zos-app", + fileNamePrefix: "zos-app", + defaultFileExtension: ".bin", + allowedFileExtensions: [".bin"], + allowedContentTypes: ["application/octet-stream"], + unsupportedContentTypeMessage: "Unsupported Content-Type. Expected application/octet-stream", + getSince: () => new Date(0), + }, + { + routeId: "garmin-dump", + importType: "garmin-dump", + fileNamePrefix: "garmin-dump", + defaultFileExtension: ".zip", + allowedFileExtensions: [".zip"], + allowedContentTypes: [ + "application/zip", + "application/x-zip-compressed", + "application/octet-stream", + ], + unsupportedContentTypeMessage: + "Unsupported Content-Type. Expected application/zip or application/octet-stream", + maxUploadBytes: GARMIN_DUMP_MAX_UPLOAD_BYTES, + getSince: () => new Date(0), + }, +]; + +function createJobFilePath(config: FileImportRouteConfig, fileExtension: string): string { + return join(JOB_FILES_DIR, `${config.fileNamePrefix}-${randomUUID()}${fileExtension}`); +} - const contentType = req.headers["content-type"]?.split(";")[0]?.trim().toLowerCase(); - if ( - contentType && - !["application/octet-stream", "application/zip", "application/xml", "text/xml"].includes( - contentType, - ) - ) { - res.status(415).json({ - error: - "Unsupported Content-Type. Expected application/octet-stream, application/zip, application/xml, or text/xml", - }); - return; - } +function createChunkDirectoryPath(config: FileImportRouteConfig): string { + return join(JOB_FILES_DIR, `${config.fileNamePrefix}-chunked-${randomUUID()}`); +} - const uploadId = - typeof req.headers["x-upload-id"] === "string" ? req.headers["x-upload-id"] : undefined; - const chunkIndex = parseInt( - typeof req.headers["x-chunk-index"] === "string" ? req.headers["x-chunk-index"] : "", - 10, - ); - const chunkTotal = parseInt( - typeof req.headers["x-chunk-total"] === "string" ? req.headers["x-chunk-total"] : "", - 10, - ); - const fileExt = - (typeof req.headers["x-file-ext"] === "string" ? req.headers["x-file-ext"] : "") || ".zip"; - const fullSync = req.query.fullSync === "true"; +function getHeaderValue(request: Request, headerName: string): string | undefined { + const value = request.headers[headerName.toLowerCase()]; + return typeof value === "string" ? value : undefined; +} - // Validate uploadId and fileExt to prevent path traversal - if (uploadId && !/^[a-zA-Z0-9_-]+$/.test(uploadId)) { - res.status(400).json({ error: "Invalid upload ID" }); - return; - } - if (![".zip", ".xml"].includes(fileExt)) { - res.status(400).json({ error: "Invalid file extension" }); - return; - } - const since = fullSync ? new Date(0) : new Date(Date.now() - 7 * 24 * 60 * 60 * 1000); - - // Non-chunked upload (single small file) - if (!uploadId || Number.isNaN(chunkTotal) || chunkTotal <= 1) { - const ext = fileExt === ".xml" || (contentType ?? "").includes("xml") ? ".xml" : ".zip"; - const tmpId = `job-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`; - const tmpFile = join(JOB_FILES_DIR, `apple-health-${tmpId}${ext}`); - try { - await streamToFile(req, tmpFile); - const jobId = await enqueueImport(importQueue, tmpFile, since, "apple-health", userId); - res.json({ status: "processing", jobId }); - } catch (err: unknown) { - logger.error(`[upload] Apple Health upload failed: ${err}`); - res.status(500).json({ error: "Upload failed" }); - } - return; - } +function getNormalizedContentType(request: Request): string | undefined { + return getHeaderValue(request, "content-type")?.split(";")[0]?.trim().toLowerCase(); +} - // Chunked upload - try { - let upload = await uploadStateStore.getUploadSession(uploadId); - if (!upload) { - const dir = join(JOB_FILES_DIR, `apple-health-chunked-${uploadId}`); - await mkdir(dir, { recursive: true }); - upload = { total: chunkTotal, dir, userId }; - await uploadStateStore.saveUploadSession( - uploadId, - upload, - IN_PROGRESS_UPLOAD_STATUS_TTL_MS, - ); - await setUploadStatus( - uploadId, - inProgressStatus("uploading", 0, "Receiving chunks...", userId), - IN_PROGRESS_UPLOAD_STATUS_TTL_MS, - ); - } else if (upload.userId !== userId) { - res.status(403).json({ error: "Forbidden" }); - return; - } - - const chunkPath = join(upload.dir, `chunk-${String(chunkIndex).padStart(6, "0")}`); - await streamToFile(req, chunkPath); - const receivedCount = await uploadStateStore.addReceivedChunk( - uploadId, - chunkIndex, - IN_PROGRESS_UPLOAD_STATUS_TTL_MS, - ); - await uploadStateStore.saveUploadSession(uploadId, upload, IN_PROGRESS_UPLOAD_STATUS_TTL_MS); +function getUploadId(request: Request): string | undefined { + return getHeaderValue(request, "x-upload-id"); +} - const uploadPercentage = Math.round((receivedCount / upload.total) * 100); - logger.info(`[upload] Chunk ${chunkIndex + 1}/${chunkTotal} for ${uploadId}`); - - if (receivedCount < upload.total) { - await setUploadStatus( - uploadId, - inProgressStatus( - "uploading", - uploadPercentage, - `Received ${receivedCount}/${upload.total} chunks`, - userId, - ), - IN_PROGRESS_UPLOAD_STATUS_TTL_MS, - ); - res.json({ - status: "uploading", - jobId: uploadId, - received: receivedCount, - total: upload.total, - }); - return; - } - - // All chunks received — respond immediately, assemble in background. - // This prevents gateway timeouts when assembly takes a long time for large files. - const chunkDir = upload.dir; - await uploadStateStore.deleteUploadSession(uploadId); - await setUploadStatus( - uploadId, - inProgressStatus("assembling", 0, "Assembling file...", userId), - IN_PROGRESS_UPLOAD_STATUS_TTL_MS, - ); - res.json({ status: "assembling", jobId: uploadId }); - - // Fire-and-forget: assemble chunks → enqueue import - (async () => { - try { - const assembledFile = join(JOB_FILES_DIR, `apple-health-${uploadId}${fileExt}`); - await assembleChunks(chunkDir, assembledFile); - await rm(chunkDir, { recursive: true, force: true }); - // Use uploadId as the BullMQ job ID so the client can poll the same ID - await enqueueImport(importQueue, assembledFile, since, "apple-health", userId, { - jobId: uploadId, - }); - // Clear upload status — BullMQ job status takes over - await uploadStateStore.deleteUploadStatus(uploadId); - } catch (err: unknown) { - logger.error(`[upload] Assembly/enqueue failed for ${uploadId}: ${err}`); - await rm(chunkDir, { recursive: true, force: true }).catch((error: unknown) => { - logger.warn("Failed to clean up chunk dir %s: %s", chunkDir, error); - }); - await setUploadStatus(uploadId, { - status: "error", - progress: 0, - message: "Failed to assemble uploaded file", - userId, - }); - } - })(); - } catch (err: unknown) { - logger.error(`[upload] Chunked upload failed: ${err}`); - const upload = await uploadStateStore.getUploadSession(uploadId); - if (upload) { - await uploadStateStore.deleteUploadSession(uploadId); - await rm(upload.dir, { recursive: true, force: true }).catch((error: unknown) => { - logger.warn("Failed to clean up upload dir %s: %s", upload.dir, error); - }); - } - await setUploadStatus(uploadId, { - status: "error", - progress: 0, - message: "Upload failed", - userId, - }); - res.status(500).json({ error: "Upload failed" }); - } - }); +function getChunkNumber(request: Request, headerName: string): number { + return Number.parseInt(getHeaderValue(request, headerName) ?? "", 10); +} - // ── Strong CSV upload ── - router.get<{ jobId: string }>( - "/strong-csv/status/:jobId", - uploadStatusRateLimiter, - async (req, res) => { - const userId = await authenticate(req, res, db); - if (!userId) return; - - const jobId = req.params.jobId; - - const status = await getImportJobStatus(importQueue, jobId); - if (!status) { - res.status(404).json({ error: "Unknown job" }); - return; - } - if (status.userId && status.userId !== userId) { - res.status(403).json({ error: "Forbidden" }); - return; - } - res.json(status); - }, +function getRequestedFileExtension(request: Request): string | undefined { + const extension = getHeaderValue(request, "x-file-ext")?.trim().toLowerCase(); + return extension && extension.length > 0 ? extension : undefined; +} + +function getFileExtension( + request: Request, + config: FileImportRouteConfig, + contentType: string | undefined, +): string { + return ( + getRequestedFileExtension(request) ?? + config.inferFileExtension?.({ contentType }) ?? + config.defaultFileExtension ); +} - router.post("/strong-csv", async (req, res) => { - const userId = await authenticate(req, res, db); - if (!userId) return; +function validateUploadRequest( + request: Request, + response: Response, + config: FileImportRouteConfig, +): { contentType: string | undefined; uploadId: string | undefined; fileExtension: string } | null { + const contentType = getNormalizedContentType(request); + if (contentType && !config.allowedContentTypes.includes(contentType)) { + response.status(415).json({ error: config.unsupportedContentTypeMessage }); + return null; + } - const contentType = req.headers["content-type"]?.split(";")[0]?.trim().toLowerCase(); - if ( - contentType && - !["text/csv", "application/octet-stream", "text/plain"].includes(contentType) - ) { - res.status(415).json({ - error: - "Unsupported Content-Type. Expected text/csv, application/octet-stream, or text/plain", - }); - return; - } + if (config.maxUploadBytes && exceedsContentLengthLimit(request, config.maxUploadBytes)) { + response.status(413).json({ + error: `Garmin dump upload exceeds maximum size of ${config.maxUploadBytes} bytes`, + }); + return null; + } - const weightUnit = req.query.units === "lbs" ? "lbs" : "kg"; - const tmpId = `job-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`; - const tmpFile = join(JOB_FILES_DIR, `strong-csv-${tmpId}.csv`); + const uploadId = getUploadId(request); + if (uploadId && !/^[a-zA-Z0-9_-]+$/.test(uploadId)) { + response.status(400).json({ error: "Invalid upload ID" }); + return null; + } - try { - await streamToFile(req, tmpFile); - const jobId = await enqueueImport(importQueue, tmpFile, new Date(0), "strong-csv", userId, { - weightUnit, - }); - res.json({ status: "processing", jobId }); - } catch (err: unknown) { - logger.error(`[strong-csv] Upload failed: ${err}`); - res.status(500).json({ error: "Upload failed" }); - } - }); + const fileExtension = getFileExtension(request, config, contentType); + if (!config.allowedFileExtensions.includes(fileExtension)) { + response.status(400).json({ + error: config.invalidFileExtensionMessage ?? "Invalid file extension", + }); + return null; + } - // ── Cronometer CSV upload ── - router.get<{ jobId: string }>( - "/cronometer-csv/status/:jobId", - uploadStatusRateLimiter, - async (req, res) => { - const userId = await authenticate(req, res, db); - if (!userId) return; - - const jobId = req.params.jobId; - - const status = await getImportJobStatus(importQueue, jobId); - if (!status) { - res.status(404).json({ error: "Unknown job" }); - return; - } - if (status.userId && status.userId !== userId) { - res.status(403).json({ error: "Forbidden" }); - return; - } - res.json(status); - }, - ); + return { contentType, uploadId, fileExtension }; +} - router.post("/cronometer-csv", async (req, res) => { - const userId = await authenticate(req, res, db); - if (!userId) return; +async function handleUploadStatus( + request: Request<{ jobId: string }>, + response: Response, + deps: UploadRouteDeps, +): Promise { + const userId = await authenticate(request, response, deps.db); + if (!userId) return; - const contentType = req.headers["content-type"]?.split(";")[0]?.trim().toLowerCase(); - if ( - contentType && - !["text/csv", "application/octet-stream", "text/plain"].includes(contentType) - ) { - res.status(415).json({ - error: - "Unsupported Content-Type. Expected text/csv, application/octet-stream, or text/plain", - }); + const uploadStatus = await uploadStateStore.getUploadStatus(request.params.jobId); + if (uploadStatus) { + if (uploadStatus.userId !== userId) { + response.status(403).json({ error: "Forbidden" }); return; } - - const tmpId = `job-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`; - const tmpFile = join(JOB_FILES_DIR, `cronometer-csv-${tmpId}.csv`); - - try { - await streamToFile(req, tmpFile); - const jobId = await enqueueImport( - importQueue, - tmpFile, - new Date(0), - "cronometer-csv", - userId, - ); - res.json({ status: "processing", jobId }); - } catch (err: unknown) { - logger.error(`[cronometer-csv] Upload failed: ${err}`); - res.status(500).json({ error: "Upload failed" }); + if (uploadStatus.expiresAt && uploadStatus.expiresAt <= Date.now()) { + response.json(stripExpiry(await expireStaleUpload(request.params.jobId, userId))); + return; } - }); + response.json(stripExpiry(uploadStatus)); + return; + } - // ── Kaya CSV export upload ── - router.get<{ jobId: string }>( - "/kaya-export/status/:jobId", - uploadStatusRateLimiter, - async (req, res) => { - const userId = await authenticate(req, res, db); - if (!userId) return; - - const jobId = req.params.jobId; - - const status = await getImportJobStatus(importQueue, jobId); - if (!status) { - res.status(404).json({ error: "Unknown job" }); - return; - } - if (status.userId && status.userId !== userId) { - res.status(403).json({ error: "Forbidden" }); - return; - } - res.json(status); - }, - ); + const status = await getImportJobStatus(deps.importQueue, request.params.jobId); + if (!status) { + response.status(404).json({ error: "Unknown job" }); + return; + } + if (!status.userId || status.userId !== userId) { + response.status(403).json({ error: "Forbidden" }); + return; + } + response.json(status); +} - router.post("/kaya-export", uploadRateLimiter, async (req, res) => { - const userId = await authenticate(req, res, db); - if (!userId) return; +async function handleSingleFileUpload({ + request, + response, + userId, + config, + fileExtension, + deps, +}: { + request: Request; + response: Response; + userId: string; + config: FileImportRouteConfig; + fileExtension: string; + deps: UploadRouteDeps; +}): Promise { + const tmpFile = createJobFilePath(config, fileExtension); - const guidance = "Kaya imports require a CSV export"; - const contentType = req.headers["content-type"]?.split(";").at(0)?.trim().toLowerCase(); - if ( - contentType && - !["text/csv", "application/octet-stream", "text/plain"].includes(contentType) - ) { - res.status(415).json({ error: guidance }); - return; + try { + if (config.maxUploadBytes) { + await streamToFile(request, tmpFile, config.maxUploadBytes); + } else { + await streamToFile(request, tmpFile); } + const jobId = await enqueueImport( + deps.importQueue, + tmpFile, + config.getSince(request), + config.importType, + userId, + config.getJobOptions?.(request), + ); + response.json({ status: "processing", jobId }); + } catch (error: unknown) { + captureException(error); + logger.error(`[${config.routeId}] Upload failed: ${error}`); + await cleanupTempFile(tmpFile); + response.status(500).json({ error: "Upload failed" }); + } +} + +async function handleChunkedFileUpload({ + request, + response, + userId, + config, + uploadId, + fileExtension, + chunkIndex, + chunkTotal, + deps, +}: { + request: Request; + response: Response; + userId: string; + config: FileImportRouteConfig; + uploadId: string; + fileExtension: string; + chunkIndex: number; + chunkTotal: number; + deps: UploadRouteDeps; +}): Promise { + if ( + !Number.isInteger(chunkIndex) || + !Number.isInteger(chunkTotal) || + chunkIndex < 0 || + chunkTotal <= 1 || + chunkTotal > MAX_UPLOAD_CHUNKS || + chunkIndex >= chunkTotal + ) { + response.status(400).json({ error: "Invalid chunk headers" }); + return; + } - const fileExt = req.get("x-file-ext") || ".csv"; - if (fileExt.toLowerCase() !== ".csv") { - res.status(400).json({ error: guidance }); + try { + let upload = await uploadStateStore.getUploadSession(uploadId); + if (!upload) { + const dir = createChunkDirectoryPath(config); + await mkdir(dir); + upload = { total: chunkTotal, dir, userId }; + await uploadStateStore.saveUploadSession(uploadId, upload, IN_PROGRESS_UPLOAD_STATUS_TTL_MS); + await setUploadStatus( + uploadId, + inProgressStatus("uploading", 0, "Receiving chunks...", userId), + IN_PROGRESS_UPLOAD_STATUS_TTL_MS, + ); + } else if (upload.userId !== userId) { + response.status(403).json({ error: "Forbidden" }); + return; + } else if (upload.total !== chunkTotal) { + response.status(400).json({ error: "Chunk total does not match existing upload" }); return; } - const tmpId = `job-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`; - const tmpFile = join(JOB_FILES_DIR, `kaya-export-${tmpId}.csv`); - - try { - await streamToFile(req, tmpFile); - const jobId = await enqueueImport(importQueue, tmpFile, new Date(0), "kaya-export", userId); - res.json({ status: "processing", jobId }); - } catch (err: unknown) { - logger.error(`[kaya-export] Upload failed: ${err}`); - await cleanupTempFile(tmpFile); - res.status(500).json({ error: "Upload failed" }); + const maxUploadBytes = config.maxUploadBytes ?? MAX_UPLOAD_BYTES; + const chunkPath = join(upload.dir, `chunk-${String(chunkIndex).padStart(6, "0")}`); + const pendingChunkPath = join( + upload.dir, + `chunk-${String(chunkIndex).padStart(6, "0")}.${randomUUID()}.tmp`, + ); + const receivedBytes = await streamToFile(request, pendingChunkPath, maxUploadBytes); + const chunkRecord = await uploadStateStore.recordReceivedChunk( + uploadId, + chunkIndex, + receivedBytes, + maxUploadBytes, + IN_PROGRESS_UPLOAD_STATUS_TTL_MS, + ); + if (!chunkRecord.accepted) { + await cleanupTempFile(pendingChunkPath); + await uploadStateStore.deleteUploadSession(uploadId); + await rm(upload.dir, { recursive: true, force: true }).catch((rmError: unknown) => { + logger.warn("Failed to clean up oversized upload dir %s: %s", upload.dir, rmError); + }); + await setUploadStatus(uploadId, { + status: "error", + progress: 0, + message: "Upload exceeds maximum size", + userId, + }); + response.status(413).json({ error: "Upload exceeds maximum size" }); + return; } - }); + await rename(pendingChunkPath, chunkPath); + await uploadStateStore.saveUploadSession(uploadId, upload, IN_PROGRESS_UPLOAD_STATUS_TTL_MS); + const receivedCount = chunkRecord.receivedCount; - // ── ZOS App bin upload ── - router.get<{ jobId: string }>( - "/zos-app/status/:jobId", - uploadStatusRateLimiter, - async (req, res) => { - const userId = await authenticate(req, res, db); - if (!userId) return; - - const jobId = req.params.jobId; - - const status = await getImportJobStatus(importQueue, jobId); - if (!status) { - res.status(404).json({ error: "Unknown job" }); - return; - } - if (status.userId && status.userId !== userId) { - res.status(403).json({ error: "Forbidden" }); - return; - } - res.json(status); - }, - ); + const uploadPercentage = Math.round((receivedCount / upload.total) * 100); + logger.info(`[upload] ${config.routeId} chunk ${chunkIndex + 1}/${chunkTotal} for ${uploadId}`); - router.post("/zos-app", async (req, res) => { - const userId = await authenticate(req, res, db); - if (!userId) return; - - const contentType = req.headers["content-type"]?.split(";")[0]?.trim().toLowerCase(); - if (contentType && !["application/octet-stream"].includes(contentType)) { - res.status(415).json({ - error: "Unsupported Content-Type. Expected application/octet-stream", + if (receivedCount < upload.total) { + await setUploadStatus( + uploadId, + inProgressStatus( + "uploading", + uploadPercentage, + `Received ${receivedCount}/${upload.total} chunks`, + userId, + ), + IN_PROGRESS_UPLOAD_STATUS_TTL_MS, + ); + response.json({ + status: "uploading", + jobId: uploadId, + received: receivedCount, + total: upload.total, }); return; } - const tmpId = `job-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`; - const tmpFile = join(JOB_FILES_DIR, `zos-app-${tmpId}.bin`); - + const chunkDir = upload.dir; + const since = config.getSince(request); + const jobOptions = config.getJobOptions?.(request); + await setUploadStatus( + uploadId, + inProgressStatus("assembling", 0, "Assembling file...", userId), + IN_PROGRESS_UPLOAD_STATUS_TTL_MS, + ); + const assembledFile = createJobFilePath(config, fileExtension); try { - await streamToFile(req, tmpFile); - const jobId = await enqueueImport(importQueue, tmpFile, new Date(0), "zos-app", userId); - res.json({ status: "processing", jobId }); - } catch (err: unknown) { - logger.error(`[zos-app] Upload failed: ${err}`); - await cleanupTempFile(tmpFile); - res.status(500).json({ error: "Upload failed" }); + await assembleChunks(chunkDir, assembledFile); + await enqueueImport(deps.importQueue, assembledFile, since, config.importType, userId, { + ...jobOptions, + jobId: uploadId, + }); + await uploadStateStore.deleteUploadSession(uploadId); + await uploadStateStore.deleteUploadStatus(uploadId); + await rm(chunkDir, { recursive: true, force: true }).catch((rmError: unknown) => { + logger.warn("Failed to clean up chunk dir %s: %s", chunkDir, rmError); + }); + response.json({ status: "processing", jobId: uploadId }); + } catch (error: unknown) { + captureException(error); + logger.error(`[upload] Assembly/enqueue failed for ${uploadId}: ${error}`); + await cleanupTempFile(assembledFile); + await rm(chunkDir, { recursive: true, force: true }).catch((rmError: unknown) => { + logger.warn("Failed to clean up chunk dir %s: %s", chunkDir, rmError); + }); + await setUploadStatus(uploadId, { + status: "error", + progress: 0, + message: "Failed to assemble uploaded file", + userId, + }); + response.status(500).json({ error: "Failed to assemble uploaded file" }); } - }); + } catch (error: unknown) { + captureException(error); + logger.error(`[upload] Chunked upload failed: ${error}`); + const upload = await uploadStateStore.getUploadSession(uploadId); + if (upload) { + await uploadStateStore.deleteUploadSession(uploadId); + await rm(upload.dir, { recursive: true, force: true }).catch((rmError: unknown) => { + logger.warn("Failed to clean up upload dir %s: %s", upload.dir, rmError); + }); + } + await setUploadStatus(uploadId, { + status: "error", + progress: 0, + message: "Upload failed", + userId, + }); + response.status(500).json({ error: "Upload failed" }); + } +} - // ── Garmin account export dump upload ── +function registerFileImportRoutes( + router: Router, + config: FileImportRouteConfig, + deps: UploadRouteDeps, +): void { router.get<{ jobId: string }>( - "/garmin-dump/status/:jobId", + `/${config.routeId}/status/:jobId`, uploadStatusRateLimiter, - async (req, res) => { - const userId = await authenticate(req, res, db); - if (!userId) return; - - const jobId = req.params.jobId; - - const status = await getImportJobStatus(importQueue, jobId); - if (!status) { - res.status(404).json({ error: "Unknown job" }); - return; - } - if (status.userId && status.userId !== userId) { - res.status(403).json({ error: "Forbidden" }); - return; - } - res.json(status); + async (request, response) => { + await handleUploadStatus(request, response, deps); }, ); - router.post("/garmin-dump", uploadRateLimiter, async (req, res) => { - const userId = await authenticate(req, res, db); + const uploadHandler = async (request: Request, response: Response) => { + const userId = await authenticate(request, response, deps.db); if (!userId) return; - const contentTypeHeader = req.headers["content-type"]; - const contentType = - typeof contentTypeHeader === "string" - ? (contentTypeHeader.split(";")[0] ?? "").trim().toLowerCase() - : undefined; - if ( - contentType && - !["application/zip", "application/x-zip-compressed", "application/octet-stream"].includes( - contentType, - ) - ) { - res.status(415).json({ - error: "Unsupported Content-Type. Expected application/zip or application/octet-stream", - }); - return; - } - if (exceedsContentLengthLimit(req, GARMIN_DUMP_MAX_UPLOAD_BYTES)) { - res.status(413).json({ - error: `Garmin dump upload exceeds maximum size of ${GARMIN_DUMP_MAX_UPLOAD_BYTES} bytes`, + const validation = validateUploadRequest(request, response, config); + if (!validation) return; + + const chunkTotal = getChunkNumber(request, "x-chunk-total"); + if (validation.uploadId && chunkTotal > 1) { + await handleChunkedFileUpload({ + request, + response, + userId, + config, + uploadId: validation.uploadId, + fileExtension: validation.fileExtension, + chunkIndex: getChunkNumber(request, "x-chunk-index"), + chunkTotal, + deps, }); return; } - const tmpId = `job-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`; - const tmpFile = join(JOB_FILES_DIR, `garmin-dump-${tmpId}.zip`); + await handleSingleFileUpload({ + request, + response, + userId, + config, + fileExtension: validation.fileExtension, + deps, + }); + }; - try { - await streamToFile(req, tmpFile, GARMIN_DUMP_MAX_UPLOAD_BYTES); - const jobId = await enqueueImport(importQueue, tmpFile, new Date(0), "garmin-dump", userId); - res.json({ status: "processing", jobId }); - } catch (err: unknown) { - logger.error(`[garmin-dump] Upload failed: ${err}`); - await cleanupTempFile(tmpFile); - res.status(500).json({ error: "Upload failed" }); - } - }); + router.post(`/${config.routeId}`, uploadRateLimiter, uploadHandler); +} +export function createUploadRouter(deps: UploadRouteDeps): Router { + const router = Router(); + for (const config of uploadRouteConfigs) { + registerFileImportRoutes(router, config, deps); + } return router; } diff --git a/packages/web/src/components/DataSourcesPanel.test.tsx b/packages/web/src/components/DataSourcesPanel.test.tsx index cdfd4dc42d..4acbfc6497 100644 --- a/packages/web/src/components/DataSourcesPanel.test.tsx +++ b/packages/web/src/components/DataSourcesPanel.test.tsx @@ -44,7 +44,14 @@ const mockSyncMutateAsync = vi.hoisted(() => vi.fn()); const mockPollSyncJob = vi.hoisted(() => vi.fn()); const mockInvalidate = vi.hoisted(() => vi.fn()); const mockSyncStatusFetch = vi.hoisted(() => vi.fn()); -const mockFileImportZone = vi.hoisted(() => vi.fn()); +const mockFileImportProviderCard = vi.hoisted(() => vi.fn()); +const mockProviderStatsQuery = vi.hoisted(() => + vi.fn<() => MockQueryResult>>>(() => ({ + data: [], + isLoading: false, + error: null, + })), +); vi.mock("../lib/trpc.ts", () => ({ trpc: { @@ -52,7 +59,7 @@ vi.mock("../lib/trpc.ts", () => ({ providers: { useQuery: mockProvidersQuery, }, - providerStats: { useQuery: () => ({ data: [], isLoading: false }) }, + providerStats: { useQuery: mockProviderStatsQuery }, logs: { useQuery: () => ({ data: [], isLoading: false }) }, activeSyncs: { useQuery: () => ({ data: [], isLoading: false }) }, dataHealth: { useQuery: mockDataHealthQuery }, @@ -85,20 +92,23 @@ vi.mock("./DataSourcesAuthModals.tsx", () => ({ ), })); -vi.mock("./FileImportZone.tsx", () => ({ - FileImportZone: (props: { +vi.mock("./FileImportProviderCard.tsx", () => ({ + FileImportProviderCard: (props: { providerId?: string; title: string; description: string; accept: string; uploadUrl: string; statusUrl: string; + chunked?: boolean; + stats?: Record; }) => { - mockFileImportZone(props); + mockFileImportProviderCard(props); return (

{props.title}

{props.description}

+
); }, @@ -180,7 +190,9 @@ describe("DataSourcesPanel", () => { mockPollSyncJob.mockResolvedValue(undefined); mockInvalidate.mockReset(); mockSyncStatusFetch.mockReset(); - mockFileImportZone.mockClear(); + mockFileImportProviderCard.mockClear(); + mockProviderStatsQuery.mockReset(); + mockProviderStatsQuery.mockReturnValue({ data: [], isLoading: false, error: null }); }); it("shows server data readiness messages above provider cards", () => { @@ -323,7 +335,7 @@ describe("DataSourcesPanel", () => { expect(screen.getByTestId("file-import-kaya-export")).toBeTruthy(); expect(screen.getByText("Kaya")).toBeTruthy(); expect(screen.getByText(".csv export from Kaya")).toBeTruthy(); - expect(mockFileImportZone).toHaveBeenCalledWith( + expect(mockFileImportProviderCard).toHaveBeenCalledWith( expect.objectContaining({ providerId: "kaya-export", title: "Kaya", @@ -331,6 +343,90 @@ describe("DataSourcesPanel", () => { accept: ".csv", uploadUrl: "/api/upload/kaya-export", statusUrl: "/api/upload/kaya-export/status", + chunked: true, + }), + ); + }); + + it("passes provider summaries to file import cards", () => { + const appleHealthStats = { + providerId: "apple_health", + activities: 0, + metricStream: 205_367, + dailyMetrics: 229, + sleepSessions: 155, + bodyMeasurements: 43, + healthEvents: 392, + foodEntries: 0, + nutritionDaily: 0, + labPanels: 0, + labResults: 0, + journalEntries: 0, + }; + const kayaStats = { + providerId: "kaya-export", + activities: 352, + metricStream: 205_367, + dailyMetrics: 229, + sleepSessions: 155, + bodyMeasurements: 43, + healthEvents: 392, + foodEntries: 0, + nutritionDaily: 0, + labPanels: 0, + labResults: 0, + journalEntries: 0, + }; + mockProvidersQuery.mockReturnValue({ + data: [ + { + id: "kaya-export", + name: "Kaya", + authorized: true, + authType: "file-import", + importOnly: true, + pushOnly: false, + needsReauth: false, + }, + ], + isLoading: false, + error: null, + }); + mockProviderStatsQuery.mockReturnValue({ + data: [appleHealthStats, kayaStats], + isLoading: false, + error: null, + }); + + render(); + + expect(mockFileImportProviderCard).toHaveBeenCalledWith( + expect.objectContaining({ + providerId: "kaya-export", + stats: kayaStats, + }), + ); + expect(mockFileImportProviderCard).toHaveBeenCalledWith( + expect.objectContaining({ + providerId: "apple_health", + stats: appleHealthStats, + }), + ); + }); + + it("renders Apple Health through the shared file import provider card", () => { + render(); + + expect(screen.getByTestId("file-import-apple_health")).toBeTruthy(); + expect(screen.getByText("Import file")).toBeTruthy(); + expect(mockFileImportProviderCard).toHaveBeenCalledWith( + expect.objectContaining({ + providerId: "apple_health", + title: "Apple Health", + description: ".zip or .xml from Health app export", + uploadUrl: "/api/upload/apple-health?fullSync=true", + statusUrl: "/api/upload/apple-health/status", + chunked: true, }), ); }); diff --git a/packages/web/src/components/DataSourcesPanel.tsx b/packages/web/src/components/DataSourcesPanel.tsx index 643005d1de..d72a27407c 100644 --- a/packages/web/src/components/DataSourcesPanel.tsx +++ b/packages/web/src/components/DataSourcesPanel.tsx @@ -5,8 +5,12 @@ import { trpc } from "../lib/trpc.ts"; import { DataReadinessBanner } from "./DataReadinessBanner.tsx"; import { CredentialAuthModal, GarminAuthModal, WhoopAuthModal } from "./DataSourcesAuthModals.tsx"; import type { ProviderState, SyncProviderSummary } from "./DataSourcesSyncTypes.ts"; -import { FileImportZone } from "./FileImportZone.tsx"; -import { appleHealthFileImportConfig, fileImportConfigs } from "./file-import-configs.ts"; +import { FileImportProviderCard } from "./FileImportProviderCard.tsx"; +import { + appleHealthFileImportConfig, + type fileImportConfigs, + getFileImportConfig, +} from "./file-import-configs.ts"; import { SyncProviderCard } from "./SyncProviderCard.tsx"; const oauthBroadcastMessage = z.object({ @@ -298,7 +302,7 @@ export function DataSourcesPanel() { }); for (const p of allProviders) { - const importConfig = fileImportConfigs[p.id]; + const importConfig = getFileImportConfig(p.id); if (importConfig) { unifiedProviders.push({ kind: "import", id: p.id, config: importConfig }); } else { @@ -359,7 +363,7 @@ export function DataSourcesPanel() { const providerStats = statsByProvider.get(entry.id); const recentLogs = (logsByProvider.get(entry.id) ?? []).slice(0, 5); return ( - {}, + }, +} satisfies Meta; + +export default meta; + +type Story = StoryObj; + +export const Default: Story = {}; + +export const Disabled: Story = { + args: { + disabled: true, + }, +}; diff --git a/packages/web/src/components/FileImportButton.tsx b/packages/web/src/components/FileImportButton.tsx new file mode 100644 index 0000000000..ac0a59ff79 --- /dev/null +++ b/packages/web/src/components/FileImportButton.tsx @@ -0,0 +1,18 @@ +export function FileImportButton({ + disabled = false, + onClick, +}: { + disabled?: boolean; + onClick: () => void; +}) { + return ( + + ); +} diff --git a/packages/web/src/components/FileImportProviderCard.stories.tsx b/packages/web/src/components/FileImportProviderCard.stories.tsx new file mode 100644 index 0000000000..6c8413563f --- /dev/null +++ b/packages/web/src/components/FileImportProviderCard.stories.tsx @@ -0,0 +1,78 @@ +import type { Meta, StoryObj } from "@storybook/react-vite"; +import { + createMemoryHistory, + createRootRoute, + createRoute, + createRouter, + Outlet, + RouterProvider, +} from "@tanstack/react-router"; +import type { ComponentType } from "react"; +import { FileImportProviderCard } from "./FileImportProviderCard.tsx"; + +function withRouter(Story: ComponentType) { + const rootRoute = createRootRoute({ + component: Outlet, + }); + const homeRoute = createRoute({ + getParentRoute: () => rootRoute, + path: "/", + component: () => , + }); + const providerRoute = createRoute({ + getParentRoute: () => rootRoute, + path: "providers/$id", + component: () => null, + }); + const routeTree = rootRoute.addChildren([homeRoute, providerRoute]); + const router = createRouter({ + routeTree, + history: createMemoryHistory({ initialEntries: ["/"] }), + }); + + return ( +
+ +
+ ); +} + +const meta = { + title: "Providers/FileImportProviderCard", + component: FileImportProviderCard, + tags: ["autodocs"], + decorators: [withRouter], + args: { + providerId: "strong-csv", + title: "Strong", + description: ".csv export from Strong app", + accept: ".csv", + uploadUrl: "/api/upload/strong-csv?units=kg", + statusUrl: "/api/upload/strong-csv/status", + chunked: true, + }, +} satisfies Meta; + +export default meta; + +type Story = StoryObj; + +export const Default: Story = {}; + +export const AppleHealth: Story = { + args: { + providerId: "apple_health", + title: "Apple Health", + description: ".zip or .xml from Health app export", + accept: ".zip,.xml", + uploadUrl: "/api/upload/apple-health?fullSync=true", + statusUrl: "/api/upload/apple-health/status", + chunked: true, + }, +}; + +export const WithoutDetailsLink: Story = { + args: { + showDetailsLink: false, + }, +}; diff --git a/packages/web/src/components/FileImportProviderCard.tsx b/packages/web/src/components/FileImportProviderCard.tsx new file mode 100644 index 0000000000..4fd95b3858 --- /dev/null +++ b/packages/web/src/components/FileImportProviderCard.tsx @@ -0,0 +1,8 @@ +import { FileImportZone, type FileImportZoneProps } from "./FileImportZone.tsx"; + +export type FileImportProviderCardProps = FileImportZoneProps & + Required>; + +export function FileImportProviderCard(props: FileImportProviderCardProps) { + return ; +} diff --git a/packages/web/src/components/FileImportZone.test.tsx b/packages/web/src/components/FileImportZone.test.tsx new file mode 100644 index 0000000000..a1da673fa4 --- /dev/null +++ b/packages/web/src/components/FileImportZone.test.tsx @@ -0,0 +1,70 @@ +// @vitest-environment jsdom +import { cleanup, render, screen } from "@testing-library/react"; +import type { ReactNode } from "react"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { FileImportZone } from "./FileImportZone.tsx"; + +vi.mock("@tanstack/react-router", () => ({ + Link: ({ children }: { children: ReactNode }) => {children}, +})); + +afterEach(cleanup); + +describe("FileImportZone", () => { + it("renders an explicit file picker button", () => { + render( + , + ); + + expect(screen.getByRole("button", { name: "Import file" })).toBeTruthy(); + expect(screen.getByText(".csv export from Strong app")).toBeTruthy(); + }); + + it("renders provider summary totals on file import tiles", () => { + render( + , + ); + + expect(screen.getByText("206,538")).toBeTruthy(); + expect(screen.getByText("records")).toBeTruthy(); + expect(screen.getByText("Activities")).toBeTruthy(); + expect(screen.getByText("352")).toBeTruthy(); + expect(screen.getByText("Metric Stream")).toBeTruthy(); + expect(screen.getByText("205,367")).toBeTruthy(); + expect(screen.getByText("Daily Metrics")).toBeTruthy(); + expect(screen.getByText("229")).toBeTruthy(); + expect(screen.getByText("Sleep")).toBeTruthy(); + expect(screen.getByText("155")).toBeTruthy(); + expect(screen.getByText("Body")).toBeTruthy(); + expect(screen.getByText("43")).toBeTruthy(); + expect(screen.getByText("Events")).toBeTruthy(); + expect(screen.getByText("392")).toBeTruthy(); + }); +}); diff --git a/packages/web/src/components/FileImportZone.tsx b/packages/web/src/components/FileImportZone.tsx index dc3bf3ad2c..dc6773b202 100644 --- a/packages/web/src/components/FileImportZone.tsx +++ b/packages/web/src/components/FileImportZone.tsx @@ -3,6 +3,7 @@ import type { ProviderStats } from "@dofek/providers/provider-stats"; import { Link } from "@tanstack/react-router"; import { useCallback, useEffect, useRef, useState } from "react"; import type { SyncLogEntry, SyncStatus } from "./DataSourcesSyncTypes.ts"; +import { FileImportButton } from "./FileImportButton.tsx"; import { ProviderLogo } from "./ProviderLogo.tsx"; import { ProviderStatsBreakdown } from "./ProviderStatsBreakdown.tsx"; import { StatusDot } from "./StatusDot.tsx"; @@ -215,22 +216,14 @@ export function FileImportZone({ {title} - + {state.status !== "idle" && state.status !== "syncing" && (
= { accept: ".csv", uploadUrl: "/api/upload/strong-csv?units=kg", statusUrl: "/api/upload/strong-csv/status", + chunked: true, }, "cronometer-csv": { title: "Cronometer", @@ -24,6 +25,7 @@ export const fileImportConfigs: Record = { accept: ".csv", uploadUrl: "/api/upload/cronometer-csv", statusUrl: "/api/upload/cronometer-csv/status", + chunked: true, }, "garmin-dump": { title: "Garmin Dump", @@ -31,6 +33,7 @@ export const fileImportConfigs: Record = { accept: ".zip", uploadUrl: "/api/upload/garmin-dump", statusUrl: "/api/upload/garmin-dump/status", + chunked: true, }, "kaya-export": { title: "Kaya", @@ -38,5 +41,12 @@ export const fileImportConfigs: Record = { accept: ".csv", uploadUrl: "/api/upload/kaya-export", statusUrl: "/api/upload/kaya-export/status", + chunked: true, }, }; + +export function getFileImportConfig(providerId: string): FileImportZoneProps | undefined { + if (providerId === "apple_health") return appleHealthFileImportConfig; + if (!Object.hasOwn(fileImportConfigs, providerId)) return undefined; + return fileImportConfigs[providerId]; +} diff --git a/packages/web/src/pages/ProviderDetailPage.test.tsx b/packages/web/src/pages/ProviderDetailPage.test.tsx index 2d8bd827da..7638296c31 100644 --- a/packages/web/src/pages/ProviderDetailPage.test.tsx +++ b/packages/web/src/pages/ProviderDetailPage.test.tsx @@ -21,21 +21,29 @@ vi.mock("../components/ProviderLogo.tsx", () => ({ ProviderLogo: () =>
, })); -vi.mock("../components/FileImportZone.tsx", () => ({ - FileImportZone: ({ +const mockFileImportProviderCard = vi.hoisted(() => vi.fn()); + +vi.mock("../components/FileImportProviderCard.tsx", () => ({ + FileImportProviderCard: ({ + providerId, title, description, showDetailsLink, }: { + providerId: string; title: string; description: string; showDetailsLink?: boolean; - }) => ( -
- {title} - {description} -
- ), + }) => { + mockFileImportProviderCard({ providerId, title, description, showDetailsLink }); + return ( +
+ {title} + {description} + +
+ ); + }, })); interface MockProvider { @@ -144,6 +152,7 @@ afterEach(() => { vi.clearAllTimers(); vi.useRealTimers(); vi.clearAllMocks(); + mockFileImportProviderCard.mockClear(); }); function queryButton(container: HTMLElement, ariaLabel: string): Element { @@ -351,17 +360,40 @@ describe("ProviderDetailPage import-only providers", () => { expect(screen.getByRole("heading", { name: "Garmin Dump" })).toBeTruthy(); expect(screen.getByText("Import")).toBeTruthy(); - expect(screen.getByTestId("file-import-zone").getAttribute("data-show-details-link")).toBe( - "false", - ); + expect( + screen.getByTestId("file-import-provider-card").getAttribute("data-show-details-link"), + ).toBe("false"); expect(screen.getByText(".zip account export from Garmin")).toBeTruthy(); + expect(screen.getByText("Import file")).toBeTruthy(); + expect(screen.queryByText("Sync Controls")).toBeNull(); + }); + + it("shows the shared upload card for Apple Health on the provider detail page", async () => { + mockUseParams.mockReturnValue({ id: "apple_health" }); + mockProviders.data = []; + + const { ProviderDetailPage } = await import("./ProviderDetailPage"); + render(); + + expect(screen.getByRole("heading", { name: "Apple Health" })).toBeTruthy(); + expect(screen.getByText("Import")).toBeTruthy(); + expect(screen.getByText(".zip or .xml from Health app export")).toBeTruthy(); + expect(screen.getByText("Import file")).toBeTruthy(); expect(screen.queryByText("Sync Controls")).toBeNull(); + expect(mockFileImportProviderCard).toHaveBeenCalledWith( + expect.objectContaining({ + providerId: "apple_health", + title: "Apple Health", + showDetailsLink: false, + }), + ); }); it("shows sync controls for non-import-only providers", async () => { + mockUseParams.mockReturnValue({ id: "wahoo" }); mockProviders.data = [ { - id: "strong-csv", + id: "wahoo", name: "Wahoo", authorized: true, authType: "oauth", @@ -379,9 +411,10 @@ describe("ProviderDetailPage import-only providers", () => { }); it("shows provider data readiness when read models are blocked", async () => { + mockUseParams.mockReturnValue({ id: "wahoo" }); mockProviders.data = [ { - id: "strong-csv", + id: "wahoo", name: "Wahoo", authorized: true, authType: "oauth", @@ -417,9 +450,10 @@ describe("ProviderDetailPage import-only providers", () => { }); it("shows single-provider cooldown outcome without polling a fake job", async () => { + mockUseParams.mockReturnValue({ id: "wahoo" }); mockProviders.data = [ { - id: "strong-csv", + id: "wahoo", name: "Wahoo", authorized: true, authType: "oauth", @@ -433,7 +467,7 @@ describe("ProviderDetailPage import-only providers", () => { providerJobs: [], providerResults: [ { - providerId: "strong-csv", + providerId: "wahoo", status: "skippedCooldown", message: "Provider sync skipped: rate-limit cooldown active", }, @@ -453,9 +487,10 @@ describe("ProviderDetailPage import-only providers", () => { }); it("does not trigger sync for an inverted date range", async () => { + mockUseParams.mockReturnValue({ id: "wahoo" }); mockProviders.data = [ { - id: "strong-csv", + id: "wahoo", name: "Wahoo", authorized: true, authType: "oauth", @@ -477,9 +512,10 @@ describe("ProviderDetailPage import-only providers", () => { }); it("does not render disconnect for providers that are not connected", async () => { + mockUseParams.mockReturnValue({ id: "wahoo" }); mockProviders.data = [ { - id: "strong-csv", + id: "wahoo", name: "Wahoo", authorized: false, authType: "oauth", diff --git a/packages/web/src/pages/ProviderDetailPage.tsx b/packages/web/src/pages/ProviderDetailPage.tsx index c3b1edff85..966b1142a2 100644 --- a/packages/web/src/pages/ProviderDetailPage.tsx +++ b/packages/web/src/pages/ProviderDetailPage.tsx @@ -16,8 +16,8 @@ import { Link, useParams } from "@tanstack/react-router"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { z } from "zod"; import { DataReadinessBanner } from "../components/DataReadinessBanner.tsx"; -import { FileImportZone } from "../components/FileImportZone.tsx"; -import { fileImportConfigs } from "../components/file-import-configs.ts"; +import { FileImportProviderCard } from "../components/FileImportProviderCard.tsx"; +import { getFileImportConfig } from "../components/file-import-configs.ts"; import { PageLayout } from "../components/PageLayout.tsx"; import { ProviderDisconnectControl } from "../components/ProviderDisconnectControl.tsx"; import { ProviderLogo } from "../components/ProviderLogo.tsx"; @@ -69,7 +69,8 @@ export function ProviderDetailPage() { const provider = (providers.data ?? []).find((p) => p.id === providerId); const providerStats = (stats.data ?? []).find((s) => s.providerId === providerId); - const importConfig = fileImportConfigs[providerId]; + const importConfig = getFileImportConfig(providerId); + const hasFileImportConfig = importConfig !== undefined; const pushOnly = provider?.pushOnly === true; const lastSyncedRelative = hasValidDateInput(provider?.lastSyncedAt) ? formatRelativeTime(provider.lastSyncedAt) @@ -268,10 +269,10 @@ export function ProviderDetailPage() { loading={dataHealth.isLoading} /> - {provider?.importOnly && importConfig && ( + {importConfig && (

Import

-

Sync Controls

diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 356951e61f..295c84d153 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -430,6 +430,9 @@ importers: expo-dev-client: specifier: ~57.0.5 version: 57.0.5(expo@57.0.4)(react-native@0.86.0(@babel/core@7.29.7)(@react-native/metro-config@0.86.0(@babel/core@7.29.7))(@types/react@19.2.14)(react@19.2.3)) + expo-document-picker: + specifier: ~57.0.0 + version: 57.0.0(expo@57.0.4) expo-file-system: specifier: 57.0.0 version: 57.0.0(expo@57.0.4)(react-native@0.86.0(@babel/core@7.29.7)(@react-native/metro-config@0.86.0(@babel/core@7.29.7))(@types/react@19.2.14)(react@19.2.3)) @@ -9801,6 +9804,11 @@ packages: expo: '*' react-native: '*' + expo-document-picker@57.0.0: + resolution: {integrity: sha512-foT1pA3tD7KvcqXwjQBh4qEDjHT9U9MhZ6H2aF38VolCiHRhXDQLtKQqCeXjbHxJhSf8HC/YPV3IRxDxFTL2sA==} + peerDependencies: + expo: '*' + expo-eas-client@57.0.0: resolution: {integrity: sha512-HYroDzqYFpALIXW5tpQD2WHqij3iUal+fXzVJhIYV3OdVcbobxt+p6209OJDARlU431CANZwp4vPmS7yRCWTCw==} @@ -23107,14 +23115,14 @@ snapshots: msw: 2.12.11(@types/node@22.19.15)(typescript@6.0.3) vite: 7.3.6(@types/node@22.19.15)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.47.1)(tsx@4.21.0)(yaml@2.9.0) - '@vitest/mocker@3.2.6(msw@2.12.11(@types/node@25.9.1)(typescript@6.0.3))(vite@7.3.6(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.47.1)(tsx@4.21.0)(yaml@2.9.0))': + '@vitest/mocker@3.2.6(msw@2.12.11(@types/node@25.9.1)(typescript@6.0.3))(vite@7.3.6(@types/node@22.19.15)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.47.1)(tsx@4.21.0)(yaml@2.9.0))': dependencies: '@vitest/spy': 3.2.6 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: msw: 2.12.11(@types/node@25.9.1)(typescript@6.0.3) - vite: 7.3.6(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.47.1)(tsx@4.21.0)(yaml@2.9.0) + vite: 7.3.6(@types/node@22.19.15)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.47.1)(tsx@4.21.0)(yaml@2.9.0) '@vitest/pretty-format@3.2.4': dependencies: @@ -25177,6 +25185,10 @@ snapshots: expo-dev-menu-interface: 57.0.0(expo@57.0.4) react-native: 0.86.0(@babel/core@7.29.7)(@react-native/metro-config@0.86.0(@babel/core@7.29.7))(@types/react@19.2.14)(react@19.2.3) + expo-document-picker@57.0.0(expo@57.0.4): + dependencies: + expo: 57.0.4(@babel/core@7.29.7)(@expo/dom-webview@55.0.5)(@expo/metro-runtime@57.0.3)(expo-router@57.0.4)(react-dom@19.2.3(react@19.2.3))(react-native-web@0.21.2(encoding@0.1.13)(react-dom@19.2.3(react@19.2.3))(react@19.2.3))(react-native-worklets@0.10.0(@babel/core@7.29.7)(@react-native/metro-config@0.86.0(@babel/core@7.29.7))(react-native@0.86.0(@babel/core@7.29.7)(@react-native/metro-config@0.86.0(@babel/core@7.29.7))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3))(react-native@0.86.0(@babel/core@7.29.7)(@react-native/metro-config@0.86.0(@babel/core@7.29.7))(@types/react@19.2.14)(react@19.2.3))(react@19.2.3)(typescript@6.0.3) + expo-eas-client@57.0.0: {} expo-file-system@57.0.0(expo@57.0.4)(react-native@0.86.0(@babel/core@7.29.7)(@react-native/metro-config@0.86.0(@babel/core@7.29.7))(@types/react@19.2.14)(react@19.2.3)): @@ -30311,7 +30323,7 @@ snapshots: dependencies: '@types/chai': 5.2.3 '@vitest/expect': 3.2.6 - '@vitest/mocker': 3.2.6(msw@2.12.11(@types/node@25.9.1)(typescript@6.0.3))(vite@7.3.6(@types/node@25.9.1)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.47.1)(tsx@4.21.0)(yaml@2.9.0)) + '@vitest/mocker': 3.2.6(msw@2.12.11(@types/node@25.9.1)(typescript@6.0.3))(vite@7.3.6(@types/node@22.19.15)(jiti@2.7.0)(lightningcss@1.32.0)(terser@5.47.1)(tsx@4.21.0)(yaml@2.9.0)) '@vitest/pretty-format': 3.2.6 '@vitest/runner': 3.2.6 '@vitest/snapshot': 3.2.6