From 4914fc329c94d667907a1aaaba49da26d230deee Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 23 Jul 2026 15:01:01 +0000 Subject: [PATCH 1/6] fix(statistics): paginate org app metrics past PostgREST max_rows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Org all-apps dashboards call get_app_metrics for every app × day and were silently truncated at 1000 rows, so bandwidth for later app_ids disappeared while single-app stats still worked. Page the RPC like storage_byte_hours and index metrics by date. Co-authored-by: Martin DONADIEU --- .../_backend/public/statistics/index.ts | 184 ++++++++++-------- ...rg-statistics-bandwidth-pagination.test.ts | 152 +++++++++++++++ ...atistics-bandwidth-pagination.unit.test.ts | 87 +++++++++ 3 files changed, 337 insertions(+), 86 deletions(-) create mode 100644 tests/org-statistics-bandwidth-pagination.test.ts create mode 100644 tests/org-statistics-bandwidth-pagination.unit.test.ts diff --git a/supabase/functions/_backend/public/statistics/index.ts b/supabase/functions/_backend/public/statistics/index.ts index 2fb10acbad..4d53bf5011 100644 --- a/supabase/functions/_backend/public/statistics/index.ts +++ b/supabase/functions/_backend/public/statistics/index.ts @@ -161,6 +161,9 @@ interface AppOwnerOrgRow { const STATS_QUERY_RETRY_ATTEMPTS = 3 const STATS_QUERY_RETRY_DELAY_MS = 250 const STORAGE_BYTE_HOURS_PAGE_SIZE = 1000 +// PostgREST max_rows is 1000. Org dashboards return apps × days and must paginate +// or bandwidth/MAU for later app_ids are silently dropped while /app/:id still works. +const APP_METRICS_PAGE_SIZE = 1000 // Helper to get authenticated supabase client based on auth type function getAuthenticatedSupabase(c: Context, auth: AuthInfo) { @@ -273,12 +276,73 @@ async function getStatsAppOwnerOrgOrThrow( return ownerOrg } +async function fetchAppMetricsRows( + c: Context, + supabase: ReturnType, + params: { + orgId: string + appId?: string | null + startDate: string + endDate: string + }, +): Promise> { + const rows: AppMetricRow[] = [] + + for (let pageStart = 0; ; pageStart += APP_METRICS_PAGE_SIZE) { + const { data, error, status } = await executeStatsQueryWithRetry( + c, + params.appId ? 'get_app_metrics_for_app' : 'get_app_metrics_for_org', + async () => { + // Org dashboards return apps × days. Without range pagination, PostgREST + // max_rows (1000) silently drops later app_ids — single-app RPC still works + // because it filters inside SQL before the row limit applies. + const query = params.appId + ? supabase.rpc('get_app_metrics' as any, { + p_org_id: params.orgId, + p_app_id: params.appId, + p_start_date: params.startDate, + p_end_date: params.endDate, + }) + : supabase.rpc('get_app_metrics', { + org_id: params.orgId, + start_date: params.startDate, + end_date: params.endDate, + }) + + return await query + .order('app_id', { ascending: true }) + .order('date', { ascending: true }) + .range(pageStart, pageStart + APP_METRICS_PAGE_SIZE - 1) as Promise> + }, + ) + + if (error) + return { data: null, error, status } + + rows.push(...(data ?? [])) + if ((data?.length ?? 0) < APP_METRICS_PAGE_SIZE) + break + } + + return { data: rows, error: null } +} + +function metricDayNumber(metricDate: string, from: Date, graphDays: number): number | null { + const dayNumber = dayjs(metricDate).utc().startOf('day').diff(dayjs(from).utc().startOf('day'), 'day') + if (dayNumber < 0 || dayNumber >= graphDays) + return null + return dayNumber +} + export const statisticsTestUtils = { + APP_METRICS_PAGE_SIZE, executeStatsQueryWithRetry, + fetchAppMetricsRows, getMissingAppStatsError, getRetryableStatus: getRetryablePostgrestStatus, isRetryableStatsError, getStatsAppOwnerOrgOrThrow, + metricDayNumber, resolveAppOwnerOrg, } @@ -390,32 +454,12 @@ async function getNormalStats(c: Context, appId: string | null, ownerOrg: string const startDate = dayjs(from).utc().format('YYYY-MM-DD') const endDate = dayjs(to).utc().format('YYYY-MM-DD') - let rawMetrics: AppMetricRow[] | null - let metricsError: unknown - - if (appId) { - ({ data: rawMetrics, error: metricsError } = await executeStatsQueryWithRetry( - c, - 'get_app_metrics_for_app', - async () => await supabase.rpc('get_app_metrics' as any, { - p_org_id: ownerOrgId!, - p_app_id: appId, - p_start_date: startDate, - p_end_date: endDate, - }) as QueryResult, - )) - } - else { - ({ data: rawMetrics, error: metricsError } = await executeStatsQueryWithRetry( - c, - 'get_app_metrics_for_org', - async () => await supabase.rpc('get_app_metrics', { - org_id: ownerOrgId!, - start_date: startDate, - end_date: endDate, - }) as QueryResult, - )) - } + const { data: rawMetrics, error: metricsError } = await fetchAppMetricsRows(c, supabase, { + orgId: ownerOrgId!, + appId, + startDate, + endDate, + }) if (metricsError) return { data: null, error: metricsError } @@ -478,36 +522,20 @@ async function getNormalStats(c: Context, appId: string | null, ownerOrg: string .forEach((arrItem) => { const sortedArrItem = arrItem.sort((a, b) => new Date(a.date).getTime() - new Date(b.date).getTime()) cloudlog({ requestId: c.get('requestId'), message: 'sortedArrItem', data: sortedArrItem }) - arrItem?.sort((a, b) => new Date(a.date).getTime() - new Date(b.date).getTime()).forEach((item, i) => { - if (item.date) { - const dayNumber = i - if (mau[dayNumber]) - mau[dayNumber] += item.mau - else - mau[dayNumber] = item.mau - - const storageVal = item.storage - if (storage[dayNumber]) - storage[dayNumber] += storageVal - else - storage[dayNumber] = storageVal - - const bandwidthVal = item.bandwidth ?? 0 - if (bandwidth[dayNumber]) - bandwidth[dayNumber] += bandwidthVal - else - bandwidth[dayNumber] = bandwidthVal - - const buildTimeVal = item.build_time_unit ?? 0 - if (buildTime[dayNumber]) - buildTime[dayNumber] += buildTimeVal - else - buildTime[dayNumber] = buildTimeVal - - if (isDashboard) { - gets[dayNumber] = item.get - } - } + sortedArrItem.forEach((item) => { + if (!item.date) + return + const dayNumber = metricDayNumber(item.date, from, graphDays) + if (dayNumber === null) + return + + mau[dayNumber] = (mau[dayNumber] ?? 0) + item.mau + storage[dayNumber] = (storage[dayNumber] ?? 0) + item.storage + bandwidth[dayNumber] = (bandwidth[dayNumber] ?? 0) + (item.bandwidth ?? 0) + buildTime[dayNumber] = (buildTime[dayNumber] ?? 0) + (item.build_time_unit ?? 0) + + if (isDashboard) + gets[dayNumber] = (gets[dayNumber] ?? 0) + item.get }) }) @@ -576,36 +604,20 @@ async function getNormalStats(c: Context, appId: string | null, ownerOrg: string let appGets = isDashboard ? createUndefinedArray(graphDays) as number[] : [] // Process metrics for this app (same logic as aggregated version) - appMetrics.sort((a, b) => new Date(a.date).getTime() - new Date(b.date).getTime()).forEach((item, i) => { - if (item.date) { - const dayNumber = i - if (appMau[dayNumber]) - appMau[dayNumber] += item.mau - else - appMau[dayNumber] = item.mau - - const storageVal = item.storage - if (appStorage[dayNumber]) - appStorage[dayNumber] += storageVal - else - appStorage[dayNumber] = storageVal - - const bandwidthVal = item.bandwidth ?? 0 - if (appBandwidth[dayNumber]) - appBandwidth[dayNumber] += bandwidthVal - else - appBandwidth[dayNumber] = bandwidthVal - - const buildTimeVal = item.build_time_unit ?? 0 - if (appBuildTime[dayNumber]) - appBuildTime[dayNumber] += buildTimeVal - else - appBuildTime[dayNumber] = buildTimeVal - - if (isDashboard) { - appGets[dayNumber] = item.get - } - } + appMetrics.sort((a, b) => new Date(a.date).getTime() - new Date(b.date).getTime()).forEach((item) => { + if (!item.date) + return + const dayNumber = metricDayNumber(item.date, from, graphDays) + if (dayNumber === null) + return + + appMau[dayNumber] = (appMau[dayNumber] ?? 0) + item.mau + appStorage[dayNumber] = (appStorage[dayNumber] ?? 0) + item.storage + appBandwidth[dayNumber] = (appBandwidth[dayNumber] ?? 0) + (item.bandwidth ?? 0) + appBuildTime[dayNumber] = (appBuildTime[dayNumber] ?? 0) + (item.build_time_unit ?? 0) + + if (isDashboard) + appGets[dayNumber] = (appGets[dayNumber] ?? 0) + item.get }) // Accumulate data if requested (default behavior for backward compatibility) diff --git a/tests/org-statistics-bandwidth-pagination.test.ts b/tests/org-statistics-bandwidth-pagination.test.ts new file mode 100644 index 0000000000..246a028b6d --- /dev/null +++ b/tests/org-statistics-bandwidth-pagination.test.ts @@ -0,0 +1,152 @@ +import { randomUUID } from 'node:crypto' +import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { + BASE_URL, + executeSQL, + getAuthHeaders, + getSupabaseClient, + USER_EMAIL, + USER_ID, +} from './test-utils.ts' + +// PostgREST max_rows is 1000. Org dashboards return apps × days; a single-app +// RPC filters in SQL and stays under the limit, so app pages look fine while +// the all-apps org dashboard silently loses later app_ids (and their bandwidth). +const DAYS = 30 +const QUIET_APPS = 40 // 40 × 30 = 1200 rows > 1000 +const expectedBandwidth = 5_242_880 +const orgId = randomUUID() +const testPrefix = `org.bw.page.${randomUUID().slice(0, 8)}` +const quietAppPrefix = `com.${testPrefix}.aaa.` +const busyAppId = `com.${testPrefix}.zzz.busy` + +function rangeDates() { + const end = new Date() + end.setUTCHours(0, 0, 0, 0) + const start = new Date(end) + start.setUTCDate(start.getUTCDate() - (DAYS - 1)) + return { + startDate: start.toISOString().slice(0, 10), + endDate: end.toISOString().slice(0, 10), + } +} + +async function seedQuietApps(count: number) { + await executeSQL( + ` + INSERT INTO public.apps (app_id, icon_url, name, last_version, owner_org, user_id, created_at, updated_at) + SELECT + $1 || lpad(i::text, 3, '0'), + '', + 'Quiet org bandwidth app ' || i::text, + '1.0.0', + $2::uuid, + $3::uuid, + NOW() - INTERVAL '90 days', + NOW() - INTERVAL '90 days' + FROM generate_series(1, $4::integer) AS i + `, + [quietAppPrefix, orgId, USER_ID, count], + ) +} + +describe('org dashboard bandwidth pagination', () => { + const { startDate, endDate } = rangeDates() + + beforeAll(async () => { + // created_by bootstraps org_users + org_super_admin role_bindings + await getSupabaseClient().from('orgs').insert({ + created_by: USER_ID, + id: orgId, + management_email: USER_EMAIL, + name: `Org Bandwidth Pagination ${orgId}`, + updated_at: new Date().toISOString(), + }).throwOnError() + + await seedQuietApps(QUIET_APPS) + + await getSupabaseClient().from('apps').insert({ + app_id: busyAppId, + created_at: new Date().toISOString(), + icon_url: '', + last_version: '1.0.0', + name: 'Busy org bandwidth app', + owner_org: orgId, + updated_at: new Date().toISOString(), + user_id: USER_ID, + }).throwOnError() + + await getSupabaseClient().from('daily_bandwidth').upsert({ + app_id: busyAppId, + date: endDate, + bandwidth: expectedBandwidth, + }, { + onConflict: 'app_id,date', + }).throwOnError() + + await getSupabaseClient().from('app_metrics_cache').delete().eq('org_id', orgId) + }, 120_000) + + afterAll(async () => { + await getSupabaseClient().from('daily_bandwidth').delete().eq('app_id', busyAppId) + await getSupabaseClient().from('app_metrics_cache').delete().eq('org_id', orgId) + await executeSQL(`DELETE FROM public.apps WHERE app_id LIKE $1`, [`${quietAppPrefix}%`]) + await getSupabaseClient().from('apps').delete().eq('app_id', busyAppId) + await getSupabaseClient().from('role_bindings').delete().eq('org_id', orgId) + await getSupabaseClient().from('org_users').delete().eq('org_id', orgId) + await getSupabaseClient().from('orgs').delete().eq('id', orgId) + }, 120_000) + + it('proves unpaginated org metrics drop the late busy app under max_rows', async () => { + const { data, error } = await getSupabaseClient() + .rpc('get_app_metrics', { + org_id: orgId, + start_date: startDate, + end_date: endDate, + }) + .order('app_id', { ascending: true }) + .order('date', { ascending: true }) + .range(0, 999) + + expect(error).toBeNull() + expect(data?.length).toBe(1000) + expect(data?.some(row => row.app_id === busyAppId)).toBe(false) + + const { data: appOnly, error: appError } = await getSupabaseClient().rpc('get_app_metrics' as any, { + p_org_id: orgId, + p_app_id: busyAppId, + p_start_date: startDate, + p_end_date: endDate, + }) + + expect(appError).toBeNull() + const busyRow = (appOnly as any[])?.find(row => row.app_id === busyAppId && row.date === endDate) + expect(busyRow?.bandwidth).toBe(expectedBandwidth) + }, 60_000) + + it('returns busy-app bandwidth on org dashboard and app dashboard', async () => { + const authHeaders = await getAuthHeaders() + + const orgRes = await fetch( + `${BASE_URL}/statistics/org/${orgId}?from=${startDate}&to=${endDate}&breakdown=true&noAccumulate=true`, + { method: 'GET', headers: authHeaders }, + ) + const orgBody = await orgRes.json() as { global: any[], byApp: any[] } + expect(orgRes.status, JSON.stringify(orgBody)).toBe(200) + + const orgBandwidth = (orgBody.global ?? []).reduce((sum, row) => sum + Number(row.bandwidth ?? 0), 0) + expect(orgBandwidth).toBe(expectedBandwidth) + expect((orgBody.byApp ?? []).some(row => + row.app_id === busyAppId && Number(row.bandwidth ?? 0) === expectedBandwidth, + )).toBe(true) + + const appRes = await fetch( + `${BASE_URL}/statistics/app/${busyAppId}?from=${startDate}&to=${endDate}&noAccumulate=true`, + { method: 'GET', headers: authHeaders }, + ) + const appBody = await appRes.json() as any[] + expect(appRes.status, JSON.stringify(appBody)).toBe(200) + const appBandwidth = (appBody ?? []).reduce((sum, row) => sum + Number(row.bandwidth ?? 0), 0) + expect(appBandwidth).toBe(expectedBandwidth) + }, 120_000) +}) diff --git a/tests/org-statistics-bandwidth-pagination.unit.test.ts b/tests/org-statistics-bandwidth-pagination.unit.test.ts new file mode 100644 index 0000000000..0d1bab7084 --- /dev/null +++ b/tests/org-statistics-bandwidth-pagination.unit.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it, vi } from 'vitest' +import { statisticsTestUtils } from '../supabase/functions/_backend/public/statistics/index.ts' + +const fakeContext = { + get: vi.fn(() => undefined), +} as any + +function makeMetric(appId: string, date: string, bandwidth = 0) { + return { + app_id: appId, + date, + mau: 0, + storage: 0, + bandwidth, + build_time_unit: 0, + get: 0, + fail: 0, + install: 0, + uninstall: 0, + } +} + +function createPagedRpcClient(pages: ReturnType[][]) { + const rangeCalls: Array<[number, number]> = [] + + return { + rangeCalls, + supabase: { + rpc: vi.fn(() => { + const builder: { + order: ReturnType + range: ReturnType + } = { + order: vi.fn(() => builder), + range: vi.fn(async (from: number, to: number) => { + rangeCalls.push([from, to]) + const pageIndex = Math.floor(from / statisticsTestUtils.APP_METRICS_PAGE_SIZE) + return { + data: pages[pageIndex] ?? [], + error: null, + status: 200, + } + }), + } + return builder + }), + } as any, + } +} + +describe('org statistics app metrics pagination', () => { + it('pages past PostgREST max_rows so late app_ids are not dropped', async () => { + const pageSize = statisticsTestUtils.APP_METRICS_PAGE_SIZE + expect(pageSize).toBe(1000) + + const firstPage = Array.from({ length: pageSize }, (_, index) => + makeMetric(`com.early.app.${String(index).padStart(4, '0')}`, '2026-07-01')) + const lateBandwidth = 7_340_032 + const secondPage = [ + makeMetric('com.zzz.busy.app', '2026-07-01', lateBandwidth), + makeMetric('com.zzz.busy.app', '2026-07-02', lateBandwidth), + ] + const { supabase, rangeCalls } = createPagedRpcClient([firstPage, secondPage]) + + const result = await statisticsTestUtils.fetchAppMetricsRows(fakeContext, supabase, { + orgId: '11111111-1111-4111-8111-111111111111', + startDate: '2026-07-01', + endDate: '2026-07-30', + }) + + expect(result.error).toBeNull() + expect(result.data).toHaveLength(pageSize + secondPage.length) + expect(rangeCalls).toEqual([ + [0, pageSize - 1], + [pageSize, pageSize * 2 - 1], + ]) + expect(result.data?.some(row => row.app_id === 'com.zzz.busy.app' && row.bandwidth === lateBandwidth)).toBe(true) + }) + + it('maps metric dates to day indexes instead of array position', () => { + const from = new Date('2026-07-01T00:00:00.000Z') + expect(statisticsTestUtils.metricDayNumber('2026-07-01', from, 30)).toBe(0) + expect(statisticsTestUtils.metricDayNumber('2026-07-15', from, 30)).toBe(14) + expect(statisticsTestUtils.metricDayNumber('2026-06-30', from, 30)).toBeNull() + expect(statisticsTestUtils.metricDayNumber('2026-07-31', from, 30)).toBeNull() + }) +}) From ded419016967c2d89a14575299386a5b138e8c39 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 23 Jul 2026 15:03:46 +0000 Subject: [PATCH 2/6] fix(statistics): correct get_app_metrics page query typing Satisfy backend typecheck for the PostgREST range pagination chain. Co-authored-by: Martin DONADIEU --- supabase/functions/_backend/public/statistics/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/supabase/functions/_backend/public/statistics/index.ts b/supabase/functions/_backend/public/statistics/index.ts index 4d53bf5011..8dda9c65c1 100644 --- a/supabase/functions/_backend/public/statistics/index.ts +++ b/supabase/functions/_backend/public/statistics/index.ts @@ -309,10 +309,10 @@ async function fetchAppMetricsRows( end_date: params.endDate, }) - return await query + return await (query as any) .order('app_id', { ascending: true }) .order('date', { ascending: true }) - .range(pageStart, pageStart + APP_METRICS_PAGE_SIZE - 1) as Promise> + .range(pageStart, pageStart + APP_METRICS_PAGE_SIZE - 1) as QueryResult }, ) From 0cf2e5bd3ba7dfb063c1b46db906a5759040d47c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 23 Jul 2026 15:13:17 +0000 Subject: [PATCH 3/6] test(statistics): seed org bandwidth inside exclusive date window getDaysBetweenDates excludes the end date from graph indexes, so end-day bandwidth never appears in /statistics aggregates. Co-authored-by: Martin DONADIEU --- tests/org-statistics-bandwidth-pagination.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/org-statistics-bandwidth-pagination.test.ts b/tests/org-statistics-bandwidth-pagination.test.ts index 246a028b6d..59705b5ea8 100644 --- a/tests/org-statistics-bandwidth-pagination.test.ts +++ b/tests/org-statistics-bandwidth-pagination.test.ts @@ -78,7 +78,7 @@ describe('org dashboard bandwidth pagination', () => { await getSupabaseClient().from('daily_bandwidth').upsert({ app_id: busyAppId, - date: endDate, + date: bandwidthDate, bandwidth: expectedBandwidth, }, { onConflict: 'app_id,date', From 9433edaffa648414ff6b8347dd62c79f51d9d636 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 23 Jul 2026 15:13:32 +0000 Subject: [PATCH 4/6] test(statistics): define mid-range bandwidthDate for org pagination fixture Co-authored-by: Martin DONADIEU --- tests/org-statistics-bandwidth-pagination.test.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/org-statistics-bandwidth-pagination.test.ts b/tests/org-statistics-bandwidth-pagination.test.ts index 59705b5ea8..97538240ae 100644 --- a/tests/org-statistics-bandwidth-pagination.test.ts +++ b/tests/org-statistics-bandwidth-pagination.test.ts @@ -25,9 +25,14 @@ function rangeDates() { end.setUTCHours(0, 0, 0, 0) const start = new Date(end) start.setUTCDate(start.getUTCDate() - (DAYS - 1)) + // getDaysBetweenDates(from, to) is exclusive of `to`, so put bandwidth on a + // day that is inside the generated graph day indexes (not the end date). + const bandwidthDate = new Date(start) + bandwidthDate.setUTCDate(bandwidthDate.getUTCDate() + 10) return { startDate: start.toISOString().slice(0, 10), endDate: end.toISOString().slice(0, 10), + bandwidthDate: bandwidthDate.toISOString().slice(0, 10), } } @@ -51,7 +56,7 @@ async function seedQuietApps(count: number) { } describe('org dashboard bandwidth pagination', () => { - const { startDate, endDate } = rangeDates() + const { startDate, endDate, bandwidthDate } = rangeDates() beforeAll(async () => { // created_by bootstraps org_users + org_super_admin role_bindings @@ -120,7 +125,7 @@ describe('org dashboard bandwidth pagination', () => { }) expect(appError).toBeNull() - const busyRow = (appOnly as any[])?.find(row => row.app_id === busyAppId && row.date === endDate) + const busyRow = (appOnly as any[])?.find(row => row.app_id === busyAppId && row.date === bandwidthDate) expect(busyRow?.bandwidth).toBe(expectedBandwidth) }, 60_000) From d4a229238c0c31bd0cb29407bf7f802eac7493a6 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 23 Jul 2026 15:21:02 +0000 Subject: [PATCH 5/6] chore(ci): retrigger tests after unrelated cloudflare flake Co-authored-by: Martin DONADIEU From 98901d8fd30800200d8dcd1e52cb5cfd3e3efccc Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 23 Jul 2026 15:28:26 +0000 Subject: [PATCH 6/6] chore(ci): retrigger after unrelated cloudflare invite flake Co-authored-by: Martin DONADIEU