-
-
Notifications
You must be signed in to change notification settings - Fork 135
fix(stats): unify success rate formula across admin and public metrics #2988
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2124,8 +2124,8 @@ export async function getUpdateStatsCF(c: Context): Promise<UpdateStats> { | |
| const apps = result | ||
| .filter(app => app.get > 0) | ||
| .map((app) => { | ||
| const totalEvents = app.set + app.get | ||
| const successRate = Number(Number(totalEvents > 0 ? (app.get / totalEvents) * 100 : 100).toFixed(2)) | ||
| const totalOutcomes = app.set + app.failed | ||
| const successRate = Number(Number(totalOutcomes > 0 ? (app.set / totalOutcomes) * 100 : 100).toFixed(2)) | ||
| return { | ||
| ...app, | ||
| success_rate: successRate, | ||
|
|
@@ -2140,8 +2140,8 @@ export async function getUpdateStatsCF(c: Context): Promise<UpdateStats> { | |
| return acc | ||
| }, { failed: 0, set: 0, get: 0 }) | ||
|
|
||
| const totalEvents = total.set + total.get | ||
| const totalSuccessRate = totalEvents > 0 ? (total.get / totalEvents) * 100 : 100 | ||
| const totalOutcomes = total.set + total.failed | ||
| const totalSuccessRate = totalOutcomes > 0 ? (total.set / totalOutcomes) * 100 : 100 | ||
|
|
||
| return { | ||
| apps, | ||
|
|
@@ -3086,7 +3086,7 @@ function buildBreakdownMetrics( | |
| } | ||
|
|
||
| export async function getPublicLiveUpdateMetricsCF(c: Context, referenceDate = new Date()): Promise<PublicLiveUpdateMetrics> { | ||
| if (!c.env.APP_LOG || !c.env.DEVICE_USAGE || !c.env.DEVICE_INFO || !getEnv(c, 'CF_ANALYTICS_TOKEN') || !getEnv(c, 'CF_ACCOUNT_ANALYTICS_ID')) | ||
| if (!c.env.APP_LOG || !c.env.VERSION_USAGE || !c.env.DEVICE_USAGE || !c.env.DEVICE_INFO || !getEnv(c, 'CF_ANALYTICS_TOKEN') || !getEnv(c, 'CF_ACCOUNT_ANALYTICS_ID')) | ||
| throw new Error('Public live update metric bindings are unavailable') | ||
|
|
||
| const end = new Date(Date.UTC(referenceDate.getUTCFullYear(), referenceDate.getUTCMonth(), referenceDate.getUTCDate())) | ||
|
|
@@ -3095,17 +3095,17 @@ export async function getPublicLiveUpdateMetricsCF(c: Context, referenceDate = n | |
| const window = `timestamp >= toDateTime('${formatDateCF(start)}') AND timestamp < toDateTime('${formatDateCF(end)}')` | ||
| const failureActions = PUBLIC_FAILURE_ACTIONS.map(action => `'${action}'`).join(', ') | ||
| const day = `formatDateTime(toStartOfInterval(timestamp, INTERVAL '1' DAY), '%Y-%m-%d')` | ||
| const outcomeBase = `SELECT ${day} AS date, index1 AS app_id, blob1 AS device_id, max(if(blob2 = 'set', 1, 0)) AS succeeded, max(if(blob2 IN (${failureActions}), 1, 0)) AS failed, argMax(blob5, timestamp) AS platform, argMax(blob6, timestamp) AS country, argMax(blob7, timestamp) AS plugin_version FROM app_log WHERE ${window} AND (blob2 = 'set' OR blob2 IN (${failureActions})) GROUP BY date, app_id, device_id` | ||
| const outcomesQuery = `SELECT date, sum(succeeded) AS successes, sum(if(succeeded = 0, failed, 0)) AS failures FROM (${outcomeBase}) GROUP BY date` | ||
| const appLogOutcomeFilter = `(blob2 = 'set' OR blob2 IN (${failureActions}))` | ||
| const dailySuccessQuery = `SELECT ${day} AS date, sum(if(blob3 = 'install', 1, 0)) AS installs, sum(if(blob3 = 'fail', 1, 0)) AS fails FROM version_usage WHERE ${window} GROUP BY date ORDER BY date ASC` | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: The headline success rate and the per-dimension (platform/country/version) success rates are now computed from two different Analytics Engine datasets with different failure taxonomies. The daily/total headline reads VERSION_USAGE (install/fail — a single generic 'fail' action), while the breakdown queries read APP_LOG (set vs. the granular PUBLIC_FAILURE_ACTIONS list). Unless those two datasets are guaranteed to record byte-identical event streams, the published headline and the dimensional success rates will disagree, which is exactly the cross-source discrepancy this PR is meant to remove. Consider driving the headline and the breakdowns from the same source (e.g. compute the daily/total aggregates from the same app_log/set + failure-actions query as the breakdowns, or derive the breakdowns from version_usage), so a single consistent success rate is published. Prompt for AI agents |
||
| const failuresQuery = `SELECT action, count() AS devices FROM (SELECT ${day} AS date, blob2 AS action, index1 AS app_id, blob1 AS device_id FROM app_log WHERE ${window} AND blob2 IN (${failureActions}) GROUP BY date, action, app_id, device_id) GROUP BY action` | ||
| const platformsShareQuery = `SELECT platform, count() AS devices FROM (SELECT double1 AS platform, index1 AS app_id, blob1 AS device_id FROM device_usage WHERE ${window} AND double1 IN (0.0, 1.0, 2.0) GROUP BY platform, app_id, device_id) GROUP BY platform` | ||
| const platformsOutcomeQuery = `SELECT platform AS key, sum(succeeded) AS successes, sum(if(succeeded = 0, failed, 0)) AS failures FROM (${outcomeBase}) WHERE platform IN ('ios', 'android', 'electron') GROUP BY platform` | ||
| const platformsOutcomeQuery = `SELECT blob5 AS key, sum(if(blob2 = 'set', 1, 0)) AS successes, sum(if(blob2 IN (${failureActions}), 1, 0)) AS failures FROM app_log WHERE ${window} AND ${appLogOutcomeFilter} AND blob5 IN ('ios', 'android', 'electron') GROUP BY blob5` | ||
| const platformsFailureQuery = `SELECT platform AS key, action, count() AS devices FROM (SELECT ${day} AS date, index1 AS app_id, blob1 AS device_id, blob2 AS action, argMax(blob5, timestamp) AS platform FROM app_log WHERE ${window} AND blob2 IN (${failureActions}) GROUP BY date, app_id, device_id, action) WHERE platform IN ('ios', 'android', 'electron') GROUP BY platform, action` | ||
| const countriesShareQuery = `SELECT country AS key, count() AS devices FROM (SELECT index1 AS app_id, blob1 AS device_id, argMax(blob10, timestamp) AS country FROM device_info WHERE ${window} AND blob10 != '' GROUP BY app_id, device_id) WHERE country != '' GROUP BY country` | ||
| const countriesOutcomeQuery = `SELECT country AS key, sum(succeeded) AS successes, sum(if(succeeded = 0, failed, 0)) AS failures FROM (${outcomeBase}) WHERE country != '' GROUP BY country` | ||
| const countriesOutcomeQuery = `SELECT blob6 AS key, sum(if(blob2 = 'set', 1, 0)) AS successes, sum(if(blob2 IN (${failureActions}), 1, 0)) AS failures FROM app_log WHERE ${window} AND ${appLogOutcomeFilter} AND blob6 != '' GROUP BY blob6` | ||
| const countriesFailureQuery = `SELECT country AS key, action, count() AS devices FROM (SELECT ${day} AS date, index1 AS app_id, blob1 AS device_id, blob2 AS action, argMax(blob6, timestamp) AS country FROM app_log WHERE ${window} AND blob2 IN (${failureActions}) GROUP BY date, app_id, device_id, action) WHERE country != '' GROUP BY country, action` | ||
| const versionsShareQuery = `SELECT version AS key, count() AS devices FROM (SELECT index1 AS app_id, blob1 AS device_id, argMax(blob3, timestamp) AS version FROM device_info WHERE ${window} AND blob3 != '' GROUP BY app_id, device_id) WHERE version != '' GROUP BY version` | ||
| const versionsOutcomeQuery = `SELECT plugin_version AS key, sum(succeeded) AS successes, sum(if(succeeded = 0, failed, 0)) AS failures FROM (${outcomeBase}) WHERE plugin_version != '' GROUP BY plugin_version` | ||
| const versionsOutcomeQuery = `SELECT blob7 AS key, sum(if(blob2 = 'set', 1, 0)) AS successes, sum(if(blob2 IN (${failureActions}), 1, 0)) AS failures FROM app_log WHERE ${window} AND ${appLogOutcomeFilter} AND blob7 != '' GROUP BY blob7` | ||
| const versionsFailureQuery = `SELECT plugin_version AS key, action, count() AS devices FROM (SELECT ${day} AS date, index1 AS app_id, blob1 AS device_id, blob2 AS action, argMax(blob7, timestamp) AS plugin_version FROM app_log WHERE ${window} AND blob2 IN (${failureActions}) GROUP BY date, app_id, device_id, action) WHERE plugin_version != '' GROUP BY plugin_version, action` | ||
|
|
||
| try { | ||
|
|
@@ -3122,7 +3122,7 @@ export async function getPublicLiveUpdateMetricsCF(c: Context, referenceDate = n | |
| versionOutcomeRows, | ||
| versionFailureRows, | ||
| ] = await Promise.all([ | ||
| runQueryToCFA<{ date: string, successes: number, failures: number }>(c, outcomesQuery), | ||
| runQueryToCFA<{ date: string, installs: number, fails: number }>(c, dailySuccessQuery), | ||
| runQueryToCFA<{ action: string, devices: number }>(c, failuresQuery), | ||
| runQueryToCFA<{ platform: number, devices: number }>(c, platformsShareQuery), | ||
| runQueryToCFA<{ key: string, successes: number, failures: number }>(c, platformsOutcomeQuery), | ||
|
|
@@ -3135,15 +3135,15 @@ export async function getPublicLiveUpdateMetricsCF(c: Context, referenceDate = n | |
| runQueryToCFA<{ key: string, action: string, devices: number }>(c, versionsFailureQuery), | ||
| ]) | ||
| const daily = outcomeRows.map((row) => { | ||
| const successes = Number(row.successes) || 0 | ||
| const failures = Number(row.failures) || 0 | ||
| const outcomes = successes + failures | ||
| return { date: row.date, success_rate: outcomes ? roundPublicPercent((successes / outcomes) * 100) : 0 } | ||
| const installs = Number(row.installs) || 0 | ||
| const fails = Number(row.fails) || 0 | ||
| const outcomes = installs + fails | ||
| return { date: row.date, success_rate: outcomes ? roundPublicPercent((installs / outcomes) * 100) : 0 } | ||
| }).sort((a, b) => a.date.localeCompare(b.date)) | ||
| const totalSuccesses = outcomeRows.reduce((sum, row) => sum + (Number(row.successes) || 0), 0) | ||
| const totalFailures = outcomeRows.reduce((sum, row) => sum + (Number(row.failures) || 0), 0) | ||
| const totalOutcomes = totalSuccesses + totalFailures | ||
| const success_rate = totalOutcomes ? roundPublicPercent((totalSuccesses / totalOutcomes) * 100) : 0 | ||
| const totalInstalls = outcomeRows.reduce((sum, row) => sum + (Number(row.installs) || 0), 0) | ||
| const totalFails = outcomeRows.reduce((sum, row) => sum + (Number(row.fails) || 0), 0) | ||
| const totalOutcomes = totalInstalls + totalFails | ||
| const success_rate = totalOutcomes ? roundPublicPercent((totalInstalls / totalOutcomes) * 100) : 0 | ||
| const failureTotal = failureRows.reduce((sum, row) => sum + (Number(row.devices) || 0), 0) | ||
| const failures = [...failureRows] | ||
| .map(row => ({ reason: row.action, devices: Number(row.devices) || 0 })) | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
P1: Global success_rate still excludes install/fail outcomes whenever their app has no
getevent in the same 60-second slice, because the total is reduced after the existingapp.get > 0filter. Removing that filter (or filtering only on the outcome fields) would make the new install/(install+fail) calculation cover all version_usage outcomes and match the admin/global_stats view.Prompt for AI agents