ci(e2e): unique CMT failure counts + restore per-OS commit statuses

.github/workflows/compatibility-matrix-testing.yml

@@ -107,6 +107,33 @@ jobs:
           description: "Compatibility Matrix Testing for ${{ inputs.DESKTOP_VERSION }} version"
           status: pending
 
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        continue-on-error: true
+        with:
+          ref: ${{ inputs.DESKTOP_VERSION }}
+          persist-credentials: false
+          sparse-checkout: |
+            e2e/utils/github-actions.js
+          sparse-checkout-cone-mode: false
+
+      - name: Post pending e2e/linux|macos|windows
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        continue-on-error: true
+        env:
+          DESKTOP_SHA: ${{ needs.calculate-commit-hash.outputs.DESKTOP_SHA }}
+          CMT_MATRIX: ${{ needs.calculate-commit-hash.outputs.CMT_MATRIX }}
+        with:
+          script: |
+            const {updateInitialOsStatuses} = require('./e2e/utils/github-actions.js');
+            const matrix = JSON.parse(process.env.CMT_MATRIX || '{}');
+            const platforms = (matrix.environment || []).map((env) => ({platform: env.os}));
+            await updateInitialOsStatuses({
+              github,
+              context,
+              sha: process.env.DESKTOP_SHA,
+              platforms,
+            });
+
   # Input follows the below schema (Matterwick ≥ v0.4.16).
   # {
   #   "environment": [
@@ -178,9 +205,10 @@ jobs:
           sparse-checkout: |
             e2e/utils/tsio-report-status.js
             e2e/utils/cmt-channel-notify.js
+            e2e/utils/github-actions.js
           sparse-checkout-cone-mode: false
 
-      - name: Render TSIO summary + flip commit status
+      - name: Render TSIO summary + flip commit statuses
         id: summary
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         env:
@@ -199,13 +227,23 @@ jobs:
           # matching failed test (hung worker teardown, crashed runner, npm ci
           # failure) would otherwise still read as "100% passed".
           UPSTREAM_JOBS_SUCCEEDED: ${{ needs.e2e.result == 'success' }}
+          CMT_MATRIX: ${{ needs.calculate-commit-hash.outputs.CMT_MATRIX }}
         with:
           script: |
+            const {canonicalizeOs} = require('./e2e/utils/github-actions.js');
+            const matrix = JSON.parse(process.env.CMT_MATRIX || '{}');
+            const expectedOs = [...new Set(
+              (matrix.environment || []).
+                map((env) => canonicalizeOs(env.os, env.runner)).
+                filter(Boolean),
+            )];
             const {reportUrl, status, stats} = await require('./e2e/utils/tsio-report-status.js')({
               core, context, github,
               compositeIdentity: JSON.parse(process.env.TSIO_COMPOSITE_IDENTITY),
               totalReportsExpected: parseInt(process.env.TSIO_TOTAL_REPORTS_EXPECTED, 10),
               commitStatusContext: process.env.COMMIT_STATUS_CONTEXT,
+              perOsCommitStatuses: true,
+              expectedOs,
               upstreamJobsSucceeded: process.env.UPSTREAM_JOBS_SUCCEEDED === 'true',
               failOnTestFailures: true,
             });

.github/workflows/e2e-functional.yml

@@ -58,7 +58,30 @@ jobs:
         run: |
           # Matterwick still dispatches macos-latest; pin explicitly so the job
           # does not drift when GitHub retargets that label to macOS 26.
-          platforms=$(echo "${INSTANCE_DETAILS}" | jq -c 'map(if .runner == "macos-latest" then .runner = "macos-26" else . end)')
+          # Normalize platform to canonical linux|macos|windows (derive from runner
+          # when needed) so job names and e2e/<os> statuses stay consistent.
+          platforms=$(echo "${INSTANCE_DETAILS}" | jq -c '
+            map(
+              (if .runner == "macos-latest" then .runner = "macos-26" else . end) |
+              . as $row |
+              ($row.platform // $row.os // "") as $raw |
+              (
+                if $raw == "linux" or $raw == "macos" or $raw == "windows" then $raw
+                elif (($row.runner // "") | test("^(ubuntu|linux)")) then "linux"
+                elif (($row.runner // "") | test("^(macos|darwin)")) then "macos"
+                elif (($row.runner // "") | test("^windows")) then "windows"
+                else null
+                end
+              ) as $canon |
+              if $canon == null then empty
+              else ($row | .platform = $canon | del(.os))
+              end
+            )
+          ')
+          if [ "$(echo "${platforms}" | jq 'length')" -eq 0 ]; then
+            echo "No supported platforms in instance_details (need linux|macos|windows)" >&2
+            exit 1
+          fi
           echo "platforms=${platforms}" >> "$GITHUB_OUTPUT"
 
       # Resolve the SHA under test from version_name (branch/tag/SHA), not github.sha
@@ -98,7 +121,7 @@ jobs:
           echo "composite-identity-json=${COMPOSITE_IDENTITY}" >> "$GITHUB_OUTPUT"
 
   update-initial-status:
-    name: Set pending TSIO status
+    name: Set pending E2E OS statuses
     runs-on: ubuntu-24.04
     needs:
       - prepare-matrix
@@ -108,16 +131,31 @@ jobs:
     steps:
       # Best-effort: a transient status-API hiccup here must never block the
       # actual test matrix from running.
-      - uses: mattermost/actions/delivery/update-commit-status@218fd96a63451259dc100e1292a5e44af92fe15d
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        continue-on-error: true
+        with:
+          ref: ${{ inputs.version_name }}
+          persist-credentials: false
+          sparse-checkout: |
+            e2e/utils/github-actions.js
+          sparse-checkout-cone-mode: false
+
+      - name: Post pending e2e/linux|macos|windows (+ policy)
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         continue-on-error: true
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PLATFORMS: ${{ needs.prepare-matrix.outputs.platforms }}
+          DESKTOP_SHA: ${{ needs.prepare-matrix.outputs.desktop-sha }}
         with:
-          repository_full_name: ${{ github.repository }}
-          commit_sha: ${{ needs.prepare-matrix.outputs.desktop-sha }}
-          context: e2e-test/desktop-playwright
-          description: "Running Electron Playwright E2E tests..."
-          status: pending
+          script: |
+            const {updateInitialOsStatuses} = require('./e2e/utils/github-actions.js');
+            await updateInitialOsStatuses({
+              github,
+              context,
+              sha: process.env.DESKTOP_SHA,
+              platforms: JSON.parse(process.env.PLATFORMS || '[]'),
+              includePolicy: true,
+            });
 
   e2e-tests:
     needs:
@@ -144,9 +182,8 @@ jobs:
       tsio-config: ${{ format('{{"composite_identity":{0},"total_reports_expected":"{1}"}}', needs.prepare-matrix.outputs.tsio-composite-identity, needs.prepare-matrix.outputs.tsio-total-reports-expected) }}
     secrets: inherit
 
-  # The single E2E status check for PR/master (e2e-test/desktop-playwright),
-  # replacing the former 5 (e2e/linux, e2e/macos, e2e/windows,
-  # policy-test/macos, policy-test/windows).
+  # Per-OS E2E commit statuses (e2e/linux, e2e/macos, e2e/windows) plus
+  # e2e/macos-policy and e2e/windows-policy for the dedicated policy legs.
   #
   # Uses e2e/utils/tsio-report-status.js, not test-system-io-summary — summary
   # only reads /api/v1/orchestration/status, which report-upload never
@@ -172,9 +209,10 @@ jobs:
           sparse-checkout: |
             e2e/utils/tsio-report-status.js
             e2e/utils/cmt-channel-notify.js
+            e2e/utils/github-actions.js
           sparse-checkout-cone-mode: false
 
-      - name: Render TSIO summary + flip commit status
+      - name: Render TSIO summary + flip per-OS commit statuses
         id: summary
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
         env:
@@ -184,21 +222,28 @@ jobs:
           # time to attach all legs before we snapshot the group for the channel post.
           TSIO_POLL_ATTEMPTS: 36
           TSIO_POLL_DELAY_MS: 5000
-          COMMIT_STATUS_CONTEXT: e2e-test/desktop-playwright
           UPSTREAM_JOBS_SUCCEEDED: ${{ needs.e2e-tests.result == 'success' && needs.e2e-policy-tests.result == 'success' }}
+          PLATFORMS: ${{ needs.prepare-matrix.outputs.platforms }}
           # PR → MM_DESKTOP_E2E_WEBHOOK_URL; master → MM_E2E_MASTER_HEALTH_WEBHOOK_URL;
           # CMT/RC uses MM_E2E_RELEASE_WEBHOOK_URL (compatibility-matrix-testing.yml).
           # Routing is by compositeIdentity.name in cmt-channel-notify.js.
           MATTERMOST_E2E_WEBHOOK_URL: ${{ secrets.MM_DESKTOP_E2E_WEBHOOK_URL }}
           MATTERMOST_MASTER_HEALTH_WEBHOOK_URL: ${{ secrets.MM_E2E_MASTER_HEALTH_WEBHOOK_URL }}
         with:
           script: |
+            const {canonicalizeOs, E2E_POLICY_OS_LIST} = require('./e2e/utils/github-actions.js');
+            const platforms = JSON.parse(process.env.PLATFORMS || '[]');
+            const expectedOs = [...new Set(
+              platforms.map((p) => canonicalizeOs(p.platform || p.os, p.runner)).filter(Boolean),
+            )];
             const {reportUrl, status, stats} = await require('./e2e/utils/tsio-report-status.js')({
               core, context, github,
               compositeIdentity: JSON.parse(process.env.TSIO_COMPOSITE_IDENTITY),
               totalReportsExpected: parseInt(process.env.TSIO_TOTAL_REPORTS_EXPECTED, 10),
               upstreamJobsSucceeded: process.env.UPSTREAM_JOBS_SUCCEEDED === 'true',
-              commitStatusContext: process.env.COMMIT_STATUS_CONTEXT,
+              perOsCommitStatuses: true,
+              expectedOs,
+              expectedPolicyOs: E2E_POLICY_OS_LIST,
               failOnTestFailures: true,
             });
             core.info(`TSIO report ${reportUrl}: ${status} (${JSON.stringify(stats)})`);

e2e/utils/cmt-channel-notify.js

@@ -11,6 +11,9 @@
  *
  * Per-leg pass/fail counts come from TSIO consolidated specs grouped by
  * contributing report id → gh_job_name (group report only has upload status).
+ *
+ * Retry attempts are collapsed to one outcome per (spec × job), matching
+ * Playwright: failed+passed → flaky; all failed → failed once (not once per attempt).
  */
 
 const OS_ORDER = {linux: 0, macos: 1, windows: 2};
@@ -272,6 +275,51 @@ function formatMetaLine(compositeIdentity) {
     return parts.join(' · ');
 }
 
+/**
+ * Prefer unique per-leg counts (retry-collapsed) over TSIO group test_stats, which
+ * counts every failed attempt and double-counts retries in channel alerts.
+ *
+ * When `expectedJobNames` is provided, per-leg totals are used only if every expected
+ * job is present — otherwise fall back to `stats` so partial consolidation cannot hide
+ * aggregate failures.
+ *
+ * @param {Record<string, {passed?: number, failed?: number, skipped?: number, flaky?: number}>} perJobCounts
+ * @param {{passed?: number, failed?: number, skipped?: number, flaky?: number}} stats
+ * @param {string[]} [expectedJobNames] - uploaded report job names; omit to trust any per-leg map
+ * @returns {{passed: number, failed: number, skipped: number}}
+ */
+function resolveChannelTotals(perJobCounts, stats = {}, expectedJobNames) {
+    const counts = perJobCounts || {};
+
+    let jobs;
+    if (expectedJobNames === undefined) {
+        jobs = Object.values(counts);
+    } else if (expectedJobNames.length > 0 &&
+        expectedJobNames.every((job) => Object.prototype.hasOwnProperty.call(counts, job))) {
+        jobs = expectedJobNames.map((job) => counts[job]);
+    } else {
+        jobs = [];
+    }
+
+    if (jobs.length > 0) {
+        let passed = 0;
+        let failed = 0;
+        let skipped = 0;
+        for (const jobCounts of jobs) {
+            passed += (jobCounts.passed || 0) + (jobCounts.flaky || 0);
+            failed += jobCounts.failed || 0;
+            skipped += jobCounts.skipped || 0;
+        }
+        return {passed, failed, skipped};
+    }
+
+    return {
+        passed: (stats.passed ?? 0) + (stats.flaky ?? 0),
+        failed: stats.failed ?? 0,
+        skipped: stats.skipped ?? 0,
+    };
+}
+
 /**
  * @param {Object} params
  * @param {Object} params.compositeIdentity
@@ -292,19 +340,22 @@ function formatCmtChannelMessage({
     upstreamJobsSucceeded = true,
     hasFailures = false,
 }) {
-    const stats = detail?.test_stats || {};
-
-    // Match buildLegSummaries: fold flaky into passed so the headline matches per-leg totals.
-    const passed = (stats.passed ?? 0) + (stats.flaky ?? 0);
-    const failed = stats.failed ?? 0;
-    const skipped = stats.skipped ?? 0;
+    const reports = detail?.reports || [];
+    const legs = buildLegSummaries(perJobCounts, reports, baseUrl);
+    const expectedJobNames = [...new Set(
+        reports.map((r) => r.gh_job_name || r.display_name).filter(Boolean),
+    )];
+    const {passed, failed, skipped} = resolveChannelTotals(
+        perJobCounts,
+        detail?.test_stats,
+        expectedJobNames,
+    );
 
     // Overall pass/fail follows tests + upstream CI — not TSIO consolidation state.
     // Stuck `in_progress` / `incomplete` with 0 failures must not render as ❌ Failed.
     const overallFailed = failed > 0 || !upstreamJobsSucceeded || hasFailures;
     const tsioPending = Boolean(detail?.status && detail.status !== 'completed');
     const title = reportTitleForIdentity(compositeIdentity);
-    const legs = buildLegSummaries(perJobCounts, detail?.reports || [], baseUrl);
     const missingLegs = legs.filter((leg) => leg.status === 'missing' || leg.status === 'no-results');
 
     const lines = [
@@ -379,6 +430,54 @@ function formatCmtChannelMessage({
     return lines.join('\n').trimEnd() + '\n';
 }
 
+/**
+ * Collapse TSIO history attempts for one spec on one job into a single Playwright-style status.
+ * failed then passed (or explicit flaky) → flaky; all failed → failed once.
+ *
+ * @param {Array<{status?: string}>} entries
+ * @returns {'passed'|'failed'|'skipped'|'flaky'|null}
+ */
+function collapseSpecAttempts(entries) {
+    if (!entries || entries.length === 0) {
+        return null;
+    }
+
+    let sawPassed = false;
+    let sawFailed = false;
+    let sawSkipped = false;
+    let sawFlaky = false;
+
+    for (const entry of entries) {
+        const status = entry.status || 'failed';
+        if (status === 'passed') {
+            sawPassed = true;
+        } else if (status === 'failed') {
+            sawFailed = true;
+        } else if (status === 'skipped') {
+            sawSkipped = true;
+        } else if (status === 'flaky') {
+            sawFlaky = true;
+        } else {
+            // Unknown statuses treated as failures so they still surface in alerts.
+            sawFailed = true;
+        }
+    }
+
+    if (sawFlaky || (sawFailed && sawPassed)) {
+        return 'flaky';
+    }
+    if (sawFailed) {
+        return 'failed';
+    }
+    if (sawPassed) {
+        return 'passed';
+    }
+    if (sawSkipped) {
+        return 'skipped';
+    }
+    return null;
+}
+
 /**
  * @param {string} baseUrl
  * @param {Object} compositeIdentity
@@ -422,6 +521,8 @@ async function fetchPerJobCountsFromConsolidated(baseUrl, compositeIdentity, gro
     const attempt = Number.parseInt(compositeIdentity.gh_run_attempt || '1', 10);
 
     for (const spec of consol.specs || []) {
+        /** @type {Record<string, Array<{status?: string}>>} */
+        const entriesByJob = {};
         for (const entry of spec.history || []) {
             if (entry.commit_sha !== commitSha) {
                 continue;
@@ -433,15 +534,21 @@ async function fetchPerJobCountsFromConsolidated(baseUrl, compositeIdentity, gro
             if (!job) {
                 continue;
             }
+            if (!entriesByJob[job]) {
+                entriesByJob[job] = [];
+            }
+            entriesByJob[job].push(entry);
+        }
+
+        for (const [job, entries] of Object.entries(entriesByJob)) {
+            const status = collapseSpecAttempts(entries);
+            if (!status) {
+                continue;
+            }
             if (!counts[job]) {
                 counts[job] = {passed: 0, failed: 0, skipped: 0, flaky: 0};
             }
-            const status = entry.status || 'failed';
-            if (Object.prototype.hasOwnProperty.call(counts[job], status)) {
-                counts[job][status] += 1;
-            } else {
-                counts[job].failed += 1;
-            }
+            counts[job][status] += 1;
         }
     }
 
@@ -541,6 +648,8 @@ module.exports = {
     buildLegSummaries,
     formatLegResultText,
     reportTitleForIdentity,
+    resolveChannelTotals,
+    collapseSpecAttempts,
     formatCmtChannelMessage,
     fetchPerJobCountsFromConsolidated,
     postMattermostWebhook,