ci(e2e): bump TSIO report-upload pin for 504 retries

.github/workflows/compatibility-matrix-testing.yml

@@ -197,7 +197,9 @@ jobs:
         env:
           TSIO_COMPOSITE_IDENTITY: ${{ needs.calculate-commit-hash.outputs.TSIO_COMPOSITE_IDENTITY }}
           TSIO_TOTAL_REPORTS_EXPECTED: ${{ needs.calculate-commit-hash.outputs.TSIO_TOTAL_REPORTS_EXPECTED }}
-          TSIO_POLL_ATTEMPTS: 24
+          # Large CMT matrices upload many screenshot batches; keep headroom for
+          # consolidation so legs are not reported as missing.
+          TSIO_POLL_ATTEMPTS: 48
           TSIO_POLL_DELAY_MS: 5000
           COMMIT_STATUS_CONTEXT: e2e/compatibility-matrix-testing
           # Incoming webhook for the Desktop CMT/RC results channel (separate from

.github/workflows/e2e-functional-template.yml

@@ -292,12 +292,18 @@ jobs:
           TSIO_GH_JOB_NAME: e2e-on-${{ inputs.runs-on }}-${{ inputs.MM_SERVER_VERSION }}
           PLAYWRIGHT_EXIT_CODE: ${{ env.PLAYWRIGHT_EXIT_CODE }}
 
+      # report-upload walks every png/jpg under screenshots-dir; Electron userdata
+      # under e2e/test-results includes cache icons, so collect failure shots only.
+      - name: e2e/collect-tsio-screenshots
+        if: ${{ always() && inputs.tsio-config != '' && hashFiles('e2e/test-results/results.json') != '' }}
+        run: node e2e/utils/collect-tsio-screenshots.mjs
+
       # Reporting only — a TSIO/upload flake (e.g. screenshot 504) must not fail
       # the job when Playwright already passed. Commit status still comes from TSIO summary.
       - name: e2e/upload-report-to-tsio
         if: ${{ always() && inputs.tsio-config != '' && hashFiles('e2e/test-results/results.json') != '' }}
         continue-on-error: true
-        uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-report-upload@81017b2d568d1fc1f1fdaa7299b177fae53c63e1 # 0.10.0 / 2026-05-16
+        uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-report-upload@606431efb597481b0857066be8130e86c22d2a03 # release-0.11.0 / 2026-07-21
         with:
           composite-identity: ${{ toJSON(fromJSON(inputs.tsio-config).composite_identity) }}
           total-reports-expected: ${{ fromJSON(inputs.tsio-config).total_reports_expected }}
@@ -306,7 +312,7 @@ jobs:
           # MUST match this job's `name:` field above.
           gh-job-name: e2e-on-${{ inputs.runs-on }}-${{ inputs.MM_SERVER_VERSION }}
           json-path: e2e/test-results/results.json
-          screenshots-dir: e2e/test-results
+          screenshots-dir: e2e/test-results/tsio-screenshots
 
       - name: e2e/handle-nonzero-playwright-exit
         if: always()

.github/workflows/e2e-functional.yml

@@ -180,7 +180,9 @@ jobs:
         env:
           TSIO_COMPOSITE_IDENTITY: ${{ needs.prepare-matrix.outputs.tsio-composite-identity }}
           TSIO_TOTAL_REPORTS_EXPECTED: ${{ needs.prepare-matrix.outputs.tsio-total-reports-expected }}
-          TSIO_POLL_ATTEMPTS: 12
+          # Uploads (esp. screenshots) can finish minutes after Playwright; give TSIO
+          # time to attach all legs before we snapshot the group for the channel post.
+          TSIO_POLL_ATTEMPTS: 36
           TSIO_POLL_DELAY_MS: 5000
           COMMIT_STATUS_CONTEXT: e2e-test/desktop-playwright
           UPSTREAM_JOBS_SUCCEEDED: ${{ needs.e2e-tests.result == 'success' && needs.e2e-policy-tests.result == 'success' }}
@@ -421,20 +423,26 @@ jobs:
           TSIO_GH_JOB_NAME: policy-tests-${{ matrix.platform }}
           PLAYWRIGHT_EXIT_CODE: ${{ env.PLAYWRIGHT_EXIT_CODE }}
 
+      # report-upload walks every png/jpg under screenshots-dir; Electron userdata
+      # under e2e/test-results includes cache icons, so collect failure shots only.
+      - name: e2e/collect-tsio-screenshots
+        if: ${{ always() && needs.prepare-matrix.outputs.tsio-composite-identity != '' && hashFiles('e2e/test-results/results.json') != '' }}
+        run: node e2e/utils/collect-tsio-screenshots.mjs
+
       # Reporting only — a TSIO/upload flake (e.g. screenshot 504) must not fail
       # the job when Playwright already passed. Commit status still comes from TSIO summary.
       - name: e2e/upload-report-to-tsio
         if: ${{ always() && needs.prepare-matrix.outputs.tsio-composite-identity != '' && hashFiles('e2e/test-results/results.json') != '' }}
         continue-on-error: true
-        uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-report-upload@81017b2d568d1fc1f1fdaa7299b177fae53c63e1 # 0.10.0 / 2026-05-16
+        uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-report-upload@606431efb597481b0857066be8130e86c22d2a03 # release-0.11.0 / 2026-07-21
         with:
           composite-identity: ${{ needs.prepare-matrix.outputs.tsio-composite-identity }}
           total-reports-expected: ${{ needs.prepare-matrix.outputs.tsio-total-reports-expected }}
           framework: playwright
           github-token: ${{ secrets.GITHUB_TOKEN }}
           gh-job-name: policy-tests-${{ matrix.platform }}
           json-path: e2e/test-results/results.json
-          screenshots-dir: e2e/test-results
+          screenshots-dir: e2e/test-results/tsio-screenshots
 
       - name: e2e/handle-nonzero-playwright-exit
         if: always()

e2e/specs/menu_bar/window_menu.test.ts

@@ -13,7 +13,7 @@ import {closeElectronAppFast, registerElectronMainProcess, waitForWindow} from '
 import {loginToMattermost} from '../../helpers/login';
 import {waitForMattermostShellReady} from '../../helpers/mattermostShell';
 import {prepareMattermostServerView} from '../../helpers/prepareServerView';
-import {buildServerMap} from '../../helpers/serverMap';
+import {buildServerMap, type ServerMap} from '../../helpers/serverMap';
 
 const windowMenuConfig = {
     ...demoMattermostConfig,
@@ -232,63 +232,74 @@ async function createExtraTabs() {
     await mainWindow.waitForSelector('.TabBar li.serverTabItem:nth-child(3)', {timeout: 15_000});
 
     const serverName = windowMenuConfig.servers[0].name;
-    let map = await buildServerMap(electronApp);
-    const deadline = Date.now() + 30_000;
-    while ((map[serverName]?.length ?? 0) < 3 && Date.now() < deadline) {
-        await new Promise((resolve) => setTimeout(resolve, 200));
-        map = await buildServerMap(electronApp);
-    }
-    expect(map[serverName]?.length, 'Three Mattermost tabs should be registered').toBeGreaterThanOrEqual(3);
-    return map;
+    await expect.poll(async () => {
+        const map = await buildServerMap(electronApp);
+        return map[serverName]?.length ?? 0;
+    }, {
+        timeout: 30_000,
+        message: 'Three Mattermost tabs should be registered',
+    }).toBeGreaterThanOrEqual(3);
+
+    return buildServerMap(electronApp);
+}
+
+function getServerWebContentsIds(map: ServerMap, serverName: string): number[] {
+    return (map[serverName] ?? []).map((entry) => entry.webContentsId);
 }
 
 async function switchToTabAndOpenChannel(
     serverName: string,
     tabIndex: number,
     channelItem: string,
+    expectedIds: number[],
 ) {
-    await expect.poll(async () => {
-        const map = await buildServerMap(electronApp);
-        return map[serverName]?.length ?? 0;
-    }, {timeout: 30_000}).toBeGreaterThanOrEqual(tabIndex);
-
-    const tab = await mainWindow.waitForSelector(
-        `.TabBar li.serverTabItem:nth-child(${tabIndex})`,
-        {timeout: 15_000},
-    );
-    await tab.click();
-
-    // Focus + re-resolve after click. Do NOT loginToMattermost on secondary tabs:
-    // a TAB_LOGIN_CHANGED(false) during reload/login tears down non-primary tabs
-    // (TabManager.handleServerLoggedInChanged), leaving stale webContentsIds.
-    await focusMainWindow();
     let localServerMap = await buildServerMap(electronApp);
     await expect.poll(async () => {
         localServerMap = await buildServerMap(electronApp);
+        const actualIds = getServerWebContentsIds(localServerMap, serverName);
+        if (!expectedIds.every((id) => actualIds.includes(id))) {
+            return null;
+        }
         return localServerMap[serverName]?.[tabIndex - 1]?.webContentsId ?? null;
     }, {
         timeout: 30_000,
-        message: `Mattermost tab ${tabIndex} must remain registered after switch`,
+        message: `Mattermost tab ${tabIndex} must be registered before switch`,
     }).not.toBeNull();
 
-    const view = localServerMap[serverName][tabIndex - 1].win;
-    await prepareMattermostServerView(electronApp, view.webContentsId);
-    await waitForMattermostShellReady(view, {channelItem});
-    await view.click(channelItem);
+    const expectedId = expectedIds[tabIndex - 1];
+    const entry = localServerMap[serverName].find((candidate) => candidate.webContentsId === expectedId) ??
+        localServerMap[serverName][tabIndex - 1];
+    expect(entry?.webContentsId, `Mattermost tab ${tabIndex} webContentsId must match createExtraTabs map`).toBe(expectedId);
+
+    // Switch via TabManager (same path the Window menu uses). Avoid DOM tab click +
+    // focusMainWindow here: that widened the race where a loading secondary tab's
+    // transient onLogout destroyed siblings before the view was re-resolved.
+    await prepareMattermostServerView(electronApp, entry.webContentsId);
+    await waitForMattermostShellReady(entry.win, {channelItem});
+    await entry.win.click(channelItem);
+
+    localServerMap = await buildServerMap(electronApp);
+    const actualIds = getServerWebContentsIds(localServerMap, serverName);
+    expect(actualIds.length, `Mattermost tabs must remain registered after opening channel on tab ${tabIndex}`).
+        toBeGreaterThanOrEqual(expectedIds.length);
+    expect(actualIds, `Mattermost tab webContentsIds must remain stable after opening channel on tab ${tabIndex}`).
+        toEqual(expect.arrayContaining(expectedIds));
 
     return localServerMap;
 }
 
-async function navigateToSecondAndThirdTabs(serverName: string) {
+async function navigateToSecondAndThirdTabs(serverName: string, expectedIds: number[]) {
     await switchToTabAndOpenChannel(
         serverName,
         2,
         '#sidebarItem_off-topic',
+        expectedIds,
     );
     return switchToTabAndOpenChannel(
         serverName,
         3,
         '#sidebarItem_town-square',
+        expectedIds,
     );
 }
 
@@ -377,8 +388,9 @@ test.describe('Menu/window_menu', () => {
 
     test.describe('MM-T4385 select tab from menu', () => {
         test('MM-T4385_1 should show the second tab', {tag: ['@P2', '@all']}, async () => {
-            await createExtraTabs();
-            await navigateToSecondAndThirdTabs(windowMenuConfig.servers[0].name);
+            const serverName = windowMenuConfig.servers[0].name;
+            const createdMap = await createExtraTabs();
+            await navigateToSecondAndThirdTabs(serverName, getServerWebContentsIds(createdMap, serverName));
 
             // Tab title updates asynchronously after channel navigation — poll for it.
             await expect(mainWindow.locator('.active')).toContainText('Town Square', {timeout: 10_000});
@@ -388,17 +400,19 @@ test.describe('Menu/window_menu', () => {
         });
 
         test('MM-T4385_2 should show the third tab', {tag: ['@P2', '@all']}, async () => {
-            await createExtraTabs();
-            await navigateToSecondAndThirdTabs(windowMenuConfig.servers[0].name);
+            const serverName = windowMenuConfig.servers[0].name;
+            const createdMap = await createExtraTabs();
+            await navigateToSecondAndThirdTabs(serverName, getServerWebContentsIds(createdMap, serverName));
 
             await clickWindowMenuItem(electronApp, {accelerator: 'CmdOrCtrl+2'});
             await clickWindowMenuItem(electronApp, {accelerator: 'CmdOrCtrl+3'});
             await expect.poll(() => getActiveTabTitle(electronApp), {timeout: 15_000}).toContain('Town Square');
         });
 
         test('MM-T4385_3 should show the first tab', {tag: ['@P2', '@all']}, async () => {
-            await createExtraTabs();
-            await navigateToSecondAndThirdTabs(windowMenuConfig.servers[0].name);
+            const serverName = windowMenuConfig.servers[0].name;
+            const createdMap = await createExtraTabs();
+            await navigateToSecondAndThirdTabs(serverName, getServerWebContentsIds(createdMap, serverName));
 
             await clickWindowMenuItem(electronApp, {accelerator: 'CmdOrCtrl+2'});
             await clickWindowMenuItem(electronApp, {accelerator: 'CmdOrCtrl+1'});
@@ -411,7 +425,17 @@ test.describe('Menu/window_menu', () => {
         await mainWindow.waitForSelector('.TabBar li.serverTabItem:nth-child(2)', {timeout: 15_000});
 
         const serverName = windowMenuConfig.servers[0].name;
-        await switchToTabAndOpenChannel(serverName, 2, '#sidebarItem_off-topic');
+        let createdMap = await buildServerMap(electronApp);
+        await expect.poll(async () => {
+            createdMap = await buildServerMap(electronApp);
+            return createdMap[serverName]?.length ?? 0;
+        }, {timeout: 30_000}).toBeGreaterThanOrEqual(2);
+        await switchToTabAndOpenChannel(
+            serverName,
+            2,
+            '#sidebarItem_off-topic',
+            getServerWebContentsIds(createdMap, serverName),
+        );
 
         await expect.poll(() => getActiveTabTitle(electronApp), {timeout: 15_000}).toContain('Off-Topic');
 

e2e/utils/cmt-channel-notify.js

@@ -292,12 +292,14 @@ function formatCmtChannelMessage({
     const passed = (stats.passed ?? 0) + (stats.flaky ?? 0);
     const failed = stats.failed ?? 0;
     const skipped = stats.skipped ?? 0;
-    const overallFailed = failed > 0 ||
-        detail?.status !== 'completed' ||
-        !upstreamJobsSucceeded ||
-        hasFailures;
+
+    // Overall pass/fail follows tests + upstream CI — not TSIO consolidation state.
+    // Stuck `in_progress` / `incomplete` with 0 failures must not render as ❌ Failed.
+    const overallFailed = failed > 0 || !upstreamJobsSucceeded || hasFailures;
+    const tsioPending = Boolean(detail?.status && detail.status !== 'completed');
     const title = reportTitleForIdentity(compositeIdentity);
     const legs = buildLegSummaries(perJobCounts, detail?.reports || [], baseUrl);
+    const missingLegs = legs.filter((leg) => leg.status === 'missing' || leg.status === 'no-results');
 
     const lines = [
         `## ${overallFailed ? '❌' : '✅'} ${title}`,
@@ -348,9 +350,21 @@ function formatCmtChannelMessage({
     } else if (hasFailures && failed === 0) {
         lines.push('_TSIO reported failed shard(s) not reflected in the test totals; check the full report._', '');
     }
-    if (detail?.status && detail.status !== 'completed' && upstreamJobsSucceeded) {
+    if (tsioPending && upstreamJobsSucceeded && !overallFailed) {
+        lines.push(
+            `_TSIO report status: \`${detail.status}\` (consolidation still catching up; not treated as a test failure)._`,
+            '',
+        );
+    } else if (tsioPending && overallFailed) {
         lines.push(`_TSIO report status: \`${detail.status}\`._`, '');
     }
+    if (missingLegs.length > 0) {
+        const labels = missingLegs.map((leg) => {
+            const {platform, suite} = formatLegLabels(leg);
+            return `${platform} / ${suite}`;
+        }).join(', ');
+        lines.push(`_Missing or empty leg report(s): ${labels}._`, '');
+    }
 
     if (reportUrl) {
         lines.push(`➡️ **Full report:** ${reportUrl}`);

e2e/utils/cmt-channel-notify.test.js

@@ -184,7 +184,45 @@ describe('cmt-channel-notify', () => {
             assert.doesNotMatch(text, /failing test/);
         });
 
-        it('explains non-completed TSIO status when overall failed', () => {
+        it('keeps overall passed when TSIO is still consolidating with 0 failures', () => {
+            const text = formatCmtChannelMessage({
+                compositeIdentity: {
+                    repository: 'mattermost/desktop',
+                    branch: 'pr-3916',
+                    commit_sha: 'b41298f0abc1234567890abcdef1234567890abcd',
+                    name: 'desktop-pr',
+                    gh_pr_number: '3916',
+                },
+                detail: {
+                    status: 'in_progress',
+                    test_stats: {passed: 674, failed: 0, skipped: 51, total: 725},
+                    reports: [
+                        {id: 'rid-linux', gh_job_name: 'e2e-on-ubuntu-latest-11.10.0-rc1', status: 'complete'},
+                        {id: 'rid-mac', gh_job_name: 'e2e-on-macos-14-11.10.0-rc1', status: 'complete'},
+                        {id: 'rid-win', gh_job_name: 'e2e-on-windows-2022-11.10.0-rc1', status: 'complete'},
+                        {id: 'rid-win-policy', gh_job_name: 'policy-tests-windows', status: 'complete'},
+                    ],
+                },
+                reportUrl: 'https://test-io.test.mattermost.com/reports/desktop/pr-3916/b41298f/desktop-pr',
+                baseUrl: 'https://test-io.test.mattermost.com',
+                perJobCounts: {
+                    'e2e-on-ubuntu-latest-11.10.0-rc1': {passed: 216, failed: 0, skipped: 10, flaky: 0},
+                    'e2e-on-macos-14-11.10.0-rc1': {passed: 220, failed: 0, skipped: 10, flaky: 0},
+                    'e2e-on-windows-2022-11.10.0-rc1': {passed: 237, failed: 0, skipped: 10, flaky: 0},
+                    'policy-tests-windows': {passed: 9, failed: 0, skipped: 0, flaky: 0},
+                    'policy-tests-macos': {passed: 0, failed: 0, skipped: 0, flaky: 0},
+                },
+                upstreamJobsSucceeded: true,
+            });
+            assert.match(text, /^## ✅ Desktop PR E2E\n/);
+            assert.match(text, /\| ✅ Passed \| \*\*674\*\* \| \*\*0\*\* \| \*\*51\*\* \|/);
+            assert.match(text, /TSIO report status: `in_progress` \(consolidation still catching up; not treated as a test failure\)/);
+            assert.match(text, /Missing or empty leg report\(s\): 🍎 macOS \/ Policy/);
+            assert.match(text, /\| 🍎 macOS \| Policy \| ⚠️ missing \|/);
+            assert.doesNotMatch(text, /\| ❌ Failed \|/);
+        });
+
+        it('still marks overall failed for incomplete TSIO when tests failed', () => {
             const text = formatCmtChannelMessage({
                 compositeIdentity: {
                     branch: 'master',
@@ -193,7 +231,7 @@ describe('cmt-channel-notify', () => {
                 },
                 detail: {
                     status: 'incomplete',
-                    test_stats: {passed: 50, failed: 0, skipped: 0, total: 50},
+                    test_stats: {passed: 49, failed: 1, skipped: 0, total: 50},
                     reports: [],
                 },
                 reportUrl: 'https://test-io.test.mattermost.com/reports/desktop/master/a1b2c3d/desktop-master',