diff --git a/.github/workflows/compatibility-matrix-testing.yml b/.github/workflows/compatibility-matrix-testing.yml index f4bfb76be86..9250190fafd 100644 --- a/.github/workflows/compatibility-matrix-testing.yml +++ b/.github/workflows/compatibility-matrix-testing.yml @@ -107,6 +107,33 @@ jobs: description: "Compatibility Matrix Testing for ${{ inputs.DESKTOP_VERSION }} version" status: pending + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + continue-on-error: true + with: + ref: ${{ inputs.DESKTOP_VERSION }} + persist-credentials: false + sparse-checkout: | + e2e/utils/github-actions.js + sparse-checkout-cone-mode: false + + - name: Post pending e2e/linux|macos|windows + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + continue-on-error: true + env: + DESKTOP_SHA: ${{ needs.calculate-commit-hash.outputs.DESKTOP_SHA }} + CMT_MATRIX: ${{ needs.calculate-commit-hash.outputs.CMT_MATRIX }} + with: + script: | + const {updateInitialOsStatuses} = require('./e2e/utils/github-actions.js'); + const matrix = JSON.parse(process.env.CMT_MATRIX || '{}'); + const platforms = (matrix.environment || []).map((env) => ({platform: env.os})); + await updateInitialOsStatuses({ + github, + context, + sha: process.env.DESKTOP_SHA, + platforms, + }); + # Input follows the below schema (Matterwick ≥ v0.4.16). # { # "environment": [ @@ -177,9 +204,11 @@ jobs: persist-credentials: false sparse-checkout: | e2e/utils/tsio-report-status.js + e2e/utils/cmt-channel-notify.js + e2e/utils/github-actions.js sparse-checkout-cone-mode: false - - name: Render TSIO summary + flip commit status + - name: Render TSIO summary + flip commit statuses id: summary uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: @@ -192,13 +221,23 @@ jobs: # matching failed test (hung worker teardown, crashed runner, npm ci # failure) would otherwise still read as "100% passed". UPSTREAM_JOBS_SUCCEEDED: ${{ needs.e2e.result == 'success' }} + CMT_MATRIX: ${{ needs.calculate-commit-hash.outputs.CMT_MATRIX }} with: script: | + const {canonicalizeOs} = require('./e2e/utils/github-actions.js'); + const matrix = JSON.parse(process.env.CMT_MATRIX || '{}'); + const expectedOs = [...new Set( + (matrix.environment || []). + map((env) => canonicalizeOs(env.os, env.runner)). + filter(Boolean), + )]; const {reportUrl, status, stats} = await require('./e2e/utils/tsio-report-status.js')({ core, context, github, compositeIdentity: JSON.parse(process.env.TSIO_COMPOSITE_IDENTITY), totalReportsExpected: parseInt(process.env.TSIO_TOTAL_REPORTS_EXPECTED, 10), commitStatusContext: process.env.COMMIT_STATUS_CONTEXT, + perOsCommitStatuses: true, + expectedOs, upstreamJobsSucceeded: process.env.UPSTREAM_JOBS_SUCCEEDED === 'true', failOnTestFailures: true, }); diff --git a/.github/workflows/e2e-functional.yml b/.github/workflows/e2e-functional.yml index 41f71d1ad31..94d03489820 100644 --- a/.github/workflows/e2e-functional.yml +++ b/.github/workflows/e2e-functional.yml @@ -58,7 +58,30 @@ jobs: run: | # Matterwick still dispatches macos-latest; pin explicitly so the job # does not drift when GitHub retargets that label to macOS 26. - platforms=$(echo "${INSTANCE_DETAILS}" | jq -c 'map(if .runner == "macos-latest" then .runner = "macos-26" else . end)') + # Normalize platform to canonical linux|macos|windows (derive from runner + # when needed) so job names and e2e/ statuses stay consistent. + platforms=$(echo "${INSTANCE_DETAILS}" | jq -c ' + map( + (if .runner == "macos-latest" then .runner = "macos-26" else . end) | + . as $row | + ($row.platform // $row.os // "") as $raw | + ( + if $raw == "linux" or $raw == "macos" or $raw == "windows" then $raw + elif (($row.runner // "") | test("^(ubuntu|linux)")) then "linux" + elif (($row.runner // "") | test("^(macos|darwin)")) then "macos" + elif (($row.runner // "") | test("^windows")) then "windows" + else null + end + ) as $canon | + if $canon == null then empty + else ($row | .platform = $canon | del(.os)) + end + ) + ') + if [ "$(echo "${platforms}" | jq 'length')" -eq 0 ]; then + echo "No supported platforms in instance_details (need linux|macos|windows)" >&2 + exit 1 + fi echo "platforms=${platforms}" >> "$GITHUB_OUTPUT" # Resolve the SHA under test from version_name (branch/tag/SHA), not github.sha @@ -98,7 +121,7 @@ jobs: echo "composite-identity-json=${COMPOSITE_IDENTITY}" >> "$GITHUB_OUTPUT" update-initial-status: - name: Set pending TSIO status + name: Set pending E2E OS statuses runs-on: ubuntu-24.04 needs: - prepare-matrix @@ -108,16 +131,31 @@ jobs: steps: # Best-effort: a transient status-API hiccup here must never block the # actual test matrix from running. - - uses: mattermost/actions/delivery/update-commit-status@218fd96a63451259dc100e1292a5e44af92fe15d + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + continue-on-error: true + with: + ref: ${{ inputs.version_name }} + persist-credentials: false + sparse-checkout: | + e2e/utils/github-actions.js + sparse-checkout-cone-mode: false + + - name: Post pending e2e/linux|macos|windows (+ policy) + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 continue-on-error: true env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PLATFORMS: ${{ needs.prepare-matrix.outputs.platforms }} + DESKTOP_SHA: ${{ needs.prepare-matrix.outputs.desktop-sha }} with: - repository_full_name: ${{ github.repository }} - commit_sha: ${{ needs.prepare-matrix.outputs.desktop-sha }} - context: e2e-test/desktop-playwright - description: "Running Electron Playwright E2E tests..." - status: pending + script: | + const {updateInitialOsStatuses} = require('./e2e/utils/github-actions.js'); + await updateInitialOsStatuses({ + github, + context, + sha: process.env.DESKTOP_SHA, + platforms: JSON.parse(process.env.PLATFORMS || '[]'), + includePolicy: true, + }); e2e-tests: needs: @@ -144,9 +182,8 @@ jobs: tsio-config: ${{ format('{{"composite_identity":{0},"total_reports_expected":"{1}"}}', needs.prepare-matrix.outputs.tsio-composite-identity, needs.prepare-matrix.outputs.tsio-total-reports-expected) }} secrets: inherit - # The single E2E status check for PR/master (e2e-test/desktop-playwright), - # replacing the former 5 (e2e/linux, e2e/macos, e2e/windows, - # policy-test/macos, policy-test/windows). + # Per-OS E2E commit statuses (e2e/linux, e2e/macos, e2e/windows) plus + # e2e/macos-policy and e2e/windows-policy for the dedicated policy legs. # # Uses e2e/utils/tsio-report-status.js, not test-system-io-summary — summary # only reads /api/v1/orchestration/status, which report-upload never @@ -171,9 +208,11 @@ jobs: persist-credentials: false sparse-checkout: | e2e/utils/tsio-report-status.js + e2e/utils/cmt-channel-notify.js + e2e/utils/github-actions.js sparse-checkout-cone-mode: false - - name: Render TSIO summary + flip commit status + - name: Render TSIO summary + flip per-OS commit statuses id: summary uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: @@ -181,16 +220,23 @@ jobs: TSIO_TOTAL_REPORTS_EXPECTED: ${{ needs.prepare-matrix.outputs.tsio-total-reports-expected }} TSIO_POLL_ATTEMPTS: 12 TSIO_POLL_DELAY_MS: 5000 - COMMIT_STATUS_CONTEXT: e2e-test/desktop-playwright UPSTREAM_JOBS_SUCCEEDED: ${{ needs.e2e-tests.result == 'success' && needs.e2e-policy-tests.result == 'success' }} + PLATFORMS: ${{ needs.prepare-matrix.outputs.platforms }} with: script: | + const {canonicalizeOs, E2E_POLICY_OS_LIST} = require('./e2e/utils/github-actions.js'); + const platforms = JSON.parse(process.env.PLATFORMS || '[]'); + const expectedOs = [...new Set( + platforms.map((p) => canonicalizeOs(p.platform || p.os, p.runner)).filter(Boolean), + )]; const {reportUrl, status, stats} = await require('./e2e/utils/tsio-report-status.js')({ core, context, github, compositeIdentity: JSON.parse(process.env.TSIO_COMPOSITE_IDENTITY), totalReportsExpected: parseInt(process.env.TSIO_TOTAL_REPORTS_EXPECTED, 10), upstreamJobsSucceeded: process.env.UPSTREAM_JOBS_SUCCEEDED === 'true', - commitStatusContext: process.env.COMMIT_STATUS_CONTEXT, + perOsCommitStatuses: true, + expectedOs, + expectedPolicyOs: E2E_POLICY_OS_LIST, failOnTestFailures: true, }); core.info(`TSIO report ${reportUrl}: ${status} (${JSON.stringify(stats)})`); diff --git a/e2e/specs/mattermost/media_preview.test.ts b/e2e/specs/mattermost/media_preview.test.ts index 6ed370f753b..1bb921349e7 100644 --- a/e2e/specs/mattermost/media_preview.test.ts +++ b/e2e/specs/mattermost/media_preview.test.ts @@ -9,8 +9,10 @@ import {prepareMattermostServerView} from '../../helpers/prepareServerView'; import {getFilePublicLink, isPublicLinkEnabled} from '../../helpers/server_api/publicLinks'; import type {ServerView} from '../../helpers/serverView'; -// 64x64 PNG — above Mattermost's 48px inline-image minimum so thumbnails render visibly. -const PREVIEW_PNG_BASE64 = 'iVBORw0KGgoAAAANSUhEUgAAAEAAAABACAIAAAAlC+aJAAAAf0lEQVR4nNXOQREAIAzAsFJJ8y8FMYjgsWsU5NwZyiRO4iRO4iRO4iRO4iRO4iRO4iRO4iRO4iRO4iRO4iRO4iRO4iRO4iRO4iRO4iRO4iRO4iRO4iRO4iRO4iRO4iRO4iRO4iRO4iRO4iRO4iRO4iRO4iRO4iRO4iRO4iRO4twO/HqSogHAzFmDswAAAABJRU5ErkJggg=='; +// Valid 128x128 PNG. The previous fixture was rejected by the server decoder +// ("png: invalid format: too much pixel data"), so no preview was generated and +// SizeAwareImage (MM-69174 / 11.10+) ignored clicks until load forever. +const PREVIEW_PNG_BASE64 = 'iVBORw0KGgoAAAANSUhEUgAAAIAAAACACAIAAABMXPacAAACx0lEQVR4nO3dsVEbQRhH8ZWHOuyAHogcEVAClTgmcGpX4hIIlNgRPRC4EjnYmRuNYARrab/3P/R+kQMh43333Z1kFm12u10b8eXnn6HH//321ec/4tPQo3V2BoAZAGYAmAFgBoAZAGYAmAFgBoAZAGYAmAFgBoAZAHaV9v74pT2/EwAzAMwAMAPADAAzAMwAMAPADAAzAMwAMAPADAAzAMwAsM3q9gdsf30//oDrp8dTnv/A7P8/uBp6NOXNRd/3fHO3/Pl4jATRAYbW/VVLjNgSoQFOX/oDvURghrgAZ1/6fc83d9vWbu8f5v0Vo7LugvZP3/NMbTwqZQJqln7RGySMQsQEFK/+ImEU+ADU6nd4AzgAu/od24AMkLD6HdgAC5Cz+h3VgAmQtvod0gAIkLn6XX0D/i7owlUHSD78u+Ih2Hz+8XvoC055fxy/6X6/5W079wd8cHUBVnT4t8JTpRMAKwqwrsO/qxkCJwBmAFhFgDWef7qCs5ATADMAzACw6QHWewHoZn//TgDMADADwAwAm74/YO0X4Tb4I71+fsDKGABmAJgBYAaATQ8QuCtoyOzv3wmAGQBmAFhFgPVeBgo2kTkBMAPAigKs8SxUs4nVCYDVBVjXEJTt4S79/IDt0Feiln/mh9ofkPCrAd6jclirrwH5DYpPlV6EYUCA5CGov1NgJiCzAXKfhp2C0hpQd8nkNSCnAfgaBb4IJzRgXyHyd0FsA/z1OR+gcQ3w1W85v7SvNyj7OcaEpe8iJmBRMwo5q9/SArTWrp8e5y3Q7f1D1Oq3nFPQgb5MZ9wlmnC79arQAN1ytP53idh1X/j5AW/w8wMOHRzUowuUJu4ifGkMADMAzAAwA8AMADMAzAAwA8AMADMAzAAwA8AMACvdH+Dzv+QEwAwAMwDMADADwAwAMwDMADADwAwAMwDMADADwAwAMwDsH5fw4xGXqkx/AAAAAElFTkSuQmCC'; const PREVIEW_MODAL_SELECTOR = [ '.file-preview-modal', @@ -19,17 +21,88 @@ const PREVIEW_MODAL_SELECTOR = [ '#viewImageModalLabel', ].join(', '); -const POSTED_IMAGE_SELECTOR = [ - '.post-image .small-image__container', - '.post-image .image-loaded-container', - '.post-image__image', - '.post-image img', - '.file-viewer-touch', - '.file-attachment', - '.post--attachment img', - 'img[src*="/api/v4/files/"]', +const LOADED_IMAGE_SELECTOR = [ + '.post-image img:not(.image-loading__placeholder)', + '.post--attachment img:not(.image-loading__placeholder)', + 'img[src*="/api/v4/files/"]:not(.image-loading__placeholder)', ].join(', '); +const PREVIEW_FILE_NAME = 'e2e-preview.png'; + +// Shared helpers injected into renderer scripts (same pattern as DOM_UTILS in serverView.ts). +const PREVIEW_IMAGE_UTILS = ` +const LOADED_IMAGE_SELECTOR = ${JSON.stringify(LOADED_IMAGE_SELECTOR)}; +const PREVIEW_FILE_NAME = ${JSON.stringify(PREVIEW_FILE_NAME)}; +const isPreviewControlVisible = (el) => el instanceof HTMLElement && window.getComputedStyle(el).display !== 'none'; +const isLoadedPreviewImage = (el) => el instanceof HTMLImageElement && + !el.classList.contains('image-loading__placeholder') && + el.complete && + el.naturalWidth > 0 && + isPreviewControlVisible(el); +const postHasPreviewFixture = (post) => { + if (post.querySelector('[aria-label*="' + PREVIEW_FILE_NAME + '" i]')) { + return true; + } + // Filename can also appear in attachment headers before the image aria-label mounts. + const attachment = post.querySelector('.post-image, .post--attachment, .file-attachment, .file-preview__button'); + return Boolean(attachment && (attachment.textContent || '').toLowerCase().includes(PREVIEW_FILE_NAME)); +}; +const findPreviewFixturePost = () => { + const posts = Array.from(document.querySelectorAll('.post')); + for (let index = posts.length - 1; index >= 0; index--) { + const post = posts[index]; + if (postHasPreviewFixture(post)) { + return post; + } + } + return null; +}; +const findVisibleLoadedPreviewButton = (root) => { + for (const button of root.querySelectorAll('.file-preview__button')) { + if (!isPreviewControlVisible(button)) { + continue; + } + const loadedImg = button.querySelector('img:not(.image-loading__placeholder)'); + if (loadedImg instanceof HTMLImageElement && loadedImg.complete && loadedImg.naturalWidth > 0) { + return button; + } + } + return null; +}; +`; + +/** + * Mattermost 11.10+ (MM-69174) SizeAwareImage ignores clicks until the real image has + * loaded, and keeps a visible placeholder button while the clickable control is + * display:none. Wait for a visible, loaded non-placeholder control before opening. + */ +async function waitForLoadedImagePreviewControl(serverWin: ServerView): Promise { + await expect.poll(async () => serverWin.runInRenderer(` + ${PREVIEW_IMAGE_UTILS} + const post = findPreviewFixturePost(); + if (!post) { + return false; + } + + const previewButton = findVisibleLoadedPreviewButton(post); + if (previewButton) { + previewButton.scrollIntoView({block: 'center'}); + return true; + } + + // Legacy servers without .file-preview__button + const legacyImg = post.querySelector(LOADED_IMAGE_SELECTOR); + if (isLoadedPreviewImage(legacyImg)) { + legacyImg.scrollIntoView({block: 'center'}); + return true; + } + return false; + `, true), { + timeout: 60_000, + message: 'Uploaded e2e-preview.png must finish loading into a visible file-preview control before it can be opened', + }).toBe(true); +} + async function submitComposerPost(serverWin: ServerView): Promise { const sent = await serverWin.runInRenderer(` const sendButton = document.querySelector( @@ -49,24 +122,20 @@ async function submitComposerPost(serverWin: ServerView): Promise { async function waitForPostedAttachment(serverWin: ServerView): Promise { await expect.poll(async () => serverWin.runInRenderer(` - const attachmentSelector = ${JSON.stringify(POSTED_IMAGE_SELECTOR)}; + ${PREVIEW_IMAGE_UTILS} const composer = document.querySelector('#post-create, .AdvancedTextEditor, .post-create, [data-testid="post-create"]'); const draftAttachment = composer?.querySelector('.file-preview, .file-preview__container, .attachment-preview'); if (draftAttachment) { return false; } - const posts = Array.from(document.querySelectorAll('.post')); - for (let index = posts.length - 1; index >= 0; index--) { - const post = posts[index]; - if (post.querySelector(attachmentSelector) || - post.querySelector('[aria-label*="e2e-preview.png" i], [aria-label*="file thumbnail" i]')) { - post.scrollIntoView({block: 'center'}); - return true; - } + const post = findPreviewFixturePost(); + if (!post) { + return false; } - return false; - `, true), {timeout: 60_000, message: 'Uploaded image must appear in the channel post list'}).toBe(true); + post.scrollIntoView({block: 'center'}); + return true; + `, true), {timeout: 60_000, message: 'Uploaded e2e-preview.png must appear in the channel post list'}).toBe(true); } async function uploadAndPostPng(serverWin: ServerView): Promise { @@ -77,7 +146,7 @@ async function uploadAndPostPng(serverWin: ServerView): Promise { for (let i = 0; i < binary.length; i++) { bytes[i] = binary.charCodeAt(i); } - const file = new File([bytes], 'e2e-preview.png', {type: 'image/png'}); + const file = new File([bytes], ${JSON.stringify(PREVIEW_FILE_NAME)}, {type: 'image/png'}); const input = document.querySelector('#fileUploadInput, input[type="file"]'); if (!(input instanceof HTMLInputElement)) { @@ -109,6 +178,7 @@ async function uploadAndPostPng(serverWin: ServerView): Promise { await recoverInteractiveChannel(serverWin, {channelItem: '#sidebarItem_town-square'}); await waitForPostedAttachment(serverWin); + await waitForLoadedImagePreviewControl(serverWin); } async function isImagePreviewOpen(serverWin: ServerView): Promise { @@ -126,45 +196,46 @@ async function isImagePreviewOpen(serverWin: ServerView): Promise { async function openImagePreview(serverWin: ServerView): Promise { return serverWin.runInRenderer(` - const attachmentSelector = ${JSON.stringify(POSTED_IMAGE_SELECTOR)}; - const posts = Array.from(document.querySelectorAll('.post')); - let root = null; - for (let index = posts.length - 1; index >= 0; index--) { - const post = posts[index]; - if (post.querySelector(attachmentSelector) || - post.querySelector('[aria-label*="e2e-preview.png" i], [aria-label*="file thumbnail" i]')) { - root = post; - break; - } - } + ${PREVIEW_IMAGE_UTILS} + const root = findPreviewFixturePost(); if (!root) { return false; } + // Prefer the visible SizeAwareImage control (11.10+/MM-69174); clicks on the + // placeholder button are intentionally ignored until the real image loads. + const previewButton = findVisibleLoadedPreviewButton(root); + if (previewButton) { + previewButton.scrollIntoView({block: 'center', inline: 'center'}); + previewButton.click(); + return true; + } + const clickTargets = [ - root.querySelector('[aria-label*="e2e-preview.png" i]'), - root.querySelector('[aria-label*="file thumbnail" i]'), - root.querySelector('.post-image .small-image__container'), + ...Array.from(root.querySelectorAll('[aria-label*="' + PREVIEW_FILE_NAME + '" i]')), + ...Array.from(root.querySelectorAll(LOADED_IMAGE_SELECTOR)), root.querySelector('.post-image .image-loaded-container'), + root.querySelector('.post-image .small-image__container'), root.querySelector('.post-image__image'), - root.querySelector('.post-image img'), root.querySelector('.file-viewer-touch'), - root.querySelector('.file-attachment'), - root.querySelector('.post--attachment img'), - root.querySelector('img[src*="/api/v4/files/"]'), - root.querySelector('.post-image'), - root.querySelector('.post--attachment'), - ].filter(Boolean); + ].filter((target) => { + if (!target) { + return false; + } + if (target instanceof HTMLImageElement) { + return isLoadedPreviewImage(target); + } + return isPreviewControlVisible(target) && + Boolean(target.querySelector?.('img:not(.image-loading__placeholder)')); + }); const target = clickTargets[0]; - if (!target) { + if (!(target instanceof HTMLElement)) { return false; } target.scrollIntoView({block: 'center', inline: 'center'}); - if (target instanceof HTMLElement) { - target.click(); - } + target.click(); return true; `, true); } @@ -185,8 +256,8 @@ async function getPreviewFileId(serverWin: ServerView): Promise { const sources = [ document.querySelector('[data-testid="imagePreview"]')?.getAttribute('src'), document.querySelector('.file-preview-modal img')?.getAttribute('src'), - document.querySelector('.post-image img[src*="/files/"]')?.getAttribute('src'), - document.querySelector('img[src*="/api/v4/files/"]')?.getAttribute('src'), + document.querySelector('.post-image img[src*="/files/"]:not(.image-loading__placeholder)')?.getAttribute('src'), + document.querySelector('img[src*="/api/v4/files/"]:not(.image-loading__placeholder)')?.getAttribute('src'), ].filter(Boolean); for (const source of sources) { diff --git a/e2e/utils/cmt-channel-notify.js b/e2e/utils/cmt-channel-notify.js new file mode 100644 index 00000000000..27d78dab42d --- /dev/null +++ b/e2e/utils/cmt-channel-notify.js @@ -0,0 +1,657 @@ +// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. +/* eslint-disable no-console -- Logging is intentional in CI utility scripts */ + +/** + * Post a CMT rollup to a Mattermost incoming webhook. + * + * Expected job names from e2e-functional-template.yml: + * e2e-on-{runner}-{serverVersion} + * e.g. e2e-on-ubuntu-latest-11.9.0, e2e-on-windows-2022-10.5.14 + * + * Per-leg pass/fail counts come from TSIO consolidated specs grouped by + * contributing report id → gh_job_name (group report only has upload status). + * + * Retry attempts are collapsed to one outcome per (spec × job), matching + * Playwright: failed+passed → flaky; all failed → failed once (not once per attempt). + */ + +const OS_ORDER = {linux: 0, macos: 1, windows: 2}; +const FETCH_TIMEOUT_MS = 15_000; + +/** + * Fetch with an abort timeout that stays armed until `readBody` finishes + * (headers + body), so a stalled `res.json()`/`res.text()` cannot hang CI. + * + * @template T + * @param {string} url + * @param {RequestInit | undefined} options + * @param {(res: Response) => Promise} readBody + * @param {number} [timeoutMs] + * @returns {Promise} + */ +async function fetchWithTimeout(url, options, readBody, timeoutMs = FETCH_TIMEOUT_MS) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + const res = await fetch(url, {...(options || {}), signal: controller.signal}); + return await readBody(res); + } finally { + clearTimeout(timer); + } +} + +/** + * @param {string} token + * @returns {string} + */ +function osFromRunnerToken(token) { + if (token.startsWith('ubuntu') || token.startsWith('linux')) { + return 'linux'; + } + if (token.startsWith('windows')) { + return 'windows'; + } + if (token.startsWith('macos') || token.startsWith('darwin')) { + return 'macos'; + } + return 'unknown'; +} + +function parseCmtJobName(jobName) { + if (!jobName || typeof jobName !== 'string') { + return null; + } + + // PR/master policy legs: policy-tests-macos / policy-tests-windows + const policyMatch = jobName.match(/^policy-tests-(macos|windows|linux)$/); + if (policyMatch) { + return { + os: policyMatch[1] === 'linux' ? 'linux' : policyMatch[1], + serverVersion: 'policy', + runner: policyMatch[1], + kind: 'policy', + }; + } + + // Server versions may include pre-release: 11.9.0-rc.3 + const match = jobName.match(/^e2e-on-(.+)-(\d+\.\d+\.\d+(?:[-.][\w.]+)?)$/); + if (!match) { + return null; + } + + const runner = match[1]; + return { + os: osFromRunnerToken(runner), + serverVersion: match[2], + runner, + kind: 'e2e', + }; +} + +/** + * Webhook routing (fail closed for named report groups): + * cmt-desktop → MATTERMOST_CMT_WEBHOOK_URL only (MM_E2E_RELEASE_WEBHOOK_URL) + * desktop-master → MATTERMOST_MASTER_HEALTH_WEBHOOK_URL only (MM_E2E_MASTER_HEALTH_WEBHOOK_URL) + * desktop-pr → MATTERMOST_E2E_WEBHOOK_URL only (MM_DESKTOP_E2E_WEBHOOK_URL) + * + * Named groups never fall back to MATTERMOST_WEBHOOK_URL (avoids posting + * CMT/PR/master to the wrong channel when a dedicated secret is missing). + * Unknown names still use MATTERMOST_WEBHOOK_URL as a generic fallback. + * + * @param {string} reportName - compositeIdentity.name + * @param {NodeJS.ProcessEnv} [env] + * @returns {string} + */ +function resolveWebhookUrl(reportName, env = process.env) { + if (reportName === 'cmt-desktop') { + return env.MATTERMOST_CMT_WEBHOOK_URL || ''; + } + if (reportName === 'desktop-master') { + return env.MATTERMOST_MASTER_HEALTH_WEBHOOK_URL || ''; + } + if (reportName === 'desktop-pr') { + return env.MATTERMOST_E2E_WEBHOOK_URL || ''; + } + return env.MATTERMOST_WEBHOOK_URL || ''; +} + +/** + * Individual leg UI path in TSIO: /reports/r/{id} + * (see mattermost-test-system-io apps/web Route path="/reports/r/:id"). + * + * @param {string} baseUrl + * @param {string} reportId + * @returns {string} + */ +function buildIndividualReportUrl(baseUrl, reportId) { + return `${baseUrl.replace(/\/$/, '')}/reports/r/${reportId}`; +} + +/** + * @param {Record} perJobCounts + * @param {Array<{id?: string, gh_job_name?: string, display_name?: string, status?: string}>} uploadedReports + * @param {string} [baseUrl] - TSIO origin used to build per-leg report links + * @returns {Array<{label: string, status: string, passed: number, failed: number, skipped: number, total: number, os: string, serverVersion: string, reportUrl?: string}>} + */ +function buildLegSummaries(perJobCounts, uploadedReports, baseUrl) { + const jobNames = new Set([ + ...Object.keys(perJobCounts || {}), + ...(uploadedReports || []).map((r) => r.gh_job_name || r.display_name).filter(Boolean), + ]); + + const rows = []; + for (const jobName of jobNames) { + const parsed = parseCmtJobName(jobName); + if (!parsed) { + continue; + } + + const counts = perJobCounts[jobName] || {}; + const passed = (counts.passed || 0) + (counts.flaky || 0); + const failed = counts.failed || 0; + const skipped = counts.skipped || 0; + const total = passed + failed + skipped; + const uploaded = (uploadedReports || []).find( + (r) => (r.gh_job_name || r.display_name) === jobName, + ); + + let status; + if (total === 0) { + const uploadedOk = uploaded?.status === 'complete' || uploaded?.status === 'completed'; + status = uploadedOk ? 'no-results' : 'missing'; + } else { + status = failed === 0 ? 'passed' : 'failed'; + } + + rows.push({ + label: `${parsed.serverVersion}-${parsed.os}`, + status, + passed, + failed, + skipped, + total, + os: parsed.os, + serverVersion: parsed.serverVersion, + kind: parsed.kind || 'e2e', + reportUrl: uploaded?.id && baseUrl ? buildIndividualReportUrl(baseUrl, uploaded.id) : undefined, + }); + } + + rows.sort((a, b) => { + const osCmp = (OS_ORDER[a.os] ?? 9) - (OS_ORDER[b.os] ?? 9); + if (osCmp !== 0) { + return osCmp; + } + if (a.kind !== b.kind) { + return a.kind === 'policy' ? 1 : -1; + } + return a.serverVersion.localeCompare(b.serverVersion, undefined, {numeric: true}); + }); + + return rows; +} + +const PLATFORM_EMOJI = { + linux: '🐧', + macos: '🍎', + windows: '🪟', + unknown: '❔', +}; + +const PLATFORM_LABEL = { + linux: 'Linux', + macos: 'macOS', + windows: 'Windows', + unknown: 'Other', +}; + +/** + * @param {{os: string, kind?: string, serverVersion: string}} leg + * @returns {{platform: string, suite: string}} + */ +function formatLegLabels(leg) { + const emoji = PLATFORM_EMOJI[leg.os] || PLATFORM_EMOJI.unknown; + const name = PLATFORM_LABEL[leg.os] || PLATFORM_LABEL.unknown; + const suite = leg.kind === 'policy' ? 'Policy' : `Server \`${leg.serverVersion}\``; + return {platform: `${emoji} ${name}`, suite}; +} + +/** + * Denominator is executed tests (passed + failed), matching Playwright summary style. + * + * @param {{status: string, passed: number, failed: number, skipped?: number}} leg + * @returns {string} + */ +function formatLegResultText(leg) { + if (leg.status === 'missing' || leg.status === 'no-results') { + return `⚠️ ${leg.status}`; + } + + // All-skipped legs are marked "passed" by buildLegSummaries (failed === 0) but + // should not render as ✅ 0/0. + if ((leg.passed || 0) === 0 && (leg.failed || 0) === 0 && (leg.skipped || 0) > 0) { + return '⚠️ not executed'; + } + const executed = leg.passed + leg.failed; + if (leg.status === 'passed') { + return `✅ ${leg.passed}/${executed || leg.passed}`; + } + return `❌ ${leg.passed}/${executed}`; +} + +function reportTitleForIdentity(compositeIdentity) { + switch (compositeIdentity?.name) { + case 'cmt-desktop': + return 'Desktop CMT'; + case 'desktop-pr': + return 'Desktop PR E2E'; + case 'desktop-master': + return 'Desktop Master E2E'; + default: + return 'Desktop E2E'; + } +} + +/** + * @param {Object} compositeIdentity + * @returns {string} + */ +function formatMetaLine(compositeIdentity) { + const branch = (compositeIdentity.branch || '').replace(/^refs\/(heads|tags)\//, ''); + const shortSha = (compositeIdentity.commit_sha || '').slice(0, 7); + const parts = []; + + if (compositeIdentity.gh_pr_number) { + const repo = compositeIdentity.repository || 'mattermost/desktop'; + parts.push(`**PR:** [#${compositeIdentity.gh_pr_number}](https://github.com/${repo}/pull/${compositeIdentity.gh_pr_number})`); + } + if (branch) { + parts.push(`**Branch:** \`${branch}\``); + } + if (shortSha) { + parts.push(`**Commit:** \`${shortSha}\``); + } + return parts.join(' · '); +} + +/** + * Prefer unique per-leg counts (retry-collapsed) over TSIO group test_stats, which + * counts every failed attempt and double-counts retries in channel alerts. + * + * When `expectedJobNames` is provided, per-leg totals are used only if every expected + * job is present — otherwise fall back to `stats` so partial consolidation cannot hide + * aggregate failures. + * + * @param {Record} perJobCounts + * @param {{passed?: number, failed?: number, skipped?: number, flaky?: number}} stats + * @param {string[]} [expectedJobNames] - uploaded report job names; omit to trust any per-leg map + * @returns {{passed: number, failed: number, skipped: number}} + */ +function resolveChannelTotals(perJobCounts, stats = {}, expectedJobNames) { + const counts = perJobCounts || {}; + + let jobs; + if (expectedJobNames === undefined) { + jobs = Object.values(counts); + } else if (expectedJobNames.length > 0 && + expectedJobNames.every((job) => Object.prototype.hasOwnProperty.call(counts, job))) { + jobs = expectedJobNames.map((job) => counts[job]); + } else { + jobs = []; + } + + if (jobs.length > 0) { + let passed = 0; + let failed = 0; + let skipped = 0; + for (const jobCounts of jobs) { + passed += (jobCounts.passed || 0) + (jobCounts.flaky || 0); + failed += jobCounts.failed || 0; + skipped += jobCounts.skipped || 0; + } + return {passed, failed, skipped}; + } + + return { + passed: (stats.passed ?? 0) + (stats.flaky ?? 0), + failed: stats.failed ?? 0, + skipped: stats.skipped ?? 0, + }; +} + +/** + * @param {Object} params + * @param {Object} params.compositeIdentity + * @param {Object} params.detail - TSIO group report detail + * @param {string} params.reportUrl - group / consolidated rollup URL + * @param {string} [params.baseUrl] - TSIO origin for per-leg /reports/r/{id} links + * @param {Record} params.perJobCounts + * @param {boolean} [params.upstreamJobsSucceeded] + * @param {boolean} [params.hasFailures] - true when TSIO reports failed shards or failed tests + * @returns {string} + */ +function formatCmtChannelMessage({ + compositeIdentity, + detail, + reportUrl, + baseUrl, + perJobCounts, + upstreamJobsSucceeded = true, + hasFailures = false, +}) { + const reports = detail?.reports || []; + const legs = buildLegSummaries(perJobCounts, reports, baseUrl); + const expectedJobNames = [...new Set( + reports.map((r) => r.gh_job_name || r.display_name).filter(Boolean), + )]; + const {passed, failed, skipped} = resolveChannelTotals( + perJobCounts, + detail?.test_stats, + expectedJobNames, + ); + + // Overall pass/fail follows tests + upstream CI — not TSIO consolidation state. + // Stuck `in_progress` / `incomplete` with 0 failures must not render as ❌ Failed. + const overallFailed = failed > 0 || !upstreamJobsSucceeded || hasFailures; + const tsioPending = Boolean(detail?.status && detail.status !== 'completed'); + const title = reportTitleForIdentity(compositeIdentity); + const missingLegs = legs.filter((leg) => leg.status === 'missing' || leg.status === 'no-results'); + + const lines = [ + `## ${overallFailed ? '❌' : '✅'} ${title}`, + '', + ]; + + const meta = formatMetaLine(compositeIdentity); + if (meta) { + lines.push(meta, ''); + } + + if (failed > 0) { + const failingLegs = legs.filter((leg) => leg.status === 'failed' && leg.failed > 0); + lines.push(`🔴 **${failed} failing test${failed === 1 ? '' : 's'}**`, ''); + if (failingLegs.length > 0) { + lines.push('| Platform | Suite | Failed |', '|----------|-------|-------:|'); + for (const leg of failingLegs) { + const {platform, suite} = formatLegLabels(leg); + lines.push(`| ${platform} | ${suite} | ${leg.failed} |`); + } + lines.push(''); + } + } + + lines.push( + '| Overall | Passed | Failed | Skipped |', + '|---------|------:|------:|-------:|', + `| ${overallFailed ? '❌ Failed' : '✅ Passed'} | **${passed}** | **${failed}** | **${skipped}** |`, + '', + ); + + if (legs.length > 0) { + // Mattermost incoming webhooks do not render GitHub
/. + lines.push('#### Detailed results', ''); + lines.push('| Platform | Suite | Result | Report |', '|----------|-------|--------|--------|'); + for (const leg of legs) { + const {platform, suite} = formatLegLabels(leg); + const report = leg.reportUrl ? `[View](${leg.reportUrl})` : '—'; + lines.push(`| ${platform} | ${suite} | ${formatLegResultText(leg)} | ${report} |`); + } + lines.push(''); + } else { + lines.push('_No per-leg results available yet._', ''); + } + + if (!upstreamJobsSucceeded) { + lines.push('_One or more CI jobs failed outside tracked tests (install/build/teardown)._', ''); + } else if (hasFailures && failed === 0) { + lines.push('_TSIO reported failed shard(s) not reflected in the test totals; check the full report._', ''); + } + if (tsioPending && upstreamJobsSucceeded && !overallFailed) { + lines.push( + `_TSIO report status: \`${detail.status}\` (consolidation still catching up; not treated as a test failure)._`, + '', + ); + } else if (tsioPending && overallFailed) { + lines.push(`_TSIO report status: \`${detail.status}\`._`, ''); + } + if (missingLegs.length > 0) { + const labels = missingLegs.map((leg) => { + const {platform, suite} = formatLegLabels(leg); + return `${platform} / ${suite}`; + }).join(', '); + lines.push(`_Missing or empty leg report(s): ${labels}._`, ''); + } + + if (reportUrl) { + lines.push(`➡️ **Full report:** ${reportUrl}`); + } + + return lines.join('\n').trimEnd() + '\n'; +} + +/** + * Collapse TSIO history attempts for one spec on one job into a single Playwright-style status. + * failed then passed (or explicit flaky) → flaky; all failed → failed once. + * + * @param {Array<{status?: string}>} entries + * @returns {'passed'|'failed'|'skipped'|'flaky'|null} + */ +function collapseSpecAttempts(entries) { + if (!entries || entries.length === 0) { + return null; + } + + let sawPassed = false; + let sawFailed = false; + let sawSkipped = false; + let sawFlaky = false; + + for (const entry of entries) { + const status = entry.status || 'failed'; + if (status === 'passed') { + sawPassed = true; + } else if (status === 'failed') { + sawFailed = true; + } else if (status === 'skipped') { + sawSkipped = true; + } else if (status === 'flaky') { + sawFlaky = true; + } else { + // Unknown statuses treated as failures so they still surface in alerts. + sawFailed = true; + } + } + + if (sawFlaky || (sawFailed && sawPassed)) { + return 'flaky'; + } + if (sawFailed) { + return 'failed'; + } + if (sawPassed) { + return 'passed'; + } + if (sawSkipped) { + return 'skipped'; + } + return null; +} + +/** + * @param {string} baseUrl + * @param {Object} compositeIdentity + * @param {Object} groupDetail + * @returns {Promise>} + */ +async function fetchPerJobCountsFromConsolidated(baseUrl, compositeIdentity, groupDetail) { + const idToJob = {}; + for (const report of groupDetail.reports || []) { + const name = report.gh_job_name || report.display_name; + if (report.id && name) { + idToJob[report.id] = name; + } + } + + const repoTrailing = (compositeIdentity.repository || '').split('/').pop() || compositeIdentity.repository; + const params = new URLSearchParams({ + repository: repoTrailing, + branch: (compositeIdentity.branch || '').replace(/^refs\/(heads|tags)\//, ''), + commit: compositeIdentity.commit_sha, + name: compositeIdentity.name, + gh_run_id: String(compositeIdentity.gh_run_id), + }); + if (compositeIdentity.gh_run_attempt) { + params.set('gh_run_attempt', String(compositeIdentity.gh_run_attempt)); + } + + const consol = await fetchWithTimeout( + `${baseUrl}/api/v1/reports/consolidated?${params}`, + undefined, + async (res) => { + if (!res.ok) { + throw new Error(`consolidated fetch failed: ${res.status} ${await res.text()}`); + } + return res.json(); + }, + ); + + const counts = {}; + const commitSha = compositeIdentity.commit_sha; + const attempt = Number.parseInt(compositeIdentity.gh_run_attempt || '1', 10); + + for (const spec of consol.specs || []) { + /** @type {Record>} */ + const entriesByJob = {}; + for (const entry of spec.history || []) { + if (entry.commit_sha !== commitSha) { + continue; + } + if (Number.parseInt(entry.run_attempt || '0', 10) !== attempt) { + continue; + } + const job = idToJob[entry.report_id]; + if (!job) { + continue; + } + if (!entriesByJob[job]) { + entriesByJob[job] = []; + } + entriesByJob[job].push(entry); + } + + for (const [job, entries] of Object.entries(entriesByJob)) { + const status = collapseSpecAttempts(entries); + if (!status) { + continue; + } + if (!counts[job]) { + counts[job] = {passed: 0, failed: 0, skipped: 0, flaky: 0}; + } + counts[job][status] += 1; + } + } + + return counts; +} + +/** + * @param {Object} params + * @param {Object} params.core + * @param {string} params.webhookUrl + * @param {string} params.text + * @param {string} [params.username] + */ +async function postMattermostWebhook({core, webhookUrl, text, username = 'Desktop E2E'}) { + if (!webhookUrl) { + core.info('Mattermost webhook URL not set — skipping E2E channel notify'); + return; + } + + const body = { + username, + icon_url: 'https://mattermost.com/wp-content/uploads/2022/02/icon.png', + text, + }; + + await fetchWithTimeout( + webhookUrl, + { + method: 'POST', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify(body), + }, + async (res) => { + if (!res.ok) { + throw new Error(`Mattermost webhook failed: ${res.status} ${await res.text()}`); + } + + // Drain body so the abort timer covers the full response, not just headers. + await res.text(); + }, + ); + core.info('Posted E2E summary to Mattermost channel'); +} + +/** + * Build + post the CMT channel message. Never throws to the caller — notify is best-effort. + * + * @param {Object} params + * @param {Object} params.core + * @param {string} params.baseUrl + * @param {Object} params.compositeIdentity + * @param {Object} params.detail + * @param {string} params.reportUrl + * @param {boolean} [params.upstreamJobsSucceeded] + * @param {boolean} [params.hasFailures] + * @param {string} [params.webhookUrl] + */ +async function notifyCmtChannel({ + core, + baseUrl, + compositeIdentity, + detail, + reportUrl, + upstreamJobsSucceeded = true, + hasFailures = false, + webhookUrl, +}) { + try { + const resolvedWebhook = webhookUrl || resolveWebhookUrl(compositeIdentity?.name); + let perJobCounts = {}; + try { + perJobCounts = await fetchPerJobCountsFromConsolidated(baseUrl, compositeIdentity, detail); + } catch (error) { + core.warning(`Could not load per-leg TSIO counts: ${error.message}`); + } + + const text = formatCmtChannelMessage({ + compositeIdentity, + detail, + reportUrl, + baseUrl, + perJobCounts, + upstreamJobsSucceeded, + hasFailures, + }); + + await postMattermostWebhook({core, webhookUrl: resolvedWebhook, text}); + } catch (error) { + core.warning(`E2E Mattermost notify failed: ${error.message}`); + } +} + +module.exports = { + parseCmtJobName, + resolveWebhookUrl, + buildIndividualReportUrl, + buildLegSummaries, + formatLegResultText, + reportTitleForIdentity, + resolveChannelTotals, + collapseSpecAttempts, + formatCmtChannelMessage, + fetchPerJobCountsFromConsolidated, + postMattermostWebhook, + notifyCmtChannel, +}; diff --git a/e2e/utils/cmt-channel-notify.test.js b/e2e/utils/cmt-channel-notify.test.js new file mode 100644 index 00000000000..089a230a636 --- /dev/null +++ b/e2e/utils/cmt-channel-notify.test.js @@ -0,0 +1,413 @@ +// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +// CI util unit tests: run with `node --test e2e/utils/cmt-channel-notify.test.js`. +// Not a Playwright Electron spec (no browser/app fixtures), so it stays under e2e/utils/. + +const {describe, it} = require('node:test'); +const assert = require('node:assert/strict'); + +const { + parseCmtJobName, + resolveWebhookUrl, + buildLegSummaries, + formatLegResultText, + formatCmtChannelMessage, + collapseSpecAttempts, + resolveChannelTotals, +} = require('./cmt-channel-notify'); + +describe('cmt-channel-notify', () => { + describe('collapseSpecAttempts', () => { + it('counts a double-failed retry as one failure', () => { + assert.equal( + collapseSpecAttempts([{status: 'failed'}, {status: 'failed'}]), + 'failed', + ); + }); + + it('treats failed-then-passed as flaky', () => { + assert.equal( + collapseSpecAttempts([{status: 'failed'}, {status: 'passed'}]), + 'flaky', + ); + }); + + it('keeps explicit flaky and single pass/skip', () => { + assert.equal(collapseSpecAttempts([{status: 'flaky'}]), 'flaky'); + assert.equal(collapseSpecAttempts([{status: 'passed'}]), 'passed'); + assert.equal(collapseSpecAttempts([{status: 'skipped'}]), 'skipped'); + assert.equal(collapseSpecAttempts([]), null); + }); + }); + + describe('resolveChannelTotals', () => { + it('sums unique per-leg counts and folds flaky into passed', () => { + assert.deepEqual( + resolveChannelTotals({ + 'e2e-on-ubuntu-latest-11.9.0': {passed: 218, failed: 1, skipped: 11, flaky: 0}, + 'e2e-on-windows-2022-10.11.23': {passed: 214, failed: 2, skipped: 34, flaky: 1}, + }), + {passed: 433, failed: 3, skipped: 45}, + ); + }); + + it('falls back to test_stats when per-leg counts are empty', () => { + assert.deepEqual( + resolveChannelTotals({}, {passed: 200, failed: 14, skipped: 5, flaky: 3}), + {passed: 203, failed: 14, skipped: 5}, + ); + }); + + it('falls back to aggregate stats when an expected leg is missing from per-job counts', () => { + assert.deepEqual( + resolveChannelTotals( + { + 'e2e-on-ubuntu-latest-11.9.0': {passed: 218, failed: 0, skipped: 11, flaky: 0}, + }, + {passed: 400, failed: 5, skipped: 20, flaky: 0}, + [ + 'e2e-on-ubuntu-latest-11.9.0', + 'e2e-on-windows-2022-11.9.0', + ], + ), + {passed: 400, failed: 5, skipped: 20}, + ); + }); + }); + + describe('formatLegResultText', () => { + it('renders all-skipped legs as not executed instead of ✅ 0/0', () => { + assert.equal( + formatLegResultText({status: 'passed', passed: 0, failed: 0, skipped: 12}), + '⚠️ not executed', + ); + }); + + it('preserves missing, no-results, passed, and failed formatting', () => { + assert.equal(formatLegResultText({status: 'missing', passed: 0, failed: 0, skipped: 0}), '⚠️ missing'); + assert.equal(formatLegResultText({status: 'no-results', passed: 0, failed: 0, skipped: 0}), '⚠️ no-results'); + assert.equal(formatLegResultText({status: 'passed', passed: 231, failed: 0, skipped: 10}), '✅ 231/231'); + assert.equal(formatLegResultText({status: 'failed', passed: 229, failed: 2, skipped: 10}), '❌ 229/231'); + }); + }); + describe('parseCmtJobName', () => { + it('parses ubuntu, windows, and macos job names', () => { + assert.deepEqual(parseCmtJobName('e2e-on-ubuntu-latest-11.9.0'), { + os: 'linux', + serverVersion: '11.9.0', + runner: 'ubuntu-latest', + kind: 'e2e', + }); + assert.deepEqual(parseCmtJobName('e2e-on-windows-2022-10.5.14'), { + os: 'windows', + serverVersion: '10.5.14', + runner: 'windows-2022', + kind: 'e2e', + }); + assert.deepEqual(parseCmtJobName('e2e-on-macos-13-11.8.3-rc.1'), { + os: 'macos', + serverVersion: '11.8.3-rc.1', + runner: 'macos-13', + kind: 'e2e', + }); + }); + + it('parses policy-tests job names', () => { + assert.deepEqual(parseCmtJobName('policy-tests-macos'), { + os: 'macos', + serverVersion: 'policy', + runner: 'macos', + kind: 'policy', + }); + assert.deepEqual(parseCmtJobName('policy-tests-windows'), { + os: 'windows', + serverVersion: 'policy', + runner: 'windows', + kind: 'policy', + }); + }); + + it('returns null for unexpected names', () => { + assert.equal(parseCmtJobName('linux-11.9.0'), null); + assert.equal(parseCmtJobName(''), null); + }); + }); + + describe('resolveWebhookUrl', () => { + const env = { + MATTERMOST_CMT_WEBHOOK_URL: 'https://mm.example/hooks/cmt', + MATTERMOST_E2E_WEBHOOK_URL: 'https://mm.example/hooks/e2e', + MATTERMOST_MASTER_HEALTH_WEBHOOK_URL: 'https://mm.example/hooks/master-health', + MATTERMOST_WEBHOOK_URL: 'https://mm.example/hooks/fallback', + }; + + it('sends CMT to the release webhook', () => { + assert.equal(resolveWebhookUrl('cmt-desktop', env), env.MATTERMOST_CMT_WEBHOOK_URL); + }); + + it('sends master runs to the master-health webhook', () => { + assert.equal(resolveWebhookUrl('desktop-master', env), env.MATTERMOST_MASTER_HEALTH_WEBHOOK_URL); + }); + + it('sends PR runs to the E2E webhook', () => { + assert.equal(resolveWebhookUrl('desktop-pr', env), env.MATTERMOST_E2E_WEBHOOK_URL); + }); + + it('does not fall back CMT to the E2E webhook when release secret is missing', () => { + assert.equal( + resolveWebhookUrl('cmt-desktop', {MATTERMOST_E2E_WEBHOOK_URL: env.MATTERMOST_E2E_WEBHOOK_URL}), + '', + ); + }); + + it('does not use the shared webhook when a dedicated CMT secret is missing', () => { + assert.equal( + resolveWebhookUrl('cmt-desktop', {MATTERMOST_WEBHOOK_URL: env.MATTERMOST_WEBHOOK_URL}), + '', + ); + }); + + it('does not use the shared or PR webhook when master-health secret is missing', () => { + assert.equal( + resolveWebhookUrl('desktop-master', { + MATTERMOST_WEBHOOK_URL: env.MATTERMOST_WEBHOOK_URL, + MATTERMOST_E2E_WEBHOOK_URL: env.MATTERMOST_E2E_WEBHOOK_URL, + }), + '', + ); + }); + + it('does not use the shared webhook when a dedicated E2E secret is missing for PR', () => { + assert.equal( + resolveWebhookUrl('desktop-pr', {MATTERMOST_WEBHOOK_URL: env.MATTERMOST_WEBHOOK_URL}), + '', + ); + }); + + it('still uses the shared webhook for unknown report names', () => { + assert.equal(resolveWebhookUrl('unknown-report', env), env.MATTERMOST_WEBHOOK_URL); + }); + }); + + describe('formatCmtChannelMessage', () => { + it('renders failure banner, overall table, and per-leg details', () => { + const text = formatCmtChannelMessage({ + compositeIdentity: { + branch: 'v6.2.0-rc.1', + commit_sha: '55afc0b839545804ee156fe95b4c1ac05c9d0cdc', + name: 'cmt-desktop', + }, + detail: { + status: 'completed', + test_stats: {passed: 460, failed: 1, skipped: 40, total: 501}, + reports: [ + {id: 'rid-linux', gh_job_name: 'e2e-on-ubuntu-latest-11.9.0', status: 'complete'}, + {id: 'rid-windows', gh_job_name: 'e2e-on-windows-2022-11.9.0', status: 'complete'}, + ], + }, + reportUrl: 'https://test-io.test.mattermost.com/reports/desktop/v6.2.0-rc.1/55afc0b/cmt-desktop', + baseUrl: 'https://test-io.test.mattermost.com', + perJobCounts: { + 'e2e-on-ubuntu-latest-11.9.0': {passed: 231, failed: 0, skipped: 20, flaky: 0}, + 'e2e-on-windows-2022-11.9.0': {passed: 230, failed: 1, skipped: 20, flaky: 0}, + }, + upstreamJobsSucceeded: true, + }); + + assert.match(text, /^## ❌ Desktop CMT\n/); + assert.match(text, /\*\*Branch:\*\* `v6\.2\.0-rc\.1` · \*\*Commit:\*\* `55afc0b`/); + assert.match(text, /🔴 \*\*1 failing test\*\*/); + assert.match(text, /\| 🪟 Windows \| Server `11\.9\.0` \| 1 \|/); + + // Overall totals come from unique per-leg counts (not inflated TSIO test_stats). + assert.match(text, /\| ❌ Failed \| \*\*461\*\* \| \*\*1\*\* \| \*\*40\*\* \|/); + assert.match(text, /#### Detailed results/); + assert.doesNotMatch(text, /
/); + assert.match(text, /\| 🐧 Linux \| Server `11\.9\.0` \| ✅ 231\/231 \| \[View\]\(https:\/\/test-io\.test\.mattermost\.com\/reports\/r\/rid-linux\) \|/); + assert.match(text, /\| 🪟 Windows \| Server `11\.9\.0` \| ❌ 230\/231 \| \[View\]\(https:\/\/test-io\.test\.mattermost\.com\/reports\/r\/rid-windows\) \|/); + assert.match(text, /➡️ \*\*Full report:\*\* https:\/\/test-io\.test\.mattermost\.com\/reports\/desktop\/v6\.2\.0-rc\.1\/55afc0b\/cmt-desktop/); + }); + + it('prefers unique per-leg failed counts over inflated TSIO attempt totals', () => { + const text = formatCmtChannelMessage({ + compositeIdentity: { + branch: 'master', + commit_sha: '5eda917312db037975bac5c3535272c61a664674', + name: 'cmt-desktop', + }, + detail: { + status: 'completed', + + // Inflated: each retry attempt counted (e.g. 1 unique fail × 2 attempts). + test_stats: {passed: 218, failed: 2, skipped: 11, total: 231}, + reports: [ + {id: 'rid-linux', gh_job_name: 'e2e-on-ubuntu-latest-11.9.1', status: 'complete'}, + ], + }, + reportUrl: 'https://test-io.test.mattermost.com/reports/desktop/master/5eda917/cmt-desktop', + baseUrl: 'https://test-io.test.mattermost.com', + perJobCounts: { + + // Unique Playwright-style count after collapsing retries. + 'e2e-on-ubuntu-latest-11.9.1': {passed: 218, failed: 1, skipped: 11, flaky: 0}, + }, + upstreamJobsSucceeded: true, + }); + + assert.match(text, /🔴 \*\*1 failing test\*\*/); + assert.match(text, /\| 🐧 Linux \| Server `11\.9\.1` \| 1 \|/); + assert.match(text, /\| ❌ Failed \| \*\*218\*\* \| \*\*1\*\* \| \*\*11\*\* \|/); + assert.match(text, /\| 🐧 Linux \| Server `11\.9\.1` \| ❌ 218\/219 \|/); + assert.doesNotMatch(text, /🔴 \*\*2 failing tests\*\*/); + }); + + it('renders a passed PR report with PR link and no failure banner', () => { + const text = formatCmtChannelMessage({ + compositeIdentity: { + repository: 'mattermost/desktop', + branch: 'pr-3891', + commit_sha: '55afc0b839545804ee156fe95b4c1ac05c9d0cdc', + name: 'desktop-pr', + gh_pr_number: '3891', + }, + detail: { + status: 'completed', + test_stats: {passed: 240, failed: 0, skipped: 10, total: 250}, + reports: [], + }, + reportUrl: 'https://test-io.test.mattermost.com/reports/desktop/pr-3891/55afc0b/desktop-pr', + baseUrl: 'https://test-io.test.mattermost.com', + perJobCounts: {}, + upstreamJobsSucceeded: true, + }); + assert.match(text, /^## ✅ Desktop PR E2E\n/); + assert.match(text, /\*\*PR:\*\* \[#3891\]\(https:\/\/github\.com\/mattermost\/desktop\/pull\/3891\)/); + assert.doesNotMatch(text, /failing test/); + assert.match(text, /\| ✅ Passed \| \*\*240\*\* \| \*\*0\*\* \| \*\*10\*\* \|/); + }); + + it('marks overall failed when hasFailures is set without test failures', () => { + const text = formatCmtChannelMessage({ + compositeIdentity: { + branch: 'master', + commit_sha: 'a1b2c3d4e5f678901234567890abcdef12345678', + name: 'desktop-master', + }, + detail: { + status: 'completed', + test_stats: {passed: 100, failed: 0, skipped: 0, total: 100}, + reports: [], + }, + reportUrl: 'https://test-io.test.mattermost.com/reports/desktop/master/a1b2c3d/desktop-master', + baseUrl: 'https://test-io.test.mattermost.com', + perJobCounts: {}, + upstreamJobsSucceeded: true, + hasFailures: true, + }); + assert.match(text, /^## ❌ Desktop Master E2E\n/); + assert.match(text, /\| ❌ Failed \| \*\*100\*\* \| \*\*0\*\* \| \*\*0\*\* \|/); + assert.match(text, /TSIO reported failed shard\(s\) not reflected in the test totals/); + assert.doesNotMatch(text, /failing test/); + }); + + it('keeps overall passed when TSIO is still consolidating with 0 failures', () => { + const text = formatCmtChannelMessage({ + compositeIdentity: { + repository: 'mattermost/desktop', + branch: 'pr-3916', + commit_sha: 'b41298f0abc1234567890abcdef1234567890abcd', + name: 'desktop-pr', + gh_pr_number: '3916', + }, + detail: { + status: 'in_progress', + test_stats: {passed: 674, failed: 0, skipped: 51, total: 725}, + reports: [ + {id: 'rid-linux', gh_job_name: 'e2e-on-ubuntu-latest-11.10.0-rc1', status: 'complete'}, + {id: 'rid-mac', gh_job_name: 'e2e-on-macos-14-11.10.0-rc1', status: 'complete'}, + {id: 'rid-win', gh_job_name: 'e2e-on-windows-2022-11.10.0-rc1', status: 'complete'}, + {id: 'rid-win-policy', gh_job_name: 'policy-tests-windows', status: 'complete'}, + ], + }, + reportUrl: 'https://test-io.test.mattermost.com/reports/desktop/pr-3916/b41298f/desktop-pr', + baseUrl: 'https://test-io.test.mattermost.com', + perJobCounts: { + 'e2e-on-ubuntu-latest-11.10.0-rc1': {passed: 216, failed: 0, skipped: 10, flaky: 0}, + 'e2e-on-macos-14-11.10.0-rc1': {passed: 220, failed: 0, skipped: 10, flaky: 0}, + 'e2e-on-windows-2022-11.10.0-rc1': {passed: 237, failed: 0, skipped: 10, flaky: 0}, + 'policy-tests-windows': {passed: 9, failed: 0, skipped: 0, flaky: 0}, + 'policy-tests-macos': {passed: 0, failed: 0, skipped: 0, flaky: 0}, + }, + upstreamJobsSucceeded: true, + }); + assert.match(text, /^## ✅ Desktop PR E2E\n/); + + // Unique per-leg sum: 216+220+237+9 = 682 passed, 30 skipped (not TSIO test_stats 674/51). + assert.match(text, /\| ✅ Passed \| \*\*682\*\* \| \*\*0\*\* \| \*\*30\*\* \|/); + assert.match(text, /TSIO report status: `in_progress` \(consolidation still catching up; not treated as a test failure\)/); + assert.match(text, /Missing or empty leg report\(s\): 🍎 macOS \/ Policy/); + assert.match(text, /\| 🍎 macOS \| Policy \| ⚠️ missing \|/); + assert.doesNotMatch(text, /\| ❌ Failed \|/); + }); + + it('still marks overall failed for incomplete TSIO when tests failed', () => { + const text = formatCmtChannelMessage({ + compositeIdentity: { + branch: 'master', + commit_sha: 'a1b2c3d4e5f678901234567890abcdef12345678', + name: 'desktop-master', + }, + detail: { + status: 'incomplete', + test_stats: {passed: 49, failed: 1, skipped: 0, total: 50}, + reports: [], + }, + reportUrl: 'https://test-io.test.mattermost.com/reports/desktop/master/a1b2c3d/desktop-master', + baseUrl: 'https://test-io.test.mattermost.com', + perJobCounts: {}, + upstreamJobsSucceeded: true, + }); + assert.match(text, /^## ❌ Desktop Master E2E\n/); + assert.match(text, /TSIO report status: `incomplete`/); + }); + + it('folds test_stats.flaky into the headline passed count', () => { + const text = formatCmtChannelMessage({ + compositeIdentity: { + branch: 'master', + commit_sha: 'a1b2c3d4e5f678901234567890abcdef12345678', + name: 'desktop-master', + }, + detail: { + status: 'completed', + test_stats: {passed: 200, failed: 0, skipped: 5, flaky: 3, total: 208}, + reports: [], + }, + reportUrl: 'https://test-io.test.mattermost.com/reports/desktop/master/a1b2c3d/desktop-master', + baseUrl: 'https://test-io.test.mattermost.com', + perJobCounts: {}, + upstreamJobsSucceeded: true, + }); + assert.match(text, /\| ✅ Passed \| \*\*203\*\* \| \*\*0\*\* \| \*\*5\*\* \|/); + }); + }); + + describe('buildLegSummaries', () => { + it('sorts by OS then suite kind then server version', () => { + const rows = buildLegSummaries( + { + 'e2e-on-windows-2022-10.5.14': {passed: 1, failed: 0, skipped: 0, flaky: 0}, + 'e2e-on-ubuntu-latest-11.9.0': {passed: 1, failed: 0, skipped: 0, flaky: 0}, + 'e2e-on-ubuntu-latest-10.5.14': {passed: 1, failed: 0, skipped: 0, flaky: 0}, + }, + [], + ); + assert.deepEqual(rows.map((r) => r.label), [ + '10.5.14-linux', + '11.9.0-linux', + '10.5.14-windows', + ]); + }); + }); +}); diff --git a/e2e/utils/github-actions.js b/e2e/utils/github-actions.js index 987a2f6ca9b..ec74a012a17 100644 --- a/e2e/utils/github-actions.js +++ b/e2e/utils/github-actions.js @@ -2,33 +2,147 @@ // See LICENSE.txt for license information. /* eslint-disable no-console -- Logging is intentional in CI utility scripts */ -const E2E_STATUS_CONTEXT = 'e2e-test/desktop-playwright'; +/** Canonical OS identifiers for e2e/ commit statuses. */ +const E2E_OS_LIST = ['linux', 'macos', 'windows']; + +/** Platforms that run dedicated policy-test legs (PR / master only). */ +const E2E_POLICY_OS_LIST = ['macos', 'windows']; + +/** Per-OS commit status contexts for PR / master / CMT (restored from pre-TSIO merge). */ +const E2E_OS_STATUS_CONTEXTS = E2E_OS_LIST.map((os) => `e2e/${os}`); + +/** Policy commit status contexts: e2e/macos-policy, e2e/windows-policy. */ +const E2E_POLICY_STATUS_CONTEXTS = E2E_POLICY_OS_LIST.map((os) => `e2e/${os}-policy`); const E2E_WORKFLOW_NAME = 'Electron Playwright Tests'; const ACTIVE_RUN_STATUSES = ['in_progress', 'queued', 'waiting']; const CANCELLED_STATUS_DESCRIPTION = 'E2E cancelled — tests skipped'; /** - * Mark the E2E commit status as cancelled/skipped on a SHA. + * @param {string} [value] - platform / os field from matrix + * @param {string} [runner] - GitHub runner label + * @returns {'linux'|'macos'|'windows'|null} + */ +function canonicalizeOs(value, runner) { + const raw = String(value || '').toLowerCase(); + if (E2E_OS_LIST.includes(raw)) { + return raw; + } + const r = String(runner || '').toLowerCase(); + if (r.startsWith('ubuntu') || r.startsWith('linux')) { + return 'linux'; + } + if (r.startsWith('macos') || r.startsWith('darwin')) { + return 'macos'; + } + if (r.startsWith('windows')) { + return 'windows'; + } + return null; +} + +/** + * @param {string} os + * @returns {string} + */ +function osStatusContext(os) { + return `e2e/${os}`; +} + +/** + * @param {string} os - macos | windows + * @returns {string} + */ +function policyStatusContext(os) { + return `e2e/${os}-policy`; +} + +/** + * Post pending e2e/ (and optionally e2e/-policy) statuses for this run. + * + * @param {Object} params + * @param {Object} params.github + * @param {Object} params.context + * @param {string} params.sha + * @param {Array<{platform?: string, os?: string, runner?: string}>} params.platforms + * @param {boolean} [params.includePolicy] - When true (PR/master), also pending policy checks + */ +async function updateInitialOsStatuses({github, context, sha, platforms, includePolicy = false}) { + const workflowUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const seen = new Set(); + const targets = []; + + for (const platform of platforms || []) { + const os = canonicalizeOs(platform.platform || platform.os, platform.runner); + if (!os || seen.has(os)) { + continue; + } + seen.add(os); + targets.push(os); + } + + if (targets.length === 0 && !includePolicy) { + console.log('No canonical OS platforms — skipping pending e2e/ statuses'); + return; + } + + const posts = targets.map((os) => + github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha, + state: 'pending', + context: osStatusContext(os), + description: `E2E tests on ${os} have started...`, + target_url: workflowUrl, + }).catch((error) => { + console.log(`Could not set pending ${osStatusContext(os)} on ${sha}: ${error.message}`); + }), + ); + + if (includePolicy) { + for (const os of E2E_POLICY_OS_LIST) { + posts.push( + github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha, + state: 'pending', + context: policyStatusContext(os), + description: `Policy tests on ${os} have started...`, + target_url: workflowUrl, + }).catch((error) => { + console.log(`Could not set pending ${policyStatusContext(os)} on ${sha}: ${error.message}`); + }), + ); + } + } + + await Promise.all(posts); +} + +/** + * Mark the E2E commit statuses as cancelled/skipped on a SHA. * GitHub commit statuses have no "skipped" state — `error` matches mobile E2E. */ async function markE2EStatusesCancelled({github, context, sha, reason = CANCELLED_STATUS_DESCRIPTION}) { const description = String(reason).substring(0, 140); const targetUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const contexts = [...E2E_OS_STATUS_CONTEXTS, ...E2E_POLICY_STATUS_CONTEXTS]; - try { - await github.rest.repos.createCommitStatus({ + await Promise.all(contexts.map((statusContext) => + github.rest.repos.createCommitStatus({ owner: context.repo.owner, repo: context.repo.repo, sha, state: 'error', - context: E2E_STATUS_CONTEXT, + context: statusContext, description, target_url: targetUrl, - }); - } catch (error) { - console.log(`Could not update ${E2E_STATUS_CONTEXT} on ${sha}: ${error.message}`); - } + }).catch((error) => { + console.log(`Could not update ${statusContext} on ${sha}: ${error.message}`); + }), + )); } /** @@ -177,6 +291,16 @@ module.exports = { removeE2ELabel, markE2EStatusesCancelled, cancelActiveE2ERuns, - E2E_STATUS_CONTEXT, + updateInitialOsStatuses, + osStatusContext, + policyStatusContext, + canonicalizeOs, + E2E_OS_LIST, + E2E_POLICY_OS_LIST, + E2E_OS_STATUS_CONTEXTS, + E2E_POLICY_STATUS_CONTEXTS, + + // Back-compat alias for callers that still import the old singular name. + E2E_STATUS_CONTEXT: E2E_OS_STATUS_CONTEXTS[0], CANCELLED_STATUS_DESCRIPTION, }; diff --git a/e2e/utils/tsio-report-status.js b/e2e/utils/tsio-report-status.js index eaeba16bb9b..57ff8cb9c2d 100644 --- a/e2e/utils/tsio-report-status.js +++ b/e2e/utils/tsio-report-status.js @@ -23,6 +23,265 @@ function positiveInt(value, fallback) { return Number.isInteger(n) && n > 0 ? n : fallback; } +const { + parseCmtJobName, + fetchPerJobCountsFromConsolidated, + buildIndividualReportUrl, +} = require('./cmt-channel-notify'); +const { + osStatusContext, + policyStatusContext, + E2E_OS_LIST, + E2E_POLICY_OS_LIST, +} = require('./github-actions'); + +/** + * Bucket key for commit-status aggregation. + * Policy legs use `-policy` so they do not fold into e2e/. + * + * @param {{os: string, kind?: string}|null} parsed + * @returns {string|null} + */ +function statusBucketKey(parsed) { + if (!parsed || parsed.os === 'unknown') { + return null; + } + if (parsed.kind === 'policy') { + return `${parsed.os}-policy`; + } + return parsed.os; +} + +/** + * Aggregate TSIO per-job counts / shard failures by status bucket + * (linux|macos|windows|macos-policy|windows-policy). + * + * @param {Object} params + * @param {Object} params.detail - TSIO group detail + * @param {Record} params.perJobCounts + * @returns {Record} + */ +function buildOsStatusTotals({detail, perJobCounts}) { + /** @type {Record} */ + const byKey = {}; + + const ensure = (key) => { + if (!byKey[key]) { + byKey[key] = {passed: 0, failed: 0, skipped: 0, shardFailed: false, hasResults: false}; + } + return byKey[key]; + }; + + for (const [jobName, counts] of Object.entries(perJobCounts || {})) { + const key = statusBucketKey(parseCmtJobName(jobName)); + if (!key) { + continue; + } + const row = ensure(key); + row.passed += (counts.passed || 0) + (counts.flaky || 0); + row.failed += counts.failed || 0; + row.skipped += counts.skipped || 0; + row.hasResults = true; + } + + for (const report of detail?.reports || []) { + const name = report.gh_job_name || report.display_name; + const key = statusBucketKey(parseCmtJobName(name)); + if (!key) { + continue; + } + const row = ensure(key); + if (report.status === 'failed') { + row.shardFailed = true; + } + } + + return byKey; +} + +/** + * Commit-status click-through for one e2e/ (or e2e/-policy) bucket. + * PR/master has one uploaded report per bucket → /reports/r/{id}. + * CMT may have several versions on the same OS: link a failed leg if any, + * otherwise keep the group rollup so the check is not one arbitrary version. + * + * @param {Object} params + * @param {Array<{id?: string, gh_job_name?: string, display_name?: string, status?: string}>} [params.reports] + * @param {string} params.bucketKey + * @param {string} [params.baseUrl] + * @param {string} params.fallbackUrl + * @returns {string} + */ +function reportUrlForStatusBucket({reports, bucketKey, baseUrl, fallbackUrl}) { + if (!baseUrl || !bucketKey) { + return fallbackUrl; + } + + const matching = (reports || []).filter((report) => { + if (!report?.id) { + return false; + } + const name = report.gh_job_name || report.display_name; + return statusBucketKey(parseCmtJobName(name)) === bucketKey; + }); + + if (matching.length === 1) { + return buildIndividualReportUrl(baseUrl, matching[0].id); + } + + if (matching.length > 1) { + const failed = matching.find((report) => report.status === 'failed'); + if (failed) { + return buildIndividualReportUrl(baseUrl, failed.id); + } + } + + return fallbackUrl; +} + +/** + * Resolve which OS contexts this run should report. + * + * @param {string[]} [expectedOs] + * @param {Record} byKey + * @returns {string[]} + */ +function resolveExpectedOs(expectedOs, byKey) { + if (Array.isArray(expectedOs) && expectedOs.length > 0) { + return expectedOs.filter((os) => E2E_OS_LIST.includes(os)); + } + const fromResults = Object.keys(byKey || {}).filter((os) => E2E_OS_LIST.includes(os)); + return fromResults.length > 0 ? fromResults : [...E2E_OS_LIST]; +} + +/** + * Resolve which policy OS contexts this run should report. + * Only flips when explicitly expected (PR/master) — CMT has no policy legs. + * + * @param {string[]} [expectedPolicyOs] + * @returns {string[]} + */ +function resolveExpectedPolicyOs(expectedPolicyOs) { + if (!Array.isArray(expectedPolicyOs) || expectedPolicyOs.length === 0) { + return []; + } + return expectedPolicyOs.filter((os) => E2E_POLICY_OS_LIST.includes(os)); +} + +/** + * @param {Object} row + * @param {boolean} upstreamJobsSucceeded + * @param {string} incompleteLabel + * @returns {{state: string, description: string}} + */ +function statusFromTotals(row, upstreamJobsSucceeded, incompleteLabel) { + const hasFailures = row.failed > 0 || row.shardFailed; + if (hasFailures) { + return { + state: 'failure', + description: `${row.passed} passed, ${row.failed} failed, ${row.skipped} skipped`, + }; + } + if (row.hasResults) { + return { + state: 'success', + description: `${row.passed} passed, ${row.failed} failed, ${row.skipped} skipped`, + }; + } + if (upstreamJobsSucceeded) { + return { + state: 'error', + description: incompleteLabel, + }; + } + return { + state: 'failure', + description: 'CI job failed (untracked by TSIO)', + }; +} + +/** + * @param {Object} params + * @param {string} params.targetUrl - Group / fallback TSIO URL + * @param {string} [params.baseUrl] - TSIO origin used to build per-leg /reports/r/{id} links + * @param {string[]} [params.expectedOs] + * @param {string[]} [params.expectedPolicyOs] + * @returns {Promise} + */ +async function flipPerOsCommitStatuses({ + github, + context, + compositeIdentity, + detail, + perJobCounts, + targetUrl, + baseUrl, + upstreamJobsSucceeded, + expectedOs, + expectedPolicyOs, + core, +}) { + const byKey = buildOsStatusTotals({detail, perJobCounts}); + const oss = resolveExpectedOs(expectedOs, byKey); + const policyOss = resolveExpectedPolicyOs(expectedPolicyOs); + const emptyRow = {passed: 0, failed: 0, skipped: 0, shardFailed: false, hasResults: false}; + const reports = detail?.reports || []; + + const urlFor = (bucketKey) => reportUrlForStatusBucket({ + reports, + bucketKey, + baseUrl, + fallbackUrl: targetUrl, + }); + + const posts = [ + ...oss.map(async (os) => { + const row = byKey[os] || emptyRow; + const {state, description} = statusFromTotals( + row, + upstreamJobsSucceeded, + 'E2E incomplete — no results for this OS', + ); + try { + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: compositeIdentity.commit_sha, + state, + context: osStatusContext(os), + description: description.slice(0, 140), + target_url: urlFor(os), + }); + } catch (error) { + core.warning(`Failed to create ${osStatusContext(os)} status: ${error.message}`); + } + }), + ...policyOss.map(async (os) => { + const row = byKey[`${os}-policy`] || emptyRow; + const {state, description} = statusFromTotals( + row, + upstreamJobsSucceeded, + 'Policy incomplete — no results for this OS', + ); + try { + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: compositeIdentity.commit_sha, + state, + context: policyStatusContext(os), + description: description.slice(0, 140), + target_url: urlFor(`${os}-policy`), + }); + } catch (error) { + core.warning(`Failed to create ${policyStatusContext(os)} status: ${error.message}`); + } + }), + ]; + + await Promise.all(posts); +} + /** * Commit-level rollup URL: /reports/{repo}/{branch}/{shortSha}/{name} * e.g. https://test-io.test.mattermost.com/reports/desktop/tsio-spike/cff190a/desktop-pr @@ -41,14 +300,17 @@ function buildDisplayReportUrl(baseUrl, compositeIdentity) { /** * Recover a report group's id via the idempotent begin endpoint, poll the * public status endpoint until the group leaves in_progress, render a step - * summary, and flip a commit status. + * summary, and flip commit status(es). * @param {Object} params - Parameters object * @param {Object} params.core - @actions/core from actions/github-script * @param {Object} params.context - GitHub Actions context * @param {Object} params.github - GitHub API client from actions/github-script * @param {Object} params.compositeIdentity - {repository, commit_sha, gh_run_id, name, gh_run_attempt, branch, gh_pr_number} * @param {number} params.totalReportsExpected - Number of per-leg reports expected in this group - * @param {string} params.commitStatusContext - Commit-status context to flip on completion + * @param {string} [params.commitStatusContext] - Optional umbrella context (e.g. CMT) + * @param {boolean} [params.perOsCommitStatuses] - When true, also flip e2e/linux|macos|windows + * @param {string[]} [params.expectedOs] - Canonical OS list for this run (linux|macos|windows) + * @param {string[]} [params.expectedPolicyOs] - Policy OS list (macos|windows); PR/master only * @param {boolean} [params.failOnTestFailures] - When true (default), throw if the group didn't complete cleanly * @param {boolean} [params.useStaging] - Target TSIO staging instead of production * @param {string} [params.oidcAudience] - OIDC audience claim TSIO expects @@ -70,6 +332,9 @@ async function reportTsioStatus({ compositeIdentity, totalReportsExpected, commitStatusContext, + perOsCommitStatuses = false, + expectedOs, + expectedPolicyOs, failOnTestFailures = true, useStaging = false, oidcAudience = 'mattermost-test-system-io', @@ -142,18 +407,53 @@ async function reportTsioStatus({ } } catch (error) { core.error(`TSIO reporting error: ${error.message}`); - try { - await github.rest.repos.createCommitStatus({ - owner: context.repo.owner, - repo: context.repo.repo, - sha: compositeIdentity.commit_sha, - state: 'failure', - context: commitStatusContext, - description: 'TSIO reporting error — see workflow run for details', - target_url: groupReportUrl || displayReportUrl || runUrl, - }); - } catch (statusError) { - core.warning(`Failed to create failure commit status: ${statusError.message}`); + const errTarget = groupReportUrl || displayReportUrl || runUrl; + if (commitStatusContext) { + try { + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: compositeIdentity.commit_sha, + state: 'failure', + context: commitStatusContext, + description: 'TSIO reporting error — see workflow run for details', + target_url: errTarget, + }); + } catch (statusError) { + core.warning(`Failed to create failure commit status: ${statusError.message}`); + } + } + if (perOsCommitStatuses) { + const oss = resolveExpectedOs(expectedOs, {}); + const policyOss = resolveExpectedPolicyOs(expectedPolicyOs); + await Promise.all([ + ...oss.map((os) => + github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: compositeIdentity.commit_sha, + state: 'failure', + context: osStatusContext(os), + description: 'TSIO reporting error — see workflow run for details', + target_url: errTarget, + }).catch((statusError) => { + core.warning(`Failed to create ${osStatusContext(os)} failure status: ${statusError.message}`); + }), + ), + ...policyOss.map((os) => + github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: compositeIdentity.commit_sha, + state: 'failure', + context: policyStatusContext(os), + description: 'TSIO reporting error — see workflow run for details', + target_url: errTarget, + }).catch((statusError) => { + core.warning(`Failed to create ${policyStatusContext(os)} failure status: ${statusError.message}`); + }), + ), + ]); } throw error; } @@ -226,15 +526,47 @@ async function reportTsioStatus({ const descriptionPrefix = !upstreamJobsSucceeded && !hasFailures ? 'CI job failed (untracked by TSIO), ' : ''; const description = `${descriptionPrefix}${stats.passed ?? 0}/${stats.total ?? 0} passed, ${stats.failed ?? 0} failed, ${stats.skipped ?? 0} skipped`.slice(0, 140); - await github.rest.repos.createCommitStatus({ - owner: context.repo.owner, - repo: context.repo.repo, - sha: compositeIdentity.commit_sha, - state: overallState, - context: commitStatusContext, - description, - target_url: targetUrl, - }); + + if (commitStatusContext) { + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: compositeIdentity.commit_sha, + state: overallState, + context: commitStatusContext, + description, + target_url: targetUrl, + }); + } + + if (perOsCommitStatuses) { + let perJobCounts; + try { + perJobCounts = await fetchPerJobCountsFromConsolidated(baseUrl, compositeIdentity, detail); + } catch (error) { + // Do not treat a failed fetch as zero results — that would flip e2e/ + // to error/failure and clear pending. Leave statuses pending until counts exist. + core.warning( + `Could not load per-OS TSIO counts — leaving e2e/ statuses pending: ${error.message}`, + ); + perJobCounts = null; + } + if (perJobCounts) { + await flipPerOsCommitStatuses({ + github, + context, + compositeIdentity, + detail, + perJobCounts, + targetUrl, + baseUrl, + upstreamJobsSucceeded, + expectedOs, + expectedPolicyOs, + core, + }); + } + } if (failOnTestFailures && overallState === 'failure') { let reason; @@ -252,3 +584,6 @@ async function reportTsioStatus({ } module.exports = reportTsioStatus; +module.exports.buildOsStatusTotals = buildOsStatusTotals; +module.exports.flipPerOsCommitStatuses = flipPerOsCommitStatuses; +module.exports.reportUrlForStatusBucket = reportUrlForStatusBucket; diff --git a/e2e/utils/tsio-report-status.test.js b/e2e/utils/tsio-report-status.test.js new file mode 100644 index 00000000000..e52c327c4a3 --- /dev/null +++ b/e2e/utils/tsio-report-status.test.js @@ -0,0 +1,330 @@ +// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +// CI util unit tests: run with `node --test e2e/utils/tsio-report-status.test.js`. + +const {describe, it} = require('node:test'); +const assert = require('node:assert/strict'); + +const { + buildOsStatusTotals, + flipPerOsCommitStatuses, + reportUrlForStatusBucket, +} = require('./tsio-report-status'); + +describe('buildOsStatusTotals', () => { + it('groups per-job counts and shard failures by OS, keeping policy separate', () => { + const byKey = buildOsStatusTotals({ + detail: { + reports: [ + {gh_job_name: 'e2e-on-ubuntu-latest-11.9.0', status: 'complete'}, + {gh_job_name: 'e2e-on-windows-2022-11.9.0', status: 'failed'}, + {gh_job_name: 'policy-tests-macos', status: 'complete'}, + ], + }, + perJobCounts: { + 'e2e-on-ubuntu-latest-11.9.0': {passed: 100, failed: 0, skipped: 5, flaky: 1}, + 'e2e-on-windows-2022-11.9.0': {passed: 90, failed: 2, skipped: 5, flaky: 0}, + 'policy-tests-macos': {passed: 9, failed: 0, skipped: 0, flaky: 0}, + }, + }); + + assert.deepEqual(byKey.linux, { + passed: 101, + failed: 0, + skipped: 5, + shardFailed: false, + hasResults: true, + }); + assert.deepEqual(byKey.windows, { + passed: 90, + failed: 2, + skipped: 5, + shardFailed: true, + hasResults: true, + }); + assert.equal(byKey.macos, undefined); + assert.deepEqual(byKey['macos-policy'], { + passed: 9, + failed: 0, + skipped: 0, + shardFailed: false, + hasResults: true, + }); + }); + + it('marks an OS with no counts but a failed shard', () => { + const byKey = buildOsStatusTotals({ + detail: { + reports: [ + {gh_job_name: 'e2e-on-macos-26-11.10.0', status: 'failed'}, + ], + }, + perJobCounts: {}, + }); + + assert.equal(byKey.macos.shardFailed, true); + assert.equal(byKey.macos.hasResults, false); + }); +}); + +describe('flipPerOsCommitStatuses', () => { + function makeHarness() { + const statuses = []; + const github = { + rest: { + repos: { + createCommitStatus: async (opts) => { + statuses.push(opts); + }, + }, + }, + }; + const core = {warning: () => {}}; + const context = {repo: {owner: 'mattermost', repo: 'desktop'}}; + const compositeIdentity = {commit_sha: 'abc123'}; + return {statuses, github, core, context, compositeIdentity}; + } + + it('maps success / failure from per-OS counts', async () => { + const {statuses, github, core, context, compositeIdentity} = makeHarness(); + + await flipPerOsCommitStatuses({ + github, + context, + compositeIdentity, + detail: {reports: []}, + perJobCounts: { + 'e2e-on-ubuntu-latest-11.9.0': {passed: 10, failed: 0, skipped: 1, flaky: 0}, + 'e2e-on-windows-2022-11.9.0': {passed: 8, failed: 2, skipped: 0, flaky: 0}, + }, + targetUrl: 'https://example.test/report', + upstreamJobsSucceeded: true, + expectedOs: ['linux', 'windows'], + core, + }); + + const byContext = Object.fromEntries(statuses.map((s) => [s.context, s])); + assert.equal(statuses.length, 2); + assert.equal(byContext['e2e/linux'].state, 'success'); + assert.match(byContext['e2e/linux'].description, /10 passed, 0 failed/); + assert.equal(byContext['e2e/windows'].state, 'failure'); + assert.match(byContext['e2e/windows'].description, /8 passed, 2 failed/); + }); + + it('flips separate e2e/-policy contexts', async () => { + const {statuses, github, core, context, compositeIdentity} = makeHarness(); + + await flipPerOsCommitStatuses({ + github, + context, + compositeIdentity, + detail: { + reports: [ + {gh_job_name: 'policy-tests-windows', status: 'failed'}, + ], + }, + perJobCounts: { + 'policy-tests-macos': {passed: 14, failed: 0, skipped: 0, flaky: 0}, + 'policy-tests-windows': {passed: 10, failed: 1, skipped: 0, flaky: 0}, + }, + targetUrl: 'https://example.test/report', + upstreamJobsSucceeded: true, + expectedOs: ['macos'], + expectedPolicyOs: ['macos', 'windows'], + core, + }); + + const byContext = Object.fromEntries(statuses.map((s) => [s.context, s])); + assert.equal(byContext['e2e/macos'].state, 'error'); + assert.equal(byContext['e2e/macos-policy'].state, 'success'); + assert.equal(byContext['e2e/windows-policy'].state, 'failure'); + assert.equal(byContext['e2e/windows'], undefined); + }); + + it('emits error when upstream succeeded but OS has no results', async () => { + const {statuses, github, core, context, compositeIdentity} = makeHarness(); + + await flipPerOsCommitStatuses({ + github, + context, + compositeIdentity, + detail: {reports: []}, + perJobCounts: {}, + targetUrl: 'https://example.test/report', + upstreamJobsSucceeded: true, + expectedOs: ['macos'], + core, + }); + + assert.equal(statuses.length, 1); + assert.equal(statuses[0].context, 'e2e/macos'); + assert.equal(statuses[0].state, 'error'); + assert.match(statuses[0].description, /incomplete/i); + }); + + it('emits failure when upstream failed and OS has no results', async () => { + const {statuses, github, core, context, compositeIdentity} = makeHarness(); + + await flipPerOsCommitStatuses({ + github, + context, + compositeIdentity, + detail: {reports: []}, + perJobCounts: {}, + targetUrl: 'https://example.test/report', + upstreamJobsSucceeded: false, + expectedOs: ['linux'], + core, + }); + + assert.equal(statuses.length, 1); + assert.equal(statuses[0].state, 'failure'); + assert.match(statuses[0].description, /untracked by TSIO/i); + }); + + it('falls back to three OS contexts when expectedOs is empty and there are no results', async () => { + const {statuses, github, core, context, compositeIdentity} = makeHarness(); + + await flipPerOsCommitStatuses({ + github, + context, + compositeIdentity, + detail: {reports: []}, + perJobCounts: {}, + targetUrl: 'https://example.test/report', + upstreamJobsSucceeded: true, + expectedOs: [], + core, + }); + + const contexts = statuses.map((s) => s.context).sort(); + assert.deepEqual(contexts, ['e2e/linux', 'e2e/macos', 'e2e/windows']); + assert.ok(statuses.every((s) => s.state === 'error')); + }); + + it('does not flip policy contexts when expectedPolicyOs is omitted', async () => { + const {statuses, github, core, context, compositeIdentity} = makeHarness(); + + await flipPerOsCommitStatuses({ + github, + context, + compositeIdentity, + detail: {reports: []}, + perJobCounts: { + 'policy-tests-macos': {passed: 1, failed: 0, skipped: 0, flaky: 0}, + }, + targetUrl: 'https://example.test/report', + upstreamJobsSucceeded: true, + expectedOs: ['linux'], + core, + }); + + assert.deepEqual(statuses.map((s) => s.context), ['e2e/linux']); + }); + + it('points each check at its individual TSIO report, not the group URL', async () => { + const {statuses, github, core, context, compositeIdentity} = makeHarness(); + + await flipPerOsCommitStatuses({ + github, + context, + compositeIdentity, + detail: { + reports: [ + {id: 'rid-linux', gh_job_name: 'e2e-on-ubuntu-latest-11.9.0', status: 'complete'}, + {id: 'rid-mac', gh_job_name: 'e2e-on-macos-14-11.9.0', status: 'complete'}, + {id: 'rid-win', gh_job_name: 'e2e-on-windows-2022-11.9.0', status: 'complete'}, + {id: 'rid-mac-policy', gh_job_name: 'policy-tests-macos', status: 'complete'}, + {id: 'rid-win-policy', gh_job_name: 'policy-tests-windows', status: 'complete'}, + ], + }, + perJobCounts: { + 'e2e-on-ubuntu-latest-11.9.0': {passed: 219, failed: 0, skipped: 11, flaky: 0}, + 'e2e-on-macos-14-11.9.0': {passed: 225, failed: 0, skipped: 20, flaky: 0}, + 'e2e-on-windows-2022-11.9.0': {passed: 236, failed: 0, skipped: 20, flaky: 0}, + 'policy-tests-macos': {passed: 9, failed: 0, skipped: 0, flaky: 0}, + 'policy-tests-windows': {passed: 9, failed: 0, skipped: 0, flaky: 0}, + }, + targetUrl: 'https://test-io.test.mattermost.com/reports/desktop/pr/abc1234/desktop-pr', + baseUrl: 'https://test-io.test.mattermost.com', + upstreamJobsSucceeded: true, + expectedOs: ['linux', 'macos', 'windows'], + expectedPolicyOs: ['macos', 'windows'], + core, + }); + + const byContext = Object.fromEntries(statuses.map((s) => [s.context, s])); + assert.equal(byContext['e2e/linux'].target_url, 'https://test-io.test.mattermost.com/reports/r/rid-linux'); + assert.equal(byContext['e2e/macos'].target_url, 'https://test-io.test.mattermost.com/reports/r/rid-mac'); + assert.equal(byContext['e2e/windows'].target_url, 'https://test-io.test.mattermost.com/reports/r/rid-win'); + assert.equal(byContext['e2e/macos-policy'].target_url, 'https://test-io.test.mattermost.com/reports/r/rid-mac-policy'); + assert.equal(byContext['e2e/windows-policy'].target_url, 'https://test-io.test.mattermost.com/reports/r/rid-win-policy'); + }); + + it('falls back to the group URL when a bucket has no uploaded report id', async () => { + const {statuses, github, core, context, compositeIdentity} = makeHarness(); + const groupUrl = 'https://test-io.test.mattermost.com/reports/desktop/pr/abc1234/desktop-pr'; + + await flipPerOsCommitStatuses({ + github, + context, + compositeIdentity, + detail: {reports: []}, + perJobCounts: { + 'e2e-on-ubuntu-latest-11.9.0': {passed: 1, failed: 0, skipped: 0, flaky: 0}, + }, + targetUrl: groupUrl, + baseUrl: 'https://test-io.test.mattermost.com', + upstreamJobsSucceeded: true, + expectedOs: ['linux'], + core, + }); + + assert.equal(statuses[0].target_url, groupUrl); + }); +}); + +describe('reportUrlForStatusBucket', () => { + const baseUrl = 'https://test-io.test.mattermost.com'; + const fallback = 'https://test-io.test.mattermost.com/reports/desktop/pr/abc/desktop-pr'; + + it('returns the individual report URL when a bucket has one uploaded report', () => { + const url = reportUrlForStatusBucket({ + reports: [ + {id: 'rid-linux', gh_job_name: 'e2e-on-ubuntu-latest-11.9.0'}, + {id: 'rid-mac', gh_job_name: 'e2e-on-macos-14-11.9.0'}, + ], + bucketKey: 'linux', + baseUrl, + fallbackUrl: fallback, + }); + assert.equal(url, `${baseUrl}/reports/r/rid-linux`); + }); + + it('prefers a failed individual report when a bucket has multiple uploads', () => { + const url = reportUrlForStatusBucket({ + reports: [ + {id: 'rid-linux-a', gh_job_name: 'e2e-on-ubuntu-latest-11.9.0', status: 'complete'}, + {id: 'rid-linux-b', gh_job_name: 'e2e-on-ubuntu-latest-11.10.0', status: 'failed'}, + ], + bucketKey: 'linux', + baseUrl, + fallbackUrl: fallback, + }); + assert.equal(url, `${baseUrl}/reports/r/rid-linux-b`); + }); + + it('keeps the group URL when a bucket has multiple successful uploads', () => { + const url = reportUrlForStatusBucket({ + reports: [ + {id: 'rid-linux-a', gh_job_name: 'e2e-on-ubuntu-latest-11.9.0', status: 'complete'}, + {id: 'rid-linux-b', gh_job_name: 'e2e-on-ubuntu-latest-11.10.0', status: 'complete'}, + ], + bucketKey: 'linux', + baseUrl, + fallbackUrl: fallback, + }); + assert.equal(url, fallback); + }); +});