diff --git a/.github/workflows/compatibility-matrix-testing.yml b/.github/workflows/compatibility-matrix-testing.yml index ad2e0d3485f..99dac323292 100644 --- a/.github/workflows/compatibility-matrix-testing.yml +++ b/.github/workflows/compatibility-matrix-testing.yml @@ -107,6 +107,33 @@ jobs: description: "Compatibility Matrix Testing for ${{ inputs.DESKTOP_VERSION }} version" status: pending + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + continue-on-error: true + with: + ref: ${{ inputs.DESKTOP_VERSION }} + persist-credentials: false + sparse-checkout: | + e2e/utils/github-actions.js + sparse-checkout-cone-mode: false + + - name: Post pending e2e/linux|macos|windows + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + continue-on-error: true + env: + DESKTOP_SHA: ${{ needs.calculate-commit-hash.outputs.DESKTOP_SHA }} + CMT_MATRIX: ${{ needs.calculate-commit-hash.outputs.CMT_MATRIX }} + with: + script: | + const {updateInitialOsStatuses} = require('./e2e/utils/github-actions.js'); + const matrix = JSON.parse(process.env.CMT_MATRIX || '{}'); + const platforms = (matrix.environment || []).map((env) => ({platform: env.os})); + await updateInitialOsStatuses({ + github, + context, + sha: process.env.DESKTOP_SHA, + platforms, + }); + # Input follows the below schema (Matterwick ≥ v0.4.16). # { # "environment": [ @@ -178,9 +205,10 @@ jobs: sparse-checkout: | e2e/utils/tsio-report-status.js e2e/utils/cmt-channel-notify.js + e2e/utils/github-actions.js sparse-checkout-cone-mode: false - - name: Render TSIO summary + flip commit status + - name: Render TSIO summary + flip commit statuses id: summary uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: @@ -199,13 +227,23 @@ jobs: # matching failed test (hung worker teardown, crashed runner, npm ci # failure) would otherwise still read as "100% passed". UPSTREAM_JOBS_SUCCEEDED: ${{ needs.e2e.result == 'success' }} + CMT_MATRIX: ${{ needs.calculate-commit-hash.outputs.CMT_MATRIX }} with: script: | + const {canonicalizeOs} = require('./e2e/utils/github-actions.js'); + const matrix = JSON.parse(process.env.CMT_MATRIX || '{}'); + const expectedOs = [...new Set( + (matrix.environment || []). + map((env) => canonicalizeOs(env.os, env.runner)). + filter(Boolean), + )]; const {reportUrl, status, stats} = await require('./e2e/utils/tsio-report-status.js')({ core, context, github, compositeIdentity: JSON.parse(process.env.TSIO_COMPOSITE_IDENTITY), totalReportsExpected: parseInt(process.env.TSIO_TOTAL_REPORTS_EXPECTED, 10), commitStatusContext: process.env.COMMIT_STATUS_CONTEXT, + perOsCommitStatuses: true, + expectedOs, upstreamJobsSucceeded: process.env.UPSTREAM_JOBS_SUCCEEDED === 'true', failOnTestFailures: true, }); diff --git a/.github/workflows/e2e-functional.yml b/.github/workflows/e2e-functional.yml index 977f15f225e..5293c063793 100644 --- a/.github/workflows/e2e-functional.yml +++ b/.github/workflows/e2e-functional.yml @@ -58,7 +58,30 @@ jobs: run: | # Matterwick still dispatches macos-latest; pin explicitly so the job # does not drift when GitHub retargets that label to macOS 26. - platforms=$(echo "${INSTANCE_DETAILS}" | jq -c 'map(if .runner == "macos-latest" then .runner = "macos-26" else . end)') + # Normalize platform to canonical linux|macos|windows (derive from runner + # when needed) so job names and e2e/ statuses stay consistent. + platforms=$(echo "${INSTANCE_DETAILS}" | jq -c ' + map( + (if .runner == "macos-latest" then .runner = "macos-26" else . end) | + . as $row | + ($row.platform // $row.os // "") as $raw | + ( + if $raw == "linux" or $raw == "macos" or $raw == "windows" then $raw + elif (($row.runner // "") | test("^(ubuntu|linux)")) then "linux" + elif (($row.runner // "") | test("^(macos|darwin)")) then "macos" + elif (($row.runner // "") | test("^windows")) then "windows" + else null + end + ) as $canon | + if $canon == null then empty + else ($row | .platform = $canon | del(.os)) + end + ) + ') + if [ "$(echo "${platforms}" | jq 'length')" -eq 0 ]; then + echo "No supported platforms in instance_details (need linux|macos|windows)" >&2 + exit 1 + fi echo "platforms=${platforms}" >> "$GITHUB_OUTPUT" # Resolve the SHA under test from version_name (branch/tag/SHA), not github.sha @@ -98,7 +121,7 @@ jobs: echo "composite-identity-json=${COMPOSITE_IDENTITY}" >> "$GITHUB_OUTPUT" update-initial-status: - name: Set pending TSIO status + name: Set pending E2E OS statuses runs-on: ubuntu-24.04 needs: - prepare-matrix @@ -108,16 +131,31 @@ jobs: steps: # Best-effort: a transient status-API hiccup here must never block the # actual test matrix from running. - - uses: mattermost/actions/delivery/update-commit-status@218fd96a63451259dc100e1292a5e44af92fe15d + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + continue-on-error: true + with: + ref: ${{ inputs.version_name }} + persist-credentials: false + sparse-checkout: | + e2e/utils/github-actions.js + sparse-checkout-cone-mode: false + + - name: Post pending e2e/linux|macos|windows (+ policy) + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 continue-on-error: true env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PLATFORMS: ${{ needs.prepare-matrix.outputs.platforms }} + DESKTOP_SHA: ${{ needs.prepare-matrix.outputs.desktop-sha }} with: - repository_full_name: ${{ github.repository }} - commit_sha: ${{ needs.prepare-matrix.outputs.desktop-sha }} - context: e2e-test/desktop-playwright - description: "Running Electron Playwright E2E tests..." - status: pending + script: | + const {updateInitialOsStatuses} = require('./e2e/utils/github-actions.js'); + await updateInitialOsStatuses({ + github, + context, + sha: process.env.DESKTOP_SHA, + platforms: JSON.parse(process.env.PLATFORMS || '[]'), + includePolicy: true, + }); e2e-tests: needs: @@ -144,9 +182,8 @@ jobs: tsio-config: ${{ format('{{"composite_identity":{0},"total_reports_expected":"{1}"}}', needs.prepare-matrix.outputs.tsio-composite-identity, needs.prepare-matrix.outputs.tsio-total-reports-expected) }} secrets: inherit - # The single E2E status check for PR/master (e2e-test/desktop-playwright), - # replacing the former 5 (e2e/linux, e2e/macos, e2e/windows, - # policy-test/macos, policy-test/windows). + # Per-OS E2E commit statuses (e2e/linux, e2e/macos, e2e/windows) plus + # e2e/macos-policy and e2e/windows-policy for the dedicated policy legs. # # Uses e2e/utils/tsio-report-status.js, not test-system-io-summary — summary # only reads /api/v1/orchestration/status, which report-upload never @@ -172,9 +209,10 @@ jobs: sparse-checkout: | e2e/utils/tsio-report-status.js e2e/utils/cmt-channel-notify.js + e2e/utils/github-actions.js sparse-checkout-cone-mode: false - - name: Render TSIO summary + flip commit status + - name: Render TSIO summary + flip per-OS commit statuses id: summary uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: @@ -184,8 +222,8 @@ jobs: # time to attach all legs before we snapshot the group for the channel post. TSIO_POLL_ATTEMPTS: 36 TSIO_POLL_DELAY_MS: 5000 - COMMIT_STATUS_CONTEXT: e2e-test/desktop-playwright UPSTREAM_JOBS_SUCCEEDED: ${{ needs.e2e-tests.result == 'success' && needs.e2e-policy-tests.result == 'success' }} + PLATFORMS: ${{ needs.prepare-matrix.outputs.platforms }} # PR → MM_DESKTOP_E2E_WEBHOOK_URL; master → MM_E2E_MASTER_HEALTH_WEBHOOK_URL; # CMT/RC uses MM_E2E_RELEASE_WEBHOOK_URL (compatibility-matrix-testing.yml). # Routing is by compositeIdentity.name in cmt-channel-notify.js. @@ -193,12 +231,19 @@ jobs: MATTERMOST_MASTER_HEALTH_WEBHOOK_URL: ${{ secrets.MM_E2E_MASTER_HEALTH_WEBHOOK_URL }} with: script: | + const {canonicalizeOs, E2E_POLICY_OS_LIST} = require('./e2e/utils/github-actions.js'); + const platforms = JSON.parse(process.env.PLATFORMS || '[]'); + const expectedOs = [...new Set( + platforms.map((p) => canonicalizeOs(p.platform || p.os, p.runner)).filter(Boolean), + )]; const {reportUrl, status, stats} = await require('./e2e/utils/tsio-report-status.js')({ core, context, github, compositeIdentity: JSON.parse(process.env.TSIO_COMPOSITE_IDENTITY), totalReportsExpected: parseInt(process.env.TSIO_TOTAL_REPORTS_EXPECTED, 10), upstreamJobsSucceeded: process.env.UPSTREAM_JOBS_SUCCEEDED === 'true', - commitStatusContext: process.env.COMMIT_STATUS_CONTEXT, + perOsCommitStatuses: true, + expectedOs, + expectedPolicyOs: E2E_POLICY_OS_LIST, failOnTestFailures: true, }); core.info(`TSIO report ${reportUrl}: ${status} (${JSON.stringify(stats)})`); diff --git a/e2e/utils/cmt-channel-notify.js b/e2e/utils/cmt-channel-notify.js index 12e10265209..27d78dab42d 100644 --- a/e2e/utils/cmt-channel-notify.js +++ b/e2e/utils/cmt-channel-notify.js @@ -11,6 +11,9 @@ * * Per-leg pass/fail counts come from TSIO consolidated specs grouped by * contributing report id → gh_job_name (group report only has upload status). + * + * Retry attempts are collapsed to one outcome per (spec × job), matching + * Playwright: failed+passed → flaky; all failed → failed once (not once per attempt). */ const OS_ORDER = {linux: 0, macos: 1, windows: 2}; @@ -272,6 +275,51 @@ function formatMetaLine(compositeIdentity) { return parts.join(' · '); } +/** + * Prefer unique per-leg counts (retry-collapsed) over TSIO group test_stats, which + * counts every failed attempt and double-counts retries in channel alerts. + * + * When `expectedJobNames` is provided, per-leg totals are used only if every expected + * job is present — otherwise fall back to `stats` so partial consolidation cannot hide + * aggregate failures. + * + * @param {Record} perJobCounts + * @param {{passed?: number, failed?: number, skipped?: number, flaky?: number}} stats + * @param {string[]} [expectedJobNames] - uploaded report job names; omit to trust any per-leg map + * @returns {{passed: number, failed: number, skipped: number}} + */ +function resolveChannelTotals(perJobCounts, stats = {}, expectedJobNames) { + const counts = perJobCounts || {}; + + let jobs; + if (expectedJobNames === undefined) { + jobs = Object.values(counts); + } else if (expectedJobNames.length > 0 && + expectedJobNames.every((job) => Object.prototype.hasOwnProperty.call(counts, job))) { + jobs = expectedJobNames.map((job) => counts[job]); + } else { + jobs = []; + } + + if (jobs.length > 0) { + let passed = 0; + let failed = 0; + let skipped = 0; + for (const jobCounts of jobs) { + passed += (jobCounts.passed || 0) + (jobCounts.flaky || 0); + failed += jobCounts.failed || 0; + skipped += jobCounts.skipped || 0; + } + return {passed, failed, skipped}; + } + + return { + passed: (stats.passed ?? 0) + (stats.flaky ?? 0), + failed: stats.failed ?? 0, + skipped: stats.skipped ?? 0, + }; +} + /** * @param {Object} params * @param {Object} params.compositeIdentity @@ -292,19 +340,22 @@ function formatCmtChannelMessage({ upstreamJobsSucceeded = true, hasFailures = false, }) { - const stats = detail?.test_stats || {}; - - // Match buildLegSummaries: fold flaky into passed so the headline matches per-leg totals. - const passed = (stats.passed ?? 0) + (stats.flaky ?? 0); - const failed = stats.failed ?? 0; - const skipped = stats.skipped ?? 0; + const reports = detail?.reports || []; + const legs = buildLegSummaries(perJobCounts, reports, baseUrl); + const expectedJobNames = [...new Set( + reports.map((r) => r.gh_job_name || r.display_name).filter(Boolean), + )]; + const {passed, failed, skipped} = resolveChannelTotals( + perJobCounts, + detail?.test_stats, + expectedJobNames, + ); // Overall pass/fail follows tests + upstream CI — not TSIO consolidation state. // Stuck `in_progress` / `incomplete` with 0 failures must not render as ❌ Failed. const overallFailed = failed > 0 || !upstreamJobsSucceeded || hasFailures; const tsioPending = Boolean(detail?.status && detail.status !== 'completed'); const title = reportTitleForIdentity(compositeIdentity); - const legs = buildLegSummaries(perJobCounts, detail?.reports || [], baseUrl); const missingLegs = legs.filter((leg) => leg.status === 'missing' || leg.status === 'no-results'); const lines = [ @@ -379,6 +430,54 @@ function formatCmtChannelMessage({ return lines.join('\n').trimEnd() + '\n'; } +/** + * Collapse TSIO history attempts for one spec on one job into a single Playwright-style status. + * failed then passed (or explicit flaky) → flaky; all failed → failed once. + * + * @param {Array<{status?: string}>} entries + * @returns {'passed'|'failed'|'skipped'|'flaky'|null} + */ +function collapseSpecAttempts(entries) { + if (!entries || entries.length === 0) { + return null; + } + + let sawPassed = false; + let sawFailed = false; + let sawSkipped = false; + let sawFlaky = false; + + for (const entry of entries) { + const status = entry.status || 'failed'; + if (status === 'passed') { + sawPassed = true; + } else if (status === 'failed') { + sawFailed = true; + } else if (status === 'skipped') { + sawSkipped = true; + } else if (status === 'flaky') { + sawFlaky = true; + } else { + // Unknown statuses treated as failures so they still surface in alerts. + sawFailed = true; + } + } + + if (sawFlaky || (sawFailed && sawPassed)) { + return 'flaky'; + } + if (sawFailed) { + return 'failed'; + } + if (sawPassed) { + return 'passed'; + } + if (sawSkipped) { + return 'skipped'; + } + return null; +} + /** * @param {string} baseUrl * @param {Object} compositeIdentity @@ -422,6 +521,8 @@ async function fetchPerJobCountsFromConsolidated(baseUrl, compositeIdentity, gro const attempt = Number.parseInt(compositeIdentity.gh_run_attempt || '1', 10); for (const spec of consol.specs || []) { + /** @type {Record>} */ + const entriesByJob = {}; for (const entry of spec.history || []) { if (entry.commit_sha !== commitSha) { continue; @@ -433,15 +534,21 @@ async function fetchPerJobCountsFromConsolidated(baseUrl, compositeIdentity, gro if (!job) { continue; } + if (!entriesByJob[job]) { + entriesByJob[job] = []; + } + entriesByJob[job].push(entry); + } + + for (const [job, entries] of Object.entries(entriesByJob)) { + const status = collapseSpecAttempts(entries); + if (!status) { + continue; + } if (!counts[job]) { counts[job] = {passed: 0, failed: 0, skipped: 0, flaky: 0}; } - const status = entry.status || 'failed'; - if (Object.prototype.hasOwnProperty.call(counts[job], status)) { - counts[job][status] += 1; - } else { - counts[job].failed += 1; - } + counts[job][status] += 1; } } @@ -541,6 +648,8 @@ module.exports = { buildLegSummaries, formatLegResultText, reportTitleForIdentity, + resolveChannelTotals, + collapseSpecAttempts, formatCmtChannelMessage, fetchPerJobCountsFromConsolidated, postMattermostWebhook, diff --git a/e2e/utils/cmt-channel-notify.test.js b/e2e/utils/cmt-channel-notify.test.js index 82d33c251e0..089a230a636 100644 --- a/e2e/utils/cmt-channel-notify.test.js +++ b/e2e/utils/cmt-channel-notify.test.js @@ -13,9 +13,69 @@ const { buildLegSummaries, formatLegResultText, formatCmtChannelMessage, + collapseSpecAttempts, + resolveChannelTotals, } = require('./cmt-channel-notify'); describe('cmt-channel-notify', () => { + describe('collapseSpecAttempts', () => { + it('counts a double-failed retry as one failure', () => { + assert.equal( + collapseSpecAttempts([{status: 'failed'}, {status: 'failed'}]), + 'failed', + ); + }); + + it('treats failed-then-passed as flaky', () => { + assert.equal( + collapseSpecAttempts([{status: 'failed'}, {status: 'passed'}]), + 'flaky', + ); + }); + + it('keeps explicit flaky and single pass/skip', () => { + assert.equal(collapseSpecAttempts([{status: 'flaky'}]), 'flaky'); + assert.equal(collapseSpecAttempts([{status: 'passed'}]), 'passed'); + assert.equal(collapseSpecAttempts([{status: 'skipped'}]), 'skipped'); + assert.equal(collapseSpecAttempts([]), null); + }); + }); + + describe('resolveChannelTotals', () => { + it('sums unique per-leg counts and folds flaky into passed', () => { + assert.deepEqual( + resolveChannelTotals({ + 'e2e-on-ubuntu-latest-11.9.0': {passed: 218, failed: 1, skipped: 11, flaky: 0}, + 'e2e-on-windows-2022-10.11.23': {passed: 214, failed: 2, skipped: 34, flaky: 1}, + }), + {passed: 433, failed: 3, skipped: 45}, + ); + }); + + it('falls back to test_stats when per-leg counts are empty', () => { + assert.deepEqual( + resolveChannelTotals({}, {passed: 200, failed: 14, skipped: 5, flaky: 3}), + {passed: 203, failed: 14, skipped: 5}, + ); + }); + + it('falls back to aggregate stats when an expected leg is missing from per-job counts', () => { + assert.deepEqual( + resolveChannelTotals( + { + 'e2e-on-ubuntu-latest-11.9.0': {passed: 218, failed: 0, skipped: 11, flaky: 0}, + }, + {passed: 400, failed: 5, skipped: 20, flaky: 0}, + [ + 'e2e-on-ubuntu-latest-11.9.0', + 'e2e-on-windows-2022-11.9.0', + ], + ), + {passed: 400, failed: 5, skipped: 20}, + ); + }); + }); + describe('formatLegResultText', () => { it('renders all-skipped legs as not executed instead of ✅ 0/0', () => { assert.equal( @@ -159,7 +219,9 @@ describe('cmt-channel-notify', () => { assert.match(text, /\*\*Branch:\*\* `v6\.2\.0-rc\.1` · \*\*Commit:\*\* `55afc0b`/); assert.match(text, /🔴 \*\*1 failing test\*\*/); assert.match(text, /\| 🪟 Windows \| Server `11\.9\.0` \| 1 \|/); - assert.match(text, /\| ❌ Failed \| \*\*460\*\* \| \*\*1\*\* \| \*\*40\*\* \|/); + + // Overall totals come from unique per-leg counts (not inflated TSIO test_stats). + assert.match(text, /\| ❌ Failed \| \*\*461\*\* \| \*\*1\*\* \| \*\*40\*\* \|/); assert.match(text, /#### Detailed results/); assert.doesNotMatch(text, /
/); assert.match(text, /\| 🐧 Linux \| Server `11\.9\.0` \| ✅ 231\/231 \| \[View\]\(https:\/\/test-io\.test\.mattermost\.com\/reports\/r\/rid-linux\) \|/); @@ -167,6 +229,39 @@ describe('cmt-channel-notify', () => { assert.match(text, /➡️ \*\*Full report:\*\* https:\/\/test-io\.test\.mattermost\.com\/reports\/desktop\/v6\.2\.0-rc\.1\/55afc0b\/cmt-desktop/); }); + it('prefers unique per-leg failed counts over inflated TSIO attempt totals', () => { + const text = formatCmtChannelMessage({ + compositeIdentity: { + branch: 'master', + commit_sha: '5eda917312db037975bac5c3535272c61a664674', + name: 'cmt-desktop', + }, + detail: { + status: 'completed', + + // Inflated: each retry attempt counted (e.g. 1 unique fail × 2 attempts). + test_stats: {passed: 218, failed: 2, skipped: 11, total: 231}, + reports: [ + {id: 'rid-linux', gh_job_name: 'e2e-on-ubuntu-latest-11.9.1', status: 'complete'}, + ], + }, + reportUrl: 'https://test-io.test.mattermost.com/reports/desktop/master/5eda917/cmt-desktop', + baseUrl: 'https://test-io.test.mattermost.com', + perJobCounts: { + + // Unique Playwright-style count after collapsing retries. + 'e2e-on-ubuntu-latest-11.9.1': {passed: 218, failed: 1, skipped: 11, flaky: 0}, + }, + upstreamJobsSucceeded: true, + }); + + assert.match(text, /🔴 \*\*1 failing test\*\*/); + assert.match(text, /\| 🐧 Linux \| Server `11\.9\.1` \| 1 \|/); + assert.match(text, /\| ❌ Failed \| \*\*218\*\* \| \*\*1\*\* \| \*\*11\*\* \|/); + assert.match(text, /\| 🐧 Linux \| Server `11\.9\.1` \| ❌ 218\/219 \|/); + assert.doesNotMatch(text, /🔴 \*\*2 failing tests\*\*/); + }); + it('renders a passed PR report with PR link and no failure banner', () => { const text = formatCmtChannelMessage({ compositeIdentity: { @@ -247,7 +342,9 @@ describe('cmt-channel-notify', () => { upstreamJobsSucceeded: true, }); assert.match(text, /^## ✅ Desktop PR E2E\n/); - assert.match(text, /\| ✅ Passed \| \*\*674\*\* \| \*\*0\*\* \| \*\*51\*\* \|/); + + // Unique per-leg sum: 216+220+237+9 = 682 passed, 30 skipped (not TSIO test_stats 674/51). + assert.match(text, /\| ✅ Passed \| \*\*682\*\* \| \*\*0\*\* \| \*\*30\*\* \|/); assert.match(text, /TSIO report status: `in_progress` \(consolidation still catching up; not treated as a test failure\)/); assert.match(text, /Missing or empty leg report\(s\): 🍎 macOS \/ Policy/); assert.match(text, /\| 🍎 macOS \| Policy \| ⚠️ missing \|/); diff --git a/e2e/utils/github-actions.js b/e2e/utils/github-actions.js index 987a2f6ca9b..ec74a012a17 100644 --- a/e2e/utils/github-actions.js +++ b/e2e/utils/github-actions.js @@ -2,33 +2,147 @@ // See LICENSE.txt for license information. /* eslint-disable no-console -- Logging is intentional in CI utility scripts */ -const E2E_STATUS_CONTEXT = 'e2e-test/desktop-playwright'; +/** Canonical OS identifiers for e2e/ commit statuses. */ +const E2E_OS_LIST = ['linux', 'macos', 'windows']; + +/** Platforms that run dedicated policy-test legs (PR / master only). */ +const E2E_POLICY_OS_LIST = ['macos', 'windows']; + +/** Per-OS commit status contexts for PR / master / CMT (restored from pre-TSIO merge). */ +const E2E_OS_STATUS_CONTEXTS = E2E_OS_LIST.map((os) => `e2e/${os}`); + +/** Policy commit status contexts: e2e/macos-policy, e2e/windows-policy. */ +const E2E_POLICY_STATUS_CONTEXTS = E2E_POLICY_OS_LIST.map((os) => `e2e/${os}-policy`); const E2E_WORKFLOW_NAME = 'Electron Playwright Tests'; const ACTIVE_RUN_STATUSES = ['in_progress', 'queued', 'waiting']; const CANCELLED_STATUS_DESCRIPTION = 'E2E cancelled — tests skipped'; /** - * Mark the E2E commit status as cancelled/skipped on a SHA. + * @param {string} [value] - platform / os field from matrix + * @param {string} [runner] - GitHub runner label + * @returns {'linux'|'macos'|'windows'|null} + */ +function canonicalizeOs(value, runner) { + const raw = String(value || '').toLowerCase(); + if (E2E_OS_LIST.includes(raw)) { + return raw; + } + const r = String(runner || '').toLowerCase(); + if (r.startsWith('ubuntu') || r.startsWith('linux')) { + return 'linux'; + } + if (r.startsWith('macos') || r.startsWith('darwin')) { + return 'macos'; + } + if (r.startsWith('windows')) { + return 'windows'; + } + return null; +} + +/** + * @param {string} os + * @returns {string} + */ +function osStatusContext(os) { + return `e2e/${os}`; +} + +/** + * @param {string} os - macos | windows + * @returns {string} + */ +function policyStatusContext(os) { + return `e2e/${os}-policy`; +} + +/** + * Post pending e2e/ (and optionally e2e/-policy) statuses for this run. + * + * @param {Object} params + * @param {Object} params.github + * @param {Object} params.context + * @param {string} params.sha + * @param {Array<{platform?: string, os?: string, runner?: string}>} params.platforms + * @param {boolean} [params.includePolicy] - When true (PR/master), also pending policy checks + */ +async function updateInitialOsStatuses({github, context, sha, platforms, includePolicy = false}) { + const workflowUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const seen = new Set(); + const targets = []; + + for (const platform of platforms || []) { + const os = canonicalizeOs(platform.platform || platform.os, platform.runner); + if (!os || seen.has(os)) { + continue; + } + seen.add(os); + targets.push(os); + } + + if (targets.length === 0 && !includePolicy) { + console.log('No canonical OS platforms — skipping pending e2e/ statuses'); + return; + } + + const posts = targets.map((os) => + github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha, + state: 'pending', + context: osStatusContext(os), + description: `E2E tests on ${os} have started...`, + target_url: workflowUrl, + }).catch((error) => { + console.log(`Could not set pending ${osStatusContext(os)} on ${sha}: ${error.message}`); + }), + ); + + if (includePolicy) { + for (const os of E2E_POLICY_OS_LIST) { + posts.push( + github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha, + state: 'pending', + context: policyStatusContext(os), + description: `Policy tests on ${os} have started...`, + target_url: workflowUrl, + }).catch((error) => { + console.log(`Could not set pending ${policyStatusContext(os)} on ${sha}: ${error.message}`); + }), + ); + } + } + + await Promise.all(posts); +} + +/** + * Mark the E2E commit statuses as cancelled/skipped on a SHA. * GitHub commit statuses have no "skipped" state — `error` matches mobile E2E. */ async function markE2EStatusesCancelled({github, context, sha, reason = CANCELLED_STATUS_DESCRIPTION}) { const description = String(reason).substring(0, 140); const targetUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const contexts = [...E2E_OS_STATUS_CONTEXTS, ...E2E_POLICY_STATUS_CONTEXTS]; - try { - await github.rest.repos.createCommitStatus({ + await Promise.all(contexts.map((statusContext) => + github.rest.repos.createCommitStatus({ owner: context.repo.owner, repo: context.repo.repo, sha, state: 'error', - context: E2E_STATUS_CONTEXT, + context: statusContext, description, target_url: targetUrl, - }); - } catch (error) { - console.log(`Could not update ${E2E_STATUS_CONTEXT} on ${sha}: ${error.message}`); - } + }).catch((error) => { + console.log(`Could not update ${statusContext} on ${sha}: ${error.message}`); + }), + )); } /** @@ -177,6 +291,16 @@ module.exports = { removeE2ELabel, markE2EStatusesCancelled, cancelActiveE2ERuns, - E2E_STATUS_CONTEXT, + updateInitialOsStatuses, + osStatusContext, + policyStatusContext, + canonicalizeOs, + E2E_OS_LIST, + E2E_POLICY_OS_LIST, + E2E_OS_STATUS_CONTEXTS, + E2E_POLICY_STATUS_CONTEXTS, + + // Back-compat alias for callers that still import the old singular name. + E2E_STATUS_CONTEXT: E2E_OS_STATUS_CONTEXTS[0], CANCELLED_STATUS_DESCRIPTION, }; diff --git a/e2e/utils/tsio-report-status.js b/e2e/utils/tsio-report-status.js index 48e6b73e378..57fe4b299ff 100644 --- a/e2e/utils/tsio-report-status.js +++ b/e2e/utils/tsio-report-status.js @@ -23,6 +23,265 @@ function positiveInt(value, fallback) { return Number.isInteger(n) && n > 0 ? n : fallback; } +const { + parseCmtJobName, + fetchPerJobCountsFromConsolidated, + buildIndividualReportUrl, +} = require('./cmt-channel-notify'); +const { + osStatusContext, + policyStatusContext, + E2E_OS_LIST, + E2E_POLICY_OS_LIST, +} = require('./github-actions'); + +/** + * Bucket key for commit-status aggregation. + * Policy legs use `-policy` so they do not fold into e2e/. + * + * @param {{os: string, kind?: string}|null} parsed + * @returns {string|null} + */ +function statusBucketKey(parsed) { + if (!parsed || parsed.os === 'unknown') { + return null; + } + if (parsed.kind === 'policy') { + return `${parsed.os}-policy`; + } + return parsed.os; +} + +/** + * Aggregate TSIO per-job counts / shard failures by status bucket + * (linux|macos|windows|macos-policy|windows-policy). + * + * @param {Object} params + * @param {Object} params.detail - TSIO group detail + * @param {Record} params.perJobCounts + * @returns {Record} + */ +function buildOsStatusTotals({detail, perJobCounts}) { + /** @type {Record} */ + const byKey = {}; + + const ensure = (key) => { + if (!byKey[key]) { + byKey[key] = {passed: 0, failed: 0, skipped: 0, shardFailed: false, hasResults: false}; + } + return byKey[key]; + }; + + for (const [jobName, counts] of Object.entries(perJobCounts || {})) { + const key = statusBucketKey(parseCmtJobName(jobName)); + if (!key) { + continue; + } + const row = ensure(key); + row.passed += (counts.passed || 0) + (counts.flaky || 0); + row.failed += counts.failed || 0; + row.skipped += counts.skipped || 0; + row.hasResults = true; + } + + for (const report of detail?.reports || []) { + const name = report.gh_job_name || report.display_name; + const key = statusBucketKey(parseCmtJobName(name)); + if (!key) { + continue; + } + const row = ensure(key); + if (report.status === 'failed') { + row.shardFailed = true; + } + } + + return byKey; +} + +/** + * Commit-status click-through for one e2e/ (or e2e/-policy) bucket. + * PR/master has one uploaded report per bucket → /reports/r/{id}. + * CMT may have several versions on the same OS: link a failed leg if any, + * otherwise keep the group rollup so the check is not one arbitrary version. + * + * @param {Object} params + * @param {Array<{id?: string, gh_job_name?: string, display_name?: string, status?: string}>} [params.reports] + * @param {string} params.bucketKey + * @param {string} [params.baseUrl] + * @param {string} params.fallbackUrl + * @returns {string} + */ +function reportUrlForStatusBucket({reports, bucketKey, baseUrl, fallbackUrl}) { + if (!baseUrl || !bucketKey) { + return fallbackUrl; + } + + const matching = (reports || []).filter((report) => { + if (!report?.id) { + return false; + } + const name = report.gh_job_name || report.display_name; + return statusBucketKey(parseCmtJobName(name)) === bucketKey; + }); + + if (matching.length === 1) { + return buildIndividualReportUrl(baseUrl, matching[0].id); + } + + if (matching.length > 1) { + const failed = matching.find((report) => report.status === 'failed'); + if (failed) { + return buildIndividualReportUrl(baseUrl, failed.id); + } + } + + return fallbackUrl; +} + +/** + * Resolve which OS contexts this run should report. + * + * @param {string[]} [expectedOs] + * @param {Record} byKey + * @returns {string[]} + */ +function resolveExpectedOs(expectedOs, byKey) { + if (Array.isArray(expectedOs) && expectedOs.length > 0) { + return expectedOs.filter((os) => E2E_OS_LIST.includes(os)); + } + const fromResults = Object.keys(byKey || {}).filter((os) => E2E_OS_LIST.includes(os)); + return fromResults.length > 0 ? fromResults : [...E2E_OS_LIST]; +} + +/** + * Resolve which policy OS contexts this run should report. + * Only flips when explicitly expected (PR/master) — CMT has no policy legs. + * + * @param {string[]} [expectedPolicyOs] + * @returns {string[]} + */ +function resolveExpectedPolicyOs(expectedPolicyOs) { + if (!Array.isArray(expectedPolicyOs) || expectedPolicyOs.length === 0) { + return []; + } + return expectedPolicyOs.filter((os) => E2E_POLICY_OS_LIST.includes(os)); +} + +/** + * @param {Object} row + * @param {boolean} upstreamJobsSucceeded + * @param {string} incompleteLabel + * @returns {{state: string, description: string}} + */ +function statusFromTotals(row, upstreamJobsSucceeded, incompleteLabel) { + const hasFailures = row.failed > 0 || row.shardFailed; + if (hasFailures) { + return { + state: 'failure', + description: `${row.passed} passed, ${row.failed} failed, ${row.skipped} skipped`, + }; + } + if (row.hasResults) { + return { + state: 'success', + description: `${row.passed} passed, ${row.failed} failed, ${row.skipped} skipped`, + }; + } + if (upstreamJobsSucceeded) { + return { + state: 'error', + description: incompleteLabel, + }; + } + return { + state: 'failure', + description: 'CI job failed (untracked by TSIO)', + }; +} + +/** + * @param {Object} params + * @param {string} params.targetUrl - Group / fallback TSIO URL + * @param {string} [params.baseUrl] - TSIO origin used to build per-leg /reports/r/{id} links + * @param {string[]} [params.expectedOs] + * @param {string[]} [params.expectedPolicyOs] + * @returns {Promise} + */ +async function flipPerOsCommitStatuses({ + github, + context, + compositeIdentity, + detail, + perJobCounts, + targetUrl, + baseUrl, + upstreamJobsSucceeded, + expectedOs, + expectedPolicyOs, + core, +}) { + const byKey = buildOsStatusTotals({detail, perJobCounts}); + const oss = resolveExpectedOs(expectedOs, byKey); + const policyOss = resolveExpectedPolicyOs(expectedPolicyOs); + const emptyRow = {passed: 0, failed: 0, skipped: 0, shardFailed: false, hasResults: false}; + const reports = detail?.reports || []; + + const urlFor = (bucketKey) => reportUrlForStatusBucket({ + reports, + bucketKey, + baseUrl, + fallbackUrl: targetUrl, + }); + + const posts = [ + ...oss.map(async (os) => { + const row = byKey[os] || emptyRow; + const {state, description} = statusFromTotals( + row, + upstreamJobsSucceeded, + 'E2E incomplete — no results for this OS', + ); + try { + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: compositeIdentity.commit_sha, + state, + context: osStatusContext(os), + description: description.slice(0, 140), + target_url: urlFor(os), + }); + } catch (error) { + core.warning(`Failed to create ${osStatusContext(os)} status: ${error.message}`); + } + }), + ...policyOss.map(async (os) => { + const row = byKey[`${os}-policy`] || emptyRow; + const {state, description} = statusFromTotals( + row, + upstreamJobsSucceeded, + 'Policy incomplete — no results for this OS', + ); + try { + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: compositeIdentity.commit_sha, + state, + context: policyStatusContext(os), + description: description.slice(0, 140), + target_url: urlFor(`${os}-policy`), + }); + } catch (error) { + core.warning(`Failed to create ${policyStatusContext(os)} status: ${error.message}`); + } + }), + ]; + + await Promise.all(posts); +} + /** * Commit-level rollup URL: /reports/{repo}/{branch}/{shortSha}/{name} * e.g. https://test-io.test.mattermost.com/reports/desktop/tsio-spike/cff190a/desktop-pr @@ -41,14 +300,17 @@ function buildDisplayReportUrl(baseUrl, compositeIdentity) { /** * Recover a report group's id via the idempotent begin endpoint, poll the * public status endpoint until the group leaves in_progress, render a step - * summary, and flip a commit status. + * summary, and flip commit status(es). * @param {Object} params - Parameters object * @param {Object} params.core - @actions/core from actions/github-script * @param {Object} params.context - GitHub Actions context * @param {Object} params.github - GitHub API client from actions/github-script * @param {Object} params.compositeIdentity - {repository, commit_sha, gh_run_id, name, gh_run_attempt, branch, gh_pr_number} * @param {number} params.totalReportsExpected - Number of per-leg reports expected in this group - * @param {string} params.commitStatusContext - Commit-status context to flip on completion + * @param {string} [params.commitStatusContext] - Optional umbrella context (e.g. CMT) + * @param {boolean} [params.perOsCommitStatuses] - When true, also flip e2e/linux|macos|windows + * @param {string[]} [params.expectedOs] - Canonical OS list for this run (linux|macos|windows) + * @param {string[]} [params.expectedPolicyOs] - Policy OS list (macos|windows); PR/master only * @param {boolean} [params.failOnTestFailures] - When true (default), throw if tests/shards/upstream CI failed (not merely TSIO still consolidating) * @param {boolean} [params.useStaging] - Target TSIO staging instead of production * @param {string} [params.oidcAudience] - OIDC audience claim TSIO expects @@ -70,6 +332,9 @@ async function reportTsioStatus({ compositeIdentity, totalReportsExpected, commitStatusContext, + perOsCommitStatuses = false, + expectedOs, + expectedPolicyOs, failOnTestFailures = true, useStaging = false, oidcAudience = 'mattermost-test-system-io', @@ -164,18 +429,53 @@ async function reportTsioStatus({ } } catch (error) { core.error(`TSIO reporting error: ${error.message}`); - try { - await github.rest.repos.createCommitStatus({ - owner: context.repo.owner, - repo: context.repo.repo, - sha: compositeIdentity.commit_sha, - state: 'failure', - context: commitStatusContext, - description: 'TSIO reporting error — see workflow run for details', - target_url: groupReportUrl || displayReportUrl || runUrl, - }); - } catch (statusError) { - core.warning(`Failed to create failure commit status: ${statusError.message}`); + const errTarget = groupReportUrl || displayReportUrl || runUrl; + if (commitStatusContext) { + try { + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: compositeIdentity.commit_sha, + state: 'failure', + context: commitStatusContext, + description: 'TSIO reporting error — see workflow run for details', + target_url: errTarget, + }); + } catch (statusError) { + core.warning(`Failed to create failure commit status: ${statusError.message}`); + } + } + if (perOsCommitStatuses) { + const oss = resolveExpectedOs(expectedOs, {}); + const policyOss = resolveExpectedPolicyOs(expectedPolicyOs); + await Promise.all([ + ...oss.map((os) => + github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: compositeIdentity.commit_sha, + state: 'failure', + context: osStatusContext(os), + description: 'TSIO reporting error — see workflow run for details', + target_url: errTarget, + }).catch((statusError) => { + core.warning(`Failed to create ${osStatusContext(os)} failure status: ${statusError.message}`); + }), + ), + ...policyOss.map((os) => + github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: compositeIdentity.commit_sha, + state: 'failure', + context: policyStatusContext(os), + description: 'TSIO reporting error — see workflow run for details', + target_url: errTarget, + }).catch((statusError) => { + core.warning(`Failed to create ${policyStatusContext(os)} failure status: ${statusError.message}`); + }), + ), + ]); } throw error; } @@ -200,7 +500,7 @@ async function reportTsioStatus({ // Commit status / job outcome must reflect test + upstream CI health — not TSIO // consolidation lag. A stuck `in_progress` / `incomplete` group with 0 failed tests - // previously flipped e2e-test/desktop-playwright red and posted "Failed" with 0 failures. + // previously flipped the commit status red and posted "Failed" with 0 failures. let overallState = 'failure'; if (!hasFailures && upstreamJobsSucceeded) { overallState = 'success'; @@ -256,15 +556,47 @@ async function reportTsioStatus({ const descriptionPrefix = !upstreamJobsSucceeded && !hasFailures ? 'CI job failed (untracked by TSIO), ' : ''; const tsioLagSuffix = !isComplete && overallState === 'success' ? ' (TSIO consolidating)' : ''; const description = `${descriptionPrefix}${passedForStatus}/${stats.total ?? 0} passed, ${stats.failed ?? 0} failed, ${stats.skipped ?? 0} skipped${tsioLagSuffix}`.slice(0, 140); - await github.rest.repos.createCommitStatus({ - owner: context.repo.owner, - repo: context.repo.repo, - sha: compositeIdentity.commit_sha, - state: overallState, - context: commitStatusContext, - description, - target_url: targetUrl, - }); + + if (commitStatusContext) { + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: compositeIdentity.commit_sha, + state: overallState, + context: commitStatusContext, + description, + target_url: targetUrl, + }); + } + + if (perOsCommitStatuses) { + let perJobCounts; + try { + perJobCounts = await fetchPerJobCountsFromConsolidated(baseUrl, compositeIdentity, detail); + } catch (error) { + // Do not treat a failed fetch as zero results — that would flip e2e/ + // to error/failure and clear pending. Leave statuses pending until counts exist. + core.warning( + `Could not load per-OS TSIO counts — leaving e2e/ statuses pending: ${error.message}`, + ); + perJobCounts = null; + } + if (perJobCounts) { + await flipPerOsCommitStatuses({ + github, + context, + compositeIdentity, + detail, + perJobCounts, + targetUrl, + baseUrl, + upstreamJobsSucceeded, + expectedOs, + expectedPolicyOs, + core, + }); + } + } // Channel notify (best-effort). Routing (see resolveWebhookUrl): // cmt-desktop → MM_E2E_RELEASE_WEBHOOK_URL @@ -319,3 +651,6 @@ async function reportTsioStatus({ } module.exports = reportTsioStatus; +module.exports.buildOsStatusTotals = buildOsStatusTotals; +module.exports.flipPerOsCommitStatuses = flipPerOsCommitStatuses; +module.exports.reportUrlForStatusBucket = reportUrlForStatusBucket; diff --git a/e2e/utils/tsio-report-status.test.js b/e2e/utils/tsio-report-status.test.js new file mode 100644 index 00000000000..e52c327c4a3 --- /dev/null +++ b/e2e/utils/tsio-report-status.test.js @@ -0,0 +1,330 @@ +// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +// CI util unit tests: run with `node --test e2e/utils/tsio-report-status.test.js`. + +const {describe, it} = require('node:test'); +const assert = require('node:assert/strict'); + +const { + buildOsStatusTotals, + flipPerOsCommitStatuses, + reportUrlForStatusBucket, +} = require('./tsio-report-status'); + +describe('buildOsStatusTotals', () => { + it('groups per-job counts and shard failures by OS, keeping policy separate', () => { + const byKey = buildOsStatusTotals({ + detail: { + reports: [ + {gh_job_name: 'e2e-on-ubuntu-latest-11.9.0', status: 'complete'}, + {gh_job_name: 'e2e-on-windows-2022-11.9.0', status: 'failed'}, + {gh_job_name: 'policy-tests-macos', status: 'complete'}, + ], + }, + perJobCounts: { + 'e2e-on-ubuntu-latest-11.9.0': {passed: 100, failed: 0, skipped: 5, flaky: 1}, + 'e2e-on-windows-2022-11.9.0': {passed: 90, failed: 2, skipped: 5, flaky: 0}, + 'policy-tests-macos': {passed: 9, failed: 0, skipped: 0, flaky: 0}, + }, + }); + + assert.deepEqual(byKey.linux, { + passed: 101, + failed: 0, + skipped: 5, + shardFailed: false, + hasResults: true, + }); + assert.deepEqual(byKey.windows, { + passed: 90, + failed: 2, + skipped: 5, + shardFailed: true, + hasResults: true, + }); + assert.equal(byKey.macos, undefined); + assert.deepEqual(byKey['macos-policy'], { + passed: 9, + failed: 0, + skipped: 0, + shardFailed: false, + hasResults: true, + }); + }); + + it('marks an OS with no counts but a failed shard', () => { + const byKey = buildOsStatusTotals({ + detail: { + reports: [ + {gh_job_name: 'e2e-on-macos-26-11.10.0', status: 'failed'}, + ], + }, + perJobCounts: {}, + }); + + assert.equal(byKey.macos.shardFailed, true); + assert.equal(byKey.macos.hasResults, false); + }); +}); + +describe('flipPerOsCommitStatuses', () => { + function makeHarness() { + const statuses = []; + const github = { + rest: { + repos: { + createCommitStatus: async (opts) => { + statuses.push(opts); + }, + }, + }, + }; + const core = {warning: () => {}}; + const context = {repo: {owner: 'mattermost', repo: 'desktop'}}; + const compositeIdentity = {commit_sha: 'abc123'}; + return {statuses, github, core, context, compositeIdentity}; + } + + it('maps success / failure from per-OS counts', async () => { + const {statuses, github, core, context, compositeIdentity} = makeHarness(); + + await flipPerOsCommitStatuses({ + github, + context, + compositeIdentity, + detail: {reports: []}, + perJobCounts: { + 'e2e-on-ubuntu-latest-11.9.0': {passed: 10, failed: 0, skipped: 1, flaky: 0}, + 'e2e-on-windows-2022-11.9.0': {passed: 8, failed: 2, skipped: 0, flaky: 0}, + }, + targetUrl: 'https://example.test/report', + upstreamJobsSucceeded: true, + expectedOs: ['linux', 'windows'], + core, + }); + + const byContext = Object.fromEntries(statuses.map((s) => [s.context, s])); + assert.equal(statuses.length, 2); + assert.equal(byContext['e2e/linux'].state, 'success'); + assert.match(byContext['e2e/linux'].description, /10 passed, 0 failed/); + assert.equal(byContext['e2e/windows'].state, 'failure'); + assert.match(byContext['e2e/windows'].description, /8 passed, 2 failed/); + }); + + it('flips separate e2e/-policy contexts', async () => { + const {statuses, github, core, context, compositeIdentity} = makeHarness(); + + await flipPerOsCommitStatuses({ + github, + context, + compositeIdentity, + detail: { + reports: [ + {gh_job_name: 'policy-tests-windows', status: 'failed'}, + ], + }, + perJobCounts: { + 'policy-tests-macos': {passed: 14, failed: 0, skipped: 0, flaky: 0}, + 'policy-tests-windows': {passed: 10, failed: 1, skipped: 0, flaky: 0}, + }, + targetUrl: 'https://example.test/report', + upstreamJobsSucceeded: true, + expectedOs: ['macos'], + expectedPolicyOs: ['macos', 'windows'], + core, + }); + + const byContext = Object.fromEntries(statuses.map((s) => [s.context, s])); + assert.equal(byContext['e2e/macos'].state, 'error'); + assert.equal(byContext['e2e/macos-policy'].state, 'success'); + assert.equal(byContext['e2e/windows-policy'].state, 'failure'); + assert.equal(byContext['e2e/windows'], undefined); + }); + + it('emits error when upstream succeeded but OS has no results', async () => { + const {statuses, github, core, context, compositeIdentity} = makeHarness(); + + await flipPerOsCommitStatuses({ + github, + context, + compositeIdentity, + detail: {reports: []}, + perJobCounts: {}, + targetUrl: 'https://example.test/report', + upstreamJobsSucceeded: true, + expectedOs: ['macos'], + core, + }); + + assert.equal(statuses.length, 1); + assert.equal(statuses[0].context, 'e2e/macos'); + assert.equal(statuses[0].state, 'error'); + assert.match(statuses[0].description, /incomplete/i); + }); + + it('emits failure when upstream failed and OS has no results', async () => { + const {statuses, github, core, context, compositeIdentity} = makeHarness(); + + await flipPerOsCommitStatuses({ + github, + context, + compositeIdentity, + detail: {reports: []}, + perJobCounts: {}, + targetUrl: 'https://example.test/report', + upstreamJobsSucceeded: false, + expectedOs: ['linux'], + core, + }); + + assert.equal(statuses.length, 1); + assert.equal(statuses[0].state, 'failure'); + assert.match(statuses[0].description, /untracked by TSIO/i); + }); + + it('falls back to three OS contexts when expectedOs is empty and there are no results', async () => { + const {statuses, github, core, context, compositeIdentity} = makeHarness(); + + await flipPerOsCommitStatuses({ + github, + context, + compositeIdentity, + detail: {reports: []}, + perJobCounts: {}, + targetUrl: 'https://example.test/report', + upstreamJobsSucceeded: true, + expectedOs: [], + core, + }); + + const contexts = statuses.map((s) => s.context).sort(); + assert.deepEqual(contexts, ['e2e/linux', 'e2e/macos', 'e2e/windows']); + assert.ok(statuses.every((s) => s.state === 'error')); + }); + + it('does not flip policy contexts when expectedPolicyOs is omitted', async () => { + const {statuses, github, core, context, compositeIdentity} = makeHarness(); + + await flipPerOsCommitStatuses({ + github, + context, + compositeIdentity, + detail: {reports: []}, + perJobCounts: { + 'policy-tests-macos': {passed: 1, failed: 0, skipped: 0, flaky: 0}, + }, + targetUrl: 'https://example.test/report', + upstreamJobsSucceeded: true, + expectedOs: ['linux'], + core, + }); + + assert.deepEqual(statuses.map((s) => s.context), ['e2e/linux']); + }); + + it('points each check at its individual TSIO report, not the group URL', async () => { + const {statuses, github, core, context, compositeIdentity} = makeHarness(); + + await flipPerOsCommitStatuses({ + github, + context, + compositeIdentity, + detail: { + reports: [ + {id: 'rid-linux', gh_job_name: 'e2e-on-ubuntu-latest-11.9.0', status: 'complete'}, + {id: 'rid-mac', gh_job_name: 'e2e-on-macos-14-11.9.0', status: 'complete'}, + {id: 'rid-win', gh_job_name: 'e2e-on-windows-2022-11.9.0', status: 'complete'}, + {id: 'rid-mac-policy', gh_job_name: 'policy-tests-macos', status: 'complete'}, + {id: 'rid-win-policy', gh_job_name: 'policy-tests-windows', status: 'complete'}, + ], + }, + perJobCounts: { + 'e2e-on-ubuntu-latest-11.9.0': {passed: 219, failed: 0, skipped: 11, flaky: 0}, + 'e2e-on-macos-14-11.9.0': {passed: 225, failed: 0, skipped: 20, flaky: 0}, + 'e2e-on-windows-2022-11.9.0': {passed: 236, failed: 0, skipped: 20, flaky: 0}, + 'policy-tests-macos': {passed: 9, failed: 0, skipped: 0, flaky: 0}, + 'policy-tests-windows': {passed: 9, failed: 0, skipped: 0, flaky: 0}, + }, + targetUrl: 'https://test-io.test.mattermost.com/reports/desktop/pr/abc1234/desktop-pr', + baseUrl: 'https://test-io.test.mattermost.com', + upstreamJobsSucceeded: true, + expectedOs: ['linux', 'macos', 'windows'], + expectedPolicyOs: ['macos', 'windows'], + core, + }); + + const byContext = Object.fromEntries(statuses.map((s) => [s.context, s])); + assert.equal(byContext['e2e/linux'].target_url, 'https://test-io.test.mattermost.com/reports/r/rid-linux'); + assert.equal(byContext['e2e/macos'].target_url, 'https://test-io.test.mattermost.com/reports/r/rid-mac'); + assert.equal(byContext['e2e/windows'].target_url, 'https://test-io.test.mattermost.com/reports/r/rid-win'); + assert.equal(byContext['e2e/macos-policy'].target_url, 'https://test-io.test.mattermost.com/reports/r/rid-mac-policy'); + assert.equal(byContext['e2e/windows-policy'].target_url, 'https://test-io.test.mattermost.com/reports/r/rid-win-policy'); + }); + + it('falls back to the group URL when a bucket has no uploaded report id', async () => { + const {statuses, github, core, context, compositeIdentity} = makeHarness(); + const groupUrl = 'https://test-io.test.mattermost.com/reports/desktop/pr/abc1234/desktop-pr'; + + await flipPerOsCommitStatuses({ + github, + context, + compositeIdentity, + detail: {reports: []}, + perJobCounts: { + 'e2e-on-ubuntu-latest-11.9.0': {passed: 1, failed: 0, skipped: 0, flaky: 0}, + }, + targetUrl: groupUrl, + baseUrl: 'https://test-io.test.mattermost.com', + upstreamJobsSucceeded: true, + expectedOs: ['linux'], + core, + }); + + assert.equal(statuses[0].target_url, groupUrl); + }); +}); + +describe('reportUrlForStatusBucket', () => { + const baseUrl = 'https://test-io.test.mattermost.com'; + const fallback = 'https://test-io.test.mattermost.com/reports/desktop/pr/abc/desktop-pr'; + + it('returns the individual report URL when a bucket has one uploaded report', () => { + const url = reportUrlForStatusBucket({ + reports: [ + {id: 'rid-linux', gh_job_name: 'e2e-on-ubuntu-latest-11.9.0'}, + {id: 'rid-mac', gh_job_name: 'e2e-on-macos-14-11.9.0'}, + ], + bucketKey: 'linux', + baseUrl, + fallbackUrl: fallback, + }); + assert.equal(url, `${baseUrl}/reports/r/rid-linux`); + }); + + it('prefers a failed individual report when a bucket has multiple uploads', () => { + const url = reportUrlForStatusBucket({ + reports: [ + {id: 'rid-linux-a', gh_job_name: 'e2e-on-ubuntu-latest-11.9.0', status: 'complete'}, + {id: 'rid-linux-b', gh_job_name: 'e2e-on-ubuntu-latest-11.10.0', status: 'failed'}, + ], + bucketKey: 'linux', + baseUrl, + fallbackUrl: fallback, + }); + assert.equal(url, `${baseUrl}/reports/r/rid-linux-b`); + }); + + it('keeps the group URL when a bucket has multiple successful uploads', () => { + const url = reportUrlForStatusBucket({ + reports: [ + {id: 'rid-linux-a', gh_job_name: 'e2e-on-ubuntu-latest-11.9.0', status: 'complete'}, + {id: 'rid-linux-b', gh_job_name: 'e2e-on-ubuntu-latest-11.10.0', status: 'complete'}, + ], + bucketKey: 'linux', + baseUrl, + fallbackUrl: fallback, + }); + assert.equal(url, fallback); + }); +});