From 8841dfd5519842be879059c82423ce1c22795c7f Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Wed, 8 Jul 2026 01:53:44 +0530 Subject: [PATCH 01/37] ci: add TSIO report-upload spike workflow Throwaway workflow_dispatch that runs a 2-shard Playwright suite and uploads via test-system-io-report-upload + finalizes via test-system-io-summary against TSIO staging. Validates the only unproven action in the planned CMT->TSIO migration (report-upload has zero production callers today) before wiring it into compatibility-matrix-testing.yml. Safe to delete once the spike passes and the real CMT is wired to TSIO. --- .github/workflows/tsio-spike.yml | 172 +++++++++++++++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 .github/workflows/tsio-spike.yml diff --git a/.github/workflows/tsio-spike.yml b/.github/workflows/tsio-spike.yml new file mode 100644 index 00000000000..da1cb2d3e72 --- /dev/null +++ b/.github/workflows/tsio-spike.yml @@ -0,0 +1,172 @@ +# Spike: validate the test-system-io-report-upload + test-system-io-summary chain +# on a throwaway Playwright run BEFORE wiring it into compatibility-matrix-testing.yml. +# +# Why standalone: production CMT needs Matterwick-provisioned servers + a built +# Electron bundle. This spike runs two trivial Playwright specs across 2 shards +# to exercise the only unproven part of the TSIO path — test-system-io-report-upload +# has zero production callers (only TSIO's own self-test). If group finalization + +# the consolidated summary render here, the desktop CMT wiring is safe to ship. +# +# Run: workflow_dispatch on a branch. No secrets required — auth is OIDC +# (permissions: id-token: write). Defaults to TSIO staging so we don't pollute prod. +# Safe to delete this file once the spike passes and the real CMT is wired to TSIO. +name: TSIO Spike + +on: + workflow_dispatch: + inputs: + use-staging: + description: "Target TSIO staging (true) instead of production" + required: false + default: "true" + type: boolean + fail-on-test-failures: + description: "Fail the workflow if any shard failed (leave false to see the summary on a red run)" + required: false + default: "false" + type: boolean + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +# Workflow-level default; jobs below narrow to least-privilege. +permissions: + contents: read + +jobs: + prepare: + runs-on: ubuntu-22.04 + outputs: + composite-identity-json: ${{ steps.identity.outputs.composite-identity-json }} + total-reports-expected: "2" + steps: + - name: Build composite identity + id: identity + env: + GITHUB_REPOSITORY: ${{ github.repository }} + MM_SHA: ${{ github.sha }} + MM_BRANCH: ${{ github.ref_name }} + run: | + # `name` groups the report on the TSIO dashboard. Distinct from any real + # CMT/PR context so the spike never collides with production report groups. + COMPOSITE_IDENTITY=$(jq -nc \ + --arg repo "${GITHUB_REPOSITORY}" \ + --arg sha "${MM_SHA}" \ + --arg run_id "${GITHUB_RUN_ID}" \ + --arg attempt "${GITHUB_RUN_ATTEMPT}" \ + --arg name "tsio-spike-desktop" \ + --arg branch "${MM_BRANCH}" \ + '{repository:$repo, commit_sha:$sha, gh_run_id:$run_id, name:$name, gh_run_attempt:$attempt, branch:$branch}') + echo "composite-identity-json=${COMPOSITE_IDENTITY}" >> "$GITHUB_OUTPUT" + echo "$COMPOSITE_IDENTITY" | jq . + + shards: + name: shard-${{ matrix.shard }} + needs: prepare + runs-on: ubuntu-22.04 + permissions: + contents: read + id-token: write # TSIO auth via OIDC + actions: read # report-upload resolves gh_job_id via the GitHub API + strategy: + fail-fast: false + matrix: + shard: [1, 2] + steps: + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '22.x' + + - name: Scaffold a tiny Playwright project + run: | + mkdir -p spike/tests + cat > spike/package.json <<'PKG' + { "name": "tsio-spike", "private": true, "type": "module" } + PKG + cat > spike/playwright.config.ts <<'CFG' + import {defineConfig, devices} from '@playwright/test'; + export default defineConfig({ + testDir: './tests', + fullyParallel: false, + workers: 1, + retries: 0, + reporter: [['line'], ['json', {outputFile: 'results.json'}], ['blob', {outputDir: 'blob-report'}]], + outputDir: './output', + use: {trace: 'off', screenshot: 'only-on-failure'}, + projects: [{name: 'chromium', use: {...devices['Desktop Chrome']}}], + }); + CFG + # Shard 1 gets a passing file; shard 2 gets pass + intentional fail + skip, + # so the consolidated summary has something real to render across legs. + cat > spike/tests/a.spec.ts <<'SPEC' + import {test, expect} from '@playwright/test'; + test('spike-pass-a', async () => { expect(1 + 1).toBe(2); }); + SPEC + cat > spike/tests/b.spec.ts <<'SPEC' + import {test, expect} from '@playwright/test'; + test('spike-pass-b', async () => { expect('ok').toBe('ok'); }); + test('spike-fail-b', async () => { expect(1).toBe(2); }); + test.skip('spike-skip-b', async () => {}); + SPEC + + - name: Install Playwright + browser + working-directory: spike + run: | + npm i -D @playwright/test + # No --with-deps: that runs apt-get, which flaked on ubuntu-22.04 + # (packages.microsoft.com NOSPLIT). Runner ships the needed libs. + npx playwright install chromium + + - name: Run tests (sharded) + continue-on-error: true + working-directory: spike + run: npx playwright test --shard=${{ matrix.shard }}/2 + + - name: Upload shard report to TSIO + if: ${{ always() && hashFiles('spike/results.json') != '' }} + uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-report-upload@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 0.10.0 / 2026-05-16 + with: + use-staging: ${{ inputs.use-staging }} + composite-identity: ${{ needs.prepare.outputs.composite-identity-json }} + total-reports-expected: ${{ needs.prepare.outputs.total-reports-expected }} + framework: playwright + github-token: ${{ secrets.GITHUB_TOKEN }} + # MUST match this job's `name:` field (shard-1 / shard-2). + gh-job-name: shard-${{ matrix.shard }} + json-path: spike/results.json + screenshots-dir: spike/output + + summary: + name: tsio-summary + needs: [prepare, shards] + if: always() + runs-on: ubuntu-22.04 + permissions: + contents: read + id-token: write + statuses: write # summary flips the pending→success/failure commit status + actions: read + steps: + - name: Render consolidated TSIO summary + id: summary + uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-summary@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 0.10.0 / 2026-05-16 + with: + use-staging: ${{ inputs.use-staging }} + composite-identity: ${{ needs.prepare.outputs.composite-identity-json }} + framework: playwright + commit-status-context: tsio-spike/desktop + fail-on-test-failures: ${{ inputs.fail-on-test-failures }} + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Show outputs + if: always() + env: + DESC: ${{ steps.summary.outputs.commit_status_description }} + PAYLOAD: ${{ steps.summary.outputs.webhook_payload }} + run: | + echo "commit_status_description:" + echo "$DESC" + echo + echo "webhook_payload:" + echo "$PAYLOAD" \ No newline at end of file From 32e83f079befd96032478dae62fcfcc0f90cec8f Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Wed, 8 Jul 2026 02:36:29 +0530 Subject: [PATCH 02/37] ci: wire desktop CMT + PR/master E2E reporting to Test System IO MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the hand-rolled cmt-leg-*/node-rollup summary in compatibility-matrix-testing.yml with test-system-io-report-upload (per leg) + test-system-io-summary (one consolidated report per CMT run, context e2e/compatibility-matrix-testing unchanged). Adds the same TSIO reporting to e2e-functional.yml (PR/master runs) additively, under its own commit-status context (e2e-test/desktop-playwright) — does not touch the existing update-final-status/E2E-label flow or its required-checks contract. Adds the native Playwright `json` reporter output (results.json) that test-system-io-report-upload's playwright parser requires; the existing blob/line/junit reporters are unchanged. Not yet run against real TSIO. test-system-io-report-upload has zero production callers anywhere (only TSIO's own self-test) — needs one real workflow_dispatch run on a throwaway branch against a real ephemeral test server before this replaces the proven hand-rolled rollup in the actual CMT gating path. --- .../compatibility-matrix-testing.yml | 75 ++++++++++++------- .github/workflows/e2e-functional-template.yml | 44 +++++++++++ .github/workflows/e2e-functional.yml | 59 +++++++++++++++ e2e/playwright.config.ts | 4 + 4 files changed, 154 insertions(+), 28 deletions(-) diff --git a/.github/workflows/compatibility-matrix-testing.yml b/.github/workflows/compatibility-matrix-testing.yml index 5aa0b1551dc..0a4a064d644 100644 --- a/.github/workflows/compatibility-matrix-testing.yml +++ b/.github/workflows/compatibility-matrix-testing.yml @@ -38,6 +38,8 @@ jobs: runs-on: ubuntu-22.04 outputs: DESKTOP_SHA: ${{ steps.repo.outputs.DESKTOP_SHA }} + TSIO_COMPOSITE_IDENTITY: ${{ steps.tsio-identity.outputs.composite-identity-json }} + TSIO_TOTAL_REPORTS_EXPECTED: ${{ steps.tsio-identity.outputs.total-reports-expected }} steps: - name: cmt/checkout-desktop uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -48,6 +50,29 @@ jobs: id: repo run: echo "DESKTOP_SHA=$(git rev-parse HEAD)" >> ${GITHUB_OUTPUT} + # One composite identity + total-reports-expected for the WHOLE CMT run + # (every OS x server-version leg), computed once here and threaded into + # every `e2e` matrix leg below so they land in a single TSIO report group. + - name: cmt/tsio-identity + id: tsio-identity + env: + GITHUB_REPOSITORY: ${{ github.repository }} + MM_SHA: ${{ steps.repo.outputs.DESKTOP_SHA }} + MM_BRANCH: ${{ inputs.DESKTOP_VERSION }} + CMT_MATRIX: ${{ inputs.CMT_MATRIX }} + run: | + TOTAL=$(echo "${CMT_MATRIX}" | jq -c '(.environment | length) * (.server | length)') + echo "total-reports-expected=${TOTAL}" >> "$GITHUB_OUTPUT" + COMPOSITE_IDENTITY=$(jq -nc \ + --arg repo "${GITHUB_REPOSITORY}" \ + --arg sha "${MM_SHA}" \ + --arg run_id "${GITHUB_RUN_ID}" \ + --arg attempt "${GITHUB_RUN_ATTEMPT}" \ + --arg name "cmt-desktop" \ + --arg branch "${MM_BRANCH}" \ + '{repository:$repo, commit_sha:$sha, gh_run_id:$run_id, name:$name, gh_run_attempt:$attempt, branch:$branch}') + echo "composite-identity-json=${COMPOSITE_IDENTITY}" >> "$GITHUB_OUTPUT" + update-initial-status: runs-on: ubuntu-22.04 needs: @@ -94,6 +119,7 @@ jobs: name: ${{ matrix.environment.os }}-${{ matrix.server.version }} uses: ./.github/workflows/e2e-functional-template.yml needs: + - calculate-commit-hash - update-initial-status strategy: fail-fast: false @@ -106,43 +132,36 @@ jobs: DESKTOP_VERSION: ${{ inputs.DESKTOP_VERSION }} MM_SERVER_VERSION: ${{ matrix.server.version }} TYPE: "CMT" + tsio-composite-identity: ${{ needs.calculate-commit-hash.outputs.TSIO_COMPOSITE_IDENTITY }} + tsio-total-reports-expected: ${{ needs.calculate-commit-hash.outputs.TSIO_TOTAL_REPORTS_EXPECTED }} - # We need to duplicate here in order to set the proper commit status + # Consolidated final status + rollup summary, via Test System IO instead of + # the hand-rolled artifact-based rollup. One TSIO report group covers every + # OS x server-version leg of the CMT run; commit status target_url deep-links + # that dashboard page. Context name unchanged (e2e/compatibility-matrix-testing). # https://mattermost.atlassian.net/browse/CLD-5815 - update-failure-final-status: + update-final-status: runs-on: ubuntu-22.04 - if: failure() || cancelled() + if: always() + permissions: + contents: read + id-token: write + statuses: write needs: - calculate-commit-hash - e2e steps: - - uses: mattermost/actions/delivery/update-commit-status@218fd96a63451259dc100e1292a5e44af92fe15d - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Render TSIO summary + flip commit status + id: summary + uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-summary@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 0.10.0 / 2026-05-16 with: - repository_full_name: mattermost/desktop - commit_sha: ${{ needs.calculate-commit-hash.outputs.DESKTOP_SHA }} - context: e2e/compatibility-matrix-testing - description: "Compatibility Matrix Testing for ${{ inputs.DESKTOP_VERSION }} version" - status: failure + composite-identity: ${{ needs.calculate-commit-hash.outputs.TSIO_COMPOSITE_IDENTITY }} + framework: playwright + commit-status-context: e2e/compatibility-matrix-testing + report-type: RELEASE + ref-branch: ${{ inputs.DESKTOP_VERSION }} + github-token: ${{ secrets.GITHUB_TOKEN }} - # https://mattermost.atlassian.net/browse/CLD-5815 - update-success-final-status: - runs-on: ubuntu-22.04 - if: success() - needs: - - calculate-commit-hash - - e2e - steps: - - uses: mattermost/actions/delivery/update-commit-status@218fd96a63451259dc100e1292a5e44af92fe15d - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - repository_full_name: mattermost/desktop - commit_sha: ${{ needs.calculate-commit-hash.outputs.DESKTOP_SHA }} - context: e2e/compatibility-matrix-testing - description: "Compatibility Matrix Testing for ${{ inputs.DESKTOP_VERSION }} version" - status: success # Instance cleanup is handled by Matterwick: when this workflow completes, GitHub sends a # workflow_run "completed" event and Matterwick destroys the servers it provisioned for this diff --git a/.github/workflows/e2e-functional-template.yml b/.github/workflows/e2e-functional-template.yml index 459afcbd798..e238c68d93a 100644 --- a/.github/workflows/e2e-functional-template.yml +++ b/.github/workflows/e2e-functional-template.yml @@ -37,6 +37,16 @@ on: MM_SERVER_VERSION: type: string required: true + tsio-composite-identity: + description: "Composite identity JSON for Test System IO (repository, commit_sha, gh_run_id, name, gh_run_attempt, branch, gh_pr_number). Empty disables TSIO reporting for this leg." + required: false + type: string + default: "" + tsio-total-reports-expected: + description: "Total leg count across the whole caller run (== the full OS or OS x server-version matrix size). Must match every leg's call. Required when tsio-composite-identity is set." + required: false + type: string + default: "" outputs: NEW_FAILURES_LINUX: description: "The output to comment" @@ -139,6 +149,16 @@ on: MM_SERVER_VERSION: type: string required: true + tsio-composite-identity: + description: "Composite identity JSON for Test System IO. Empty disables TSIO reporting for this leg." + required: false + type: string + default: "" + tsio-total-reports-expected: + description: "Total leg count across the whole caller run. Required when tsio-composite-identity is set." + required: false + type: string + default: "" env: BRANCH: ${{ github.head_ref || github.ref_name }} @@ -157,8 +177,12 @@ jobs: name: e2e-on-${{ inputs.runs-on }} runs-on: ${{ inputs.runs-on }} # Runs untrusted PR code (npm ci / Playwright) with a read-only token. + # id-token/actions:read are additive for TSIO reporting (OIDC auth + + # gh_job_id lookup) — still no write access to repo contents. permissions: contents: read + id-token: write + actions: read defaults: run: shell: bash @@ -419,3 +443,23 @@ jobs: default: throw new Error(`Unsupported OS: ${os}`); } + + # TSIO reporting: upload this leg's native Playwright JSON + screenshots. + # Gated on tsio-composite-identity so callers that don't opt in (ad-hoc + # workflow_dispatch without identity) are untouched. Used by both plain + # PR/master runs and CMT — the caller decides grouping via `name` in the + # composite identity and computes tsio-total-reports-expected once for + # the whole run's matrix. + - name: e2e/upload-report-to-tsio + if: ${{ always() && inputs.tsio-composite-identity != '' }} + uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-report-upload@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 0.10.0 / 2026-05-16 + with: + composite-identity: ${{ inputs.tsio-composite-identity }} + total-reports-expected: ${{ inputs.tsio-total-reports-expected }} + framework: playwright + github-token: ${{ secrets.GITHUB_TOKEN }} + # MUST match this job's `name:` field above (e2e-on-). + gh-job-name: e2e-on-${{ inputs.runs-on }} + json-path: e2e/test-results/results.json + screenshots-dir: e2e/test-results + diff --git a/.github/workflows/e2e-functional.yml b/.github/workflows/e2e-functional.yml index 3dfb2df31b4..44213c04a96 100644 --- a/.github/workflows/e2e-functional.yml +++ b/.github/workflows/e2e-functional.yml @@ -44,6 +44,8 @@ jobs: runs-on: ubuntu-latest outputs: platforms: ${{ steps.generate.outputs.platforms }} + tsio-composite-identity: ${{ steps.tsio-identity.outputs.composite-identity-json }} + tsio-total-reports-expected: ${{ steps.tsio-identity.outputs.total-reports-expected }} steps: - id: generate env: @@ -54,6 +56,33 @@ jobs: platforms=$(echo "${INSTANCE_DETAILS}" | jq -c 'map(if .runner == "macos-latest" then .runner = "macos-26" else . end)') echo "platforms=${platforms}" >> "$GITHUB_OUTPUT" + # One composite identity + total-reports-expected for the whole run + # (every platform leg), so all legs land in a single TSIO report group. + - id: tsio-identity + env: + GITHUB_REPOSITORY: ${{ github.repository }} + MM_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + MM_BRANCH: ${{ inputs.version_name }} + PR_NUMBER: ${{ inputs.pr_number }} + RUN_TYPE: ${{ inputs.run_type || 'PR' }} + PLATFORMS: ${{ steps.generate.outputs.platforms }} + run: | + TOTAL=$(echo "${PLATFORMS}" | jq -c 'length') + echo "total-reports-expected=${TOTAL}" >> "$GITHUB_OUTPUT" + NAME="desktop-$(echo "${RUN_TYPE}" | tr '[:upper:]' '[:lower:]')" + if [ -n "${PR_NUMBER}" ]; then + COMPOSITE_IDENTITY=$(jq -nc \ + --arg repo "${GITHUB_REPOSITORY}" --arg sha "${MM_SHA}" --arg run_id "${GITHUB_RUN_ID}" \ + --arg name "${NAME}" --arg attempt "${GITHUB_RUN_ATTEMPT}" --arg branch "${MM_BRANCH}" --arg pr "${PR_NUMBER}" \ + '{repository:$repo, commit_sha:$sha, gh_run_id:$run_id, name:$name, gh_run_attempt:$attempt, branch:$branch, gh_pr_number:$pr}') + else + COMPOSITE_IDENTITY=$(jq -nc \ + --arg repo "${GITHUB_REPOSITORY}" --arg sha "${MM_SHA}" --arg run_id "${GITHUB_RUN_ID}" \ + --arg name "${NAME}" --arg attempt "${GITHUB_RUN_ATTEMPT}" --arg branch "${MM_BRANCH}" \ + '{repository:$repo, commit_sha:$sha, gh_run_id:$run_id, name:$name, gh_run_attempt:$attempt, branch:$branch}') + fi + echo "composite-identity-json=${COMPOSITE_IDENTITY}" >> "$GITHUB_OUTPUT" + update-initial-status: name: Update initial status needs: prepare-matrix @@ -100,8 +129,38 @@ jobs: # below). The previous `startsWith(version_name, 'release-') && 'RELEASE'` fallback was # for matterwick's release-branch push, which is no longer wired — removed. TYPE: ${{ inputs.run_type || 'PR' }} + tsio-composite-identity: ${{ needs.prepare-matrix.outputs.tsio-composite-identity }} + tsio-total-reports-expected: ${{ needs.prepare-matrix.outputs.tsio-total-reports-expected }} secrets: inherit + # Additive TSIO reporting, separate commit-status context from the existing + # per-platform status/label flow below (update-final-status / remove-e2e-label). + # Does not replace or affect that flow — required checks and the E2E label + # contract are untouched. + tsio-summary: + name: TSIO summary + runs-on: ubuntu-24.04 + needs: + - prepare-matrix + - e2e-tests + if: always() + permissions: + contents: read + id-token: write + statuses: write + steps: + - name: Render TSIO summary + flip commit status + uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-summary@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 0.10.0 / 2026-05-16 + with: + composite-identity: ${{ needs.prepare-matrix.outputs.tsio-composite-identity }} + framework: playwright + commit-status-context: e2e-test/desktop-playwright + report-type: ${{ inputs.run_type || 'PR' }} + pr-number: ${{ inputs.pr_number }} + ref-branch: ${{ inputs.version_name }} + fail-on-test-failures: "false" + github-token: ${{ secrets.GITHUB_TOKEN }} + update-final-status: name: Update final status runs-on: ubuntu-24.04 diff --git a/e2e/playwright.config.ts b/e2e/playwright.config.ts index 5e7b449a5ee..af60b4e377a 100644 --- a/e2e/playwright.config.ts +++ b/e2e/playwright.config.ts @@ -81,6 +81,10 @@ const reporters = process.env.CI ? [ ['blob', {outputDir: 'blob-report'}], ['line'], ['junit', {outputFile: 'test-results/e2e-junit.xml'}], + // Native Playwright JSON — required by test-system-io-report-upload's + // `framework: playwright` parser (it reads suites[].specs[].tests[].results[], + // not the JUnit shape analyze-flaky-test.js uses). + ['json', {outputFile: 'test-results/results.json'}], ] as const : [ ['html', {open: 'never', outputFolder: 'playwright-report'}], ['list'], From e0b2c126eb2d9dc58b19ef819a914bb3a707e88a Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Wed, 8 Jul 2026 02:43:15 +0530 Subject: [PATCH 03/37] ci: fix e2e-pr-trigger concurrency race dropping E2E/Run label PR #3891 reproduced this: a same-second, unrelated `labeled` event (mm-cloud-bot's release-notes label) canceled the `opened` event's in-progress run via cancel-in-progress before add-e2e-label could run, and the winning run's own job conditions didn't match that label (add-e2e-label needs action in [opened,reopened,ready_for_review, synchronize]; honor-e2e-override needs label.name == 'E2E/Override'). Net effect: E2E/Run never got added, silently. Fix: bucket the concurrency group by event instead of PR number alone. opened/reopened/ready_for_review/synchronize still share one group (preserves "only the most recent push proceeds" for rapid pushes); every other labeled/unlabeled event gets its own action+label-keyed group so it can't cancel a real trigger run. --- .github/workflows/e2e-pr-trigger.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e-pr-trigger.yml b/.github/workflows/e2e-pr-trigger.yml index c484573332c..b1d2ad312ea 100644 --- a/.github/workflows/e2e-pr-trigger.yml +++ b/.github/workflows/e2e-pr-trigger.yml @@ -15,7 +15,15 @@ name: E2E PR Trigger # # The concurrency group ensures rapid pushes to the same PR don't queue multiple # label operations: only the most recent push proceeds. - +# +# Group key is bucketed by event, not just PR number (PR #3891 regression: a +# same-second, unrelated `labeled` event from a release-notes bot cancelled the +# `opened` event's in-progress run via cancel-in-progress before it could add +# E2E/Run, and the winning run's own job conditions didn't match that label — +# so E2E/Run never got added at all). opened/reopened/ready_for_review/synchronize +# still share one group (preserves "only the most recent push proceeds" for +# rapid pushes); every other event (any other label add/remove) gets its own +# action+label-keyed group so it can't cancel a real trigger run. on: pull_request: types: @@ -29,7 +37,10 @@ on: - master concurrency: - group: e2e-pr-trigger-${{ github.event.pull_request.number }} + group: >- + e2e-pr-trigger-${{ github.event.pull_request.number }}-${{ + contains(fromJSON('["opened", "reopened", "ready_for_review", "synchronize"]'), github.event.action) + && 'trigger' || format('other-{0}-{1}', github.event.action, github.event.label.name) }} cancel-in-progress: true jobs: From 78a13cc47075fac0b3c865fc09a11a701b1c60a8 Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Wed, 8 Jul 2026 03:01:25 +0530 Subject: [PATCH 04/37] ci: pack TSIO inputs into one field to stay under workflow_call's 10-input cap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run 28899437045 (Electron Playwright Tests, tsio-spike) failed with startup_failure and zero jobs — a workflow-definition-level rejection, not a test failure. e2e-functional-template.yml's workflow_call/ workflow_dispatch inputs had hit exactly 10 (8 existing + the 2 new tsio-composite-identity/tsio-total-reports-expected), which is GitHub's documented cap for reusable workflows. actionlint doesn't check this — it's a GitHub API-side limit, not a syntax rule. Fix: collapse the two new inputs into one JSON-packed `tsio-config` ({composite_identity, total_reports_expected}), unpacked via fromJSON() in the report-upload step. Brings the template back to 9 inputs. Callers (compatibility-matrix-testing.yml, e2e-functional.yml) now build that JSON with format() instead of passing two separate `with:` fields. Jobs that call the TSIO actions directly (not through the workflow_call boundary) are unaffected and still use the unpacked job outputs. --- .../compatibility-matrix-testing.yml | 5 ++- .github/workflows/e2e-functional-template.yml | 38 ++++++++----------- .github/workflows/e2e-functional.yml | 5 ++- 3 files changed, 22 insertions(+), 26 deletions(-) diff --git a/.github/workflows/compatibility-matrix-testing.yml b/.github/workflows/compatibility-matrix-testing.yml index 0a4a064d644..36baf590b84 100644 --- a/.github/workflows/compatibility-matrix-testing.yml +++ b/.github/workflows/compatibility-matrix-testing.yml @@ -132,8 +132,9 @@ jobs: DESKTOP_VERSION: ${{ inputs.DESKTOP_VERSION }} MM_SERVER_VERSION: ${{ matrix.server.version }} TYPE: "CMT" - tsio-composite-identity: ${{ needs.calculate-commit-hash.outputs.TSIO_COMPOSITE_IDENTITY }} - tsio-total-reports-expected: ${{ needs.calculate-commit-hash.outputs.TSIO_TOTAL_REPORTS_EXPECTED }} + # Packed as one JSON input (see e2e-functional-template.yml's tsio-config) — + # workflow_call caps reusable workflows at 10 inputs total. + tsio-config: ${{ format('{{"composite_identity":{0},"total_reports_expected":"{1}"}}', needs.calculate-commit-hash.outputs.TSIO_COMPOSITE_IDENTITY, needs.calculate-commit-hash.outputs.TSIO_TOTAL_REPORTS_EXPECTED) }} # Consolidated final status + rollup summary, via Test System IO instead of # the hand-rolled artifact-based rollup. One TSIO report group covers every diff --git a/.github/workflows/e2e-functional-template.yml b/.github/workflows/e2e-functional-template.yml index e238c68d93a..fcbfa6e906f 100644 --- a/.github/workflows/e2e-functional-template.yml +++ b/.github/workflows/e2e-functional-template.yml @@ -37,13 +37,11 @@ on: MM_SERVER_VERSION: type: string required: true - tsio-composite-identity: - description: "Composite identity JSON for Test System IO (repository, commit_sha, gh_run_id, name, gh_run_attempt, branch, gh_pr_number). Empty disables TSIO reporting for this leg." - required: false - type: string - default: "" - tsio-total-reports-expected: - description: "Total leg count across the whole caller run (== the full OS or OS x server-version matrix size). Must match every leg's call. Required when tsio-composite-identity is set." + # Packed as one JSON input, not two, because workflow_call caps reusable + # workflows at 10 inputs total — this file was already at 8. + # Shape: {"composite_identity": {...}, "total_reports_expected": "N"} + tsio-config: + description: "TSIO reporting config: {composite_identity, total_reports_expected}. Empty disables TSIO reporting for this leg." required: false type: string default: "" @@ -149,13 +147,8 @@ on: MM_SERVER_VERSION: type: string required: true - tsio-composite-identity: - description: "Composite identity JSON for Test System IO. Empty disables TSIO reporting for this leg." - required: false - type: string - default: "" - tsio-total-reports-expected: - description: "Total leg count across the whole caller run. Required when tsio-composite-identity is set." + tsio-config: + description: "TSIO reporting config: {composite_identity, total_reports_expected}. Empty disables TSIO reporting for this leg." required: false type: string default: "" @@ -445,17 +438,18 @@ jobs: } # TSIO reporting: upload this leg's native Playwright JSON + screenshots. - # Gated on tsio-composite-identity so callers that don't opt in (ad-hoc - # workflow_dispatch without identity) are untouched. Used by both plain - # PR/master runs and CMT — the caller decides grouping via `name` in the - # composite identity and computes tsio-total-reports-expected once for - # the whole run's matrix. + # Gated on tsio-config so callers that don't opt in (ad-hoc + # workflow_dispatch without config) are untouched. Used by both plain + # PR/master runs and CMT — the caller decides grouping via `name` inside + # composite_identity and computes total_reports_expected once for the + # whole run's matrix. Packed into one JSON input (see tsio-config above) + # to stay under workflow_call's 10-input cap. - name: e2e/upload-report-to-tsio - if: ${{ always() && inputs.tsio-composite-identity != '' }} + if: ${{ always() && inputs.tsio-config != '' }} uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-report-upload@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 0.10.0 / 2026-05-16 with: - composite-identity: ${{ inputs.tsio-composite-identity }} - total-reports-expected: ${{ inputs.tsio-total-reports-expected }} + composite-identity: ${{ toJSON(fromJSON(inputs.tsio-config).composite_identity) }} + total-reports-expected: ${{ fromJSON(inputs.tsio-config).total_reports_expected }} framework: playwright github-token: ${{ secrets.GITHUB_TOKEN }} # MUST match this job's `name:` field above (e2e-on-). diff --git a/.github/workflows/e2e-functional.yml b/.github/workflows/e2e-functional.yml index 44213c04a96..d4d3899fc92 100644 --- a/.github/workflows/e2e-functional.yml +++ b/.github/workflows/e2e-functional.yml @@ -129,8 +129,9 @@ jobs: # below). The previous `startsWith(version_name, 'release-') && 'RELEASE'` fallback was # for matterwick's release-branch push, which is no longer wired — removed. TYPE: ${{ inputs.run_type || 'PR' }} - tsio-composite-identity: ${{ needs.prepare-matrix.outputs.tsio-composite-identity }} - tsio-total-reports-expected: ${{ needs.prepare-matrix.outputs.tsio-total-reports-expected }} + # Packed as one JSON input (see e2e-functional-template.yml's tsio-config) — + # workflow_call caps reusable workflows at 10 inputs total. + tsio-config: ${{ format('{{"composite_identity":{0},"total_reports_expected":"{1}"}}', needs.prepare-matrix.outputs.tsio-composite-identity, needs.prepare-matrix.outputs.tsio-total-reports-expected) }} secrets: inherit # Additive TSIO reporting, separate commit-status context from the existing From 0c2009ec0a1dc31917991b86a02a0784a932d765 Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Wed, 8 Jul 2026 03:28:05 +0530 Subject: [PATCH 05/37] ci: fix eslint, guard missing results.json, grant TSIO permissions to callers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three issues from the tsio-spike CI run and review: 1. eslint lines-around-comment: missing blank line before the new json reporter comment in e2e/playwright.config.ts. 2. e2e/upload-report-to-tsio could run against a results.json that was never written (e.g. Playwright crashes before any reporter flushes). Gate on hashFiles('e2e/test-results/results.json') != '' in addition to the existing tsio-config check, same fix already needed in the tsio-spike.yml spike for the identical failure mode. 3. Real blocker: "Invalid workflow file ... nested job 'e2e' is requesting 'actions: read, id-token: write', but is only allowed 'actions: none, id-token: none'". A reusable workflow's job can only receive permissions the CALLING job already holds. e2e-tests (e2e-functional.yml) and e2e (compatibility-matrix-testing.yml) both call e2e-functional-template.yml without granting those scopes themselves. Added explicit `permissions: contents: read, actions: read, id-token: write` to both calling jobs — exactly matching what the template's own e2e job declares, no more. Fixed both callers even though only the PR path has actually been exercised yet, since compatibility-matrix-testing.yml has the identical structure and would hit the identical error on its first real CMT dispatch. --- .github/workflows/compatibility-matrix-testing.yml | 10 ++++++++++ .github/workflows/e2e-functional-template.yml | 2 +- .github/workflows/e2e-functional.yml | 12 +++++++++--- e2e/playwright.config.ts | 1 + 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/.github/workflows/compatibility-matrix-testing.yml b/.github/workflows/compatibility-matrix-testing.yml index 36baf590b84..04ea0ed38af 100644 --- a/.github/workflows/compatibility-matrix-testing.yml +++ b/.github/workflows/compatibility-matrix-testing.yml @@ -124,6 +124,16 @@ jobs: strategy: fail-fast: false matrix: ${{ fromJson(inputs.CMT_MATRIX) }} + # A reusable workflow can only receive permissions the calling job itself + # already holds — the nested e2e job's TSIO reporting steps need + # actions:read + id-token:write (same fix as e2e-functional.yml's e2e-tests + # job; this workflow has no top-level `permissions:` default, so without + # this the nested job would hit the same "requesting actions:read, + # id-token:write but only allowed none" rejection e2e-functional.yml hit). + permissions: + contents: read + actions: read + id-token: write secrets: inherit with: runs-on: ${{ matrix.environment.runner }} diff --git a/.github/workflows/e2e-functional-template.yml b/.github/workflows/e2e-functional-template.yml index fcbfa6e906f..2af223538dc 100644 --- a/.github/workflows/e2e-functional-template.yml +++ b/.github/workflows/e2e-functional-template.yml @@ -445,7 +445,7 @@ jobs: # whole run's matrix. Packed into one JSON input (see tsio-config above) # to stay under workflow_call's 10-input cap. - name: e2e/upload-report-to-tsio - if: ${{ always() && inputs.tsio-config != '' }} + if: ${{ always() && inputs.tsio-config != '' && hashFiles('e2e/test-results/results.json') != '' }} uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-report-upload@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 0.10.0 / 2026-05-16 with: composite-identity: ${{ toJSON(fromJSON(inputs.tsio-config).composite_identity) }} diff --git a/.github/workflows/e2e-functional.yml b/.github/workflows/e2e-functional.yml index d4d3899fc92..59b1914cd29 100644 --- a/.github/workflows/e2e-functional.yml +++ b/.github/workflows/e2e-functional.yml @@ -106,7 +106,7 @@ jobs: await updateInitialStatus({ github, context, platforms }); e2e-tests: - needs: + needs: - prepare-matrix - update-initial-status name: ${{ matrix.platform }} @@ -114,8 +114,14 @@ jobs: matrix: include: ${{ fromJson(needs.prepare-matrix.outputs.platforms) }} fail-fast: false - # The reusable template runs only untrusted PR code (npm ci / Playwright) with a read-only - # token; it inherits the workflow-level contents:read default. + # The reusable template runs untrusted PR code (npm ci / Playwright) with a read-only + # token; actions:read + id-token:write are additive, needed only to grant the nested + # e2e job's TSIO reporting steps (OIDC auth + gh_job_id lookup) — a reusable workflow + # can only receive permissions the calling job itself already holds. + permissions: + contents: read + actions: read + id-token: write uses: ./.github/workflows/e2e-functional-template.yml with: runs-on: ${{ matrix.runner }} diff --git a/e2e/playwright.config.ts b/e2e/playwright.config.ts index af60b4e377a..53360f4746c 100644 --- a/e2e/playwright.config.ts +++ b/e2e/playwright.config.ts @@ -81,6 +81,7 @@ const reporters = process.env.CI ? [ ['blob', {outputDir: 'blob-report'}], ['line'], ['junit', {outputFile: 'test-results/e2e-junit.xml'}], + // Native Playwright JSON — required by test-system-io-report-upload's // `framework: playwright` parser (it reads suites[].specs[].tests[].results[], // not the JUnit shape analyze-flaky-test.js uses). From b91302d4dfbc42c8b438c537a0ec21491123eaa0 Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Wed, 8 Jul 2026 03:55:03 +0530 Subject: [PATCH 06/37] ci: fix cancel-on-manual-unlabel wiping .github/actions/cancel-e2e-runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "Can't find 'action.yml' ... under .github/actions/cancel-e2e-runs" — pre-existing bug, surfaced now that the concurrency-race fix lets unlabeled events actually reach this job for real. Root cause: two checkouts into the same default path. The first does a full checkout of base.ref (includes .github/actions/cancel-e2e-runs/). The second sparse-checks out head.sha with only e2e/utils/github-actions.js — which replaces the whole working tree per its own sparse rules, deleting cancel-e2e-runs/ moments before the next step tries to `uses:` it locally. Fix: sparse-checkout the untrusted head ref into its own path (untrusted-head/) instead of the default workspace, then copy just that one file over the base-ref version. Preserves the original security intent (only that one file ever comes from the untrusted PR branch; this job runs with actions:write/statuses:write) while no longer clobbering the trusted local action definition. --- .github/workflows/e2e-pr-trigger.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/e2e-pr-trigger.yml b/.github/workflows/e2e-pr-trigger.yml index b1d2ad312ea..d618bce10db 100644 --- a/.github/workflows/e2e-pr-trigger.yml +++ b/.github/workflows/e2e-pr-trigger.yml @@ -221,6 +221,10 @@ jobs: with: ref: ${{ github.event.pull_request.base.ref }} + # Sparse-checked into a separate path, not the default workspace: a plain + # second checkout of head.sha here would replace the base-ref tree with a + # sparse one containing only this file — wiping out + # .github/actions/cancel-e2e-runs/ before the step below can use it. - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 if: github.event.pull_request.head.repo.full_name == github.repository with: @@ -228,6 +232,11 @@ jobs: sparse-checkout: | e2e/utils/github-actions.js sparse-checkout-cone-mode: false + path: untrusted-head + + - name: Use PR's github-actions.js + if: github.event.pull_request.head.repo.full_name == github.repository + run: cp untrusted-head/e2e/utils/github-actions.js e2e/utils/github-actions.js - name: Cancel E2E runs and mark statuses skipped uses: ./.github/actions/cancel-e2e-runs From ae340160baa015dd3572e78d38c13e9db601a57a Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Wed, 8 Jul 2026 04:32:10 +0530 Subject: [PATCH 07/37] fix tsio job --- .../compatibility-matrix-testing.yml | 33 ++++- .github/workflows/e2e-functional.yml | 37 ++++-- e2e/utils/tsio-report-status.js | 122 ++++++++++++++++++ 3 files changed, 176 insertions(+), 16 deletions(-) create mode 100644 e2e/utils/tsio-report-status.js diff --git a/.github/workflows/compatibility-matrix-testing.yml b/.github/workflows/compatibility-matrix-testing.yml index 04ea0ed38af..cdd6c5c6ba9 100644 --- a/.github/workflows/compatibility-matrix-testing.yml +++ b/.github/workflows/compatibility-matrix-testing.yml @@ -150,6 +150,10 @@ jobs: # the hand-rolled artifact-based rollup. One TSIO report group covers every # OS x server-version leg of the CMT run; commit status target_url deep-links # that dashboard page. Context name unchanged (e2e/compatibility-matrix-testing). + # + # Uses e2e/utils/tsio-report-status.js, not test-system-io-summary — summary + # only reads /api/v1/orchestration/status, which report-upload never + # populates (see that file's header comment). # https://mattermost.atlassian.net/browse/CLD-5815 update-final-status: runs-on: ubuntu-22.04 @@ -162,16 +166,31 @@ jobs: - calculate-commit-hash - e2e steps: + # tsio-report-status.js is repo code, not fetched by github-script itself. + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ inputs.DESKTOP_VERSION }} + sparse-checkout: | + e2e/utils/tsio-report-status.js + sparse-checkout-cone-mode: false + - name: Render TSIO summary + flip commit status id: summary - uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-summary@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 0.10.0 / 2026-05-16 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + TSIO_COMPOSITE_IDENTITY: ${{ needs.calculate-commit-hash.outputs.TSIO_COMPOSITE_IDENTITY }} + TSIO_TOTAL_REPORTS_EXPECTED: ${{ needs.calculate-commit-hash.outputs.TSIO_TOTAL_REPORTS_EXPECTED }} + COMMIT_STATUS_CONTEXT: e2e/compatibility-matrix-testing with: - composite-identity: ${{ needs.calculate-commit-hash.outputs.TSIO_COMPOSITE_IDENTITY }} - framework: playwright - commit-status-context: e2e/compatibility-matrix-testing - report-type: RELEASE - ref-branch: ${{ inputs.DESKTOP_VERSION }} - github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const {reportUrl, status, stats} = await require('./e2e/utils/tsio-report-status.js')({ + core, context, github, + compositeIdentity: JSON.parse(process.env.TSIO_COMPOSITE_IDENTITY), + totalReportsExpected: parseInt(process.env.TSIO_TOTAL_REPORTS_EXPECTED, 10), + commitStatusContext: process.env.COMMIT_STATUS_CONTEXT, + failOnTestFailures: true, + }); + core.info(`TSIO report ${reportUrl}: ${status} (${JSON.stringify(stats)})`); # Instance cleanup is handled by Matterwick: when this workflow completes, GitHub sends a diff --git a/.github/workflows/e2e-functional.yml b/.github/workflows/e2e-functional.yml index 59b1914cd29..92b61cef6c6 100644 --- a/.github/workflows/e2e-functional.yml +++ b/.github/workflows/e2e-functional.yml @@ -144,6 +144,11 @@ jobs: # per-platform status/label flow below (update-final-status / remove-e2e-label). # Does not replace or affect that flow — required checks and the E2E label # contract are untouched. + # + # Uses e2e/utils/tsio-report-status.js, not test-system-io-summary — summary + # only reads /api/v1/orchestration/status, which report-upload never + # populates (see that file's header comment). failOnTestFailures stays + # false: this job is additive reporting, not the required-check gate. tsio-summary: name: TSIO summary runs-on: ubuntu-24.04 @@ -156,17 +161,31 @@ jobs: id-token: write statuses: write steps: + # tsio-report-status.js is repo code, not fetched by github-script itself. + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ inputs.version_name }} + sparse-checkout: | + e2e/utils/tsio-report-status.js + sparse-checkout-cone-mode: false + - name: Render TSIO summary + flip commit status - uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-summary@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 0.10.0 / 2026-05-16 + id: summary + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + TSIO_COMPOSITE_IDENTITY: ${{ needs.prepare-matrix.outputs.tsio-composite-identity }} + TSIO_TOTAL_REPORTS_EXPECTED: ${{ needs.prepare-matrix.outputs.tsio-total-reports-expected }} + COMMIT_STATUS_CONTEXT: e2e-test/desktop-playwright with: - composite-identity: ${{ needs.prepare-matrix.outputs.tsio-composite-identity }} - framework: playwright - commit-status-context: e2e-test/desktop-playwright - report-type: ${{ inputs.run_type || 'PR' }} - pr-number: ${{ inputs.pr_number }} - ref-branch: ${{ inputs.version_name }} - fail-on-test-failures: "false" - github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const {reportUrl, status, stats} = await require('./e2e/utils/tsio-report-status.js')({ + core, context, github, + compositeIdentity: JSON.parse(process.env.TSIO_COMPOSITE_IDENTITY), + totalReportsExpected: parseInt(process.env.TSIO_TOTAL_REPORTS_EXPECTED, 10), + commitStatusContext: process.env.COMMIT_STATUS_CONTEXT, + failOnTestFailures: false, + }); + core.info(`TSIO report ${reportUrl}: ${status} (${JSON.stringify(stats)})`); update-final-status: name: Update final status diff --git a/e2e/utils/tsio-report-status.js b/e2e/utils/tsio-report-status.js new file mode 100644 index 00000000000..e0f5e1cf83b --- /dev/null +++ b/e2e/utils/tsio-report-status.js @@ -0,0 +1,122 @@ +// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. +/* eslint-disable no-console -- Logging is intentional in CI utility scripts */ + +// Not test-system-io-summary: it reads /api/v1/orchestration/status, which only +// the dispatch-begin/dispatch-run flow populates. report-upload posts to a +// disjoint /api/v1/reports/* subsystem, so this re-opens the idempotent begin +// endpoint to recover the report id, then polls the public status endpoint. + +const PRODUCTION_URL = 'https://test-io.test.mattermost.com'; +const STAGING_URL = 'https://staging-test-io.test.mattermost.com'; + +const TERMINAL_STATUSES = ['completed', 'incomplete']; +const POLL_ATTEMPTS = 6; +const POLL_DELAY_MS = 5000; + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +/** + * Recover a report group's id via the idempotent begin endpoint, poll the + * public status endpoint until the group leaves in_progress, render a step + * summary, and flip a commit status. + * @param {Object} params - Parameters object + * @param {Object} params.core - @actions/core from actions/github-script + * @param {Object} params.context - GitHub Actions context + * @param {Object} params.github - GitHub API client from actions/github-script + * @param {Object} params.compositeIdentity - {repository, commit_sha, gh_run_id, name, gh_run_attempt, branch, gh_pr_number} + * @param {number} params.totalReportsExpected - Number of per-leg reports expected in this group + * @param {string} params.commitStatusContext - Commit-status context to flip on completion + * @param {boolean} [params.failOnTestFailures] - When true (default), throw if the group didn't complete cleanly + * @param {boolean} [params.useStaging] - Target TSIO staging instead of production + * @param {string} [params.oidcAudience] - OIDC audience claim TSIO expects + * @returns {Promise<{reportUrl: string, status: string, stats: Object}>} + */ +async function reportTsioStatus({ + core, + context, + github, + compositeIdentity, + totalReportsExpected, + commitStatusContext, + failOnTestFailures = true, + useStaging = false, + oidcAudience = 'mattermost-test-system-io', +}) { + const baseUrl = useStaging ? STAGING_URL : PRODUCTION_URL; + const idToken = await core.getIDToken(oidcAudience); + core.setSecret(idToken); + + const beginRes = await fetch(`${baseUrl}/api/v1/reports/begin`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${idToken}`, + }, + body: JSON.stringify({ + repository: compositeIdentity.repository, + commit: compositeIdentity.commit_sha, + gh_run_id: compositeIdentity.gh_run_id, + gh_run_attempt: compositeIdentity.gh_run_attempt, + framework: 'playwright', + name: compositeIdentity.name, + branch: compositeIdentity.branch, + total_reports_expected: totalReportsExpected, + ...(compositeIdentity.gh_pr_number ? {gh_pr_number: parseInt(compositeIdentity.gh_pr_number, 10)} : {}), + }), + }); + if (!beginRes.ok) { + throw new Error(`reports/begin failed: ${beginRes.status} ${await beginRes.text()}`); + } + const {report_id: reportId} = await beginRes.json(); + const reportUrl = `${baseUrl}/reports/${reportId}`; + + let detail; + for (let attempt = 0; attempt < POLL_ATTEMPTS; attempt++) { + const statusRes = await fetch(`${baseUrl}/api/v1/reports/${reportId}`); + if (!statusRes.ok) { + throw new Error(`reports/${reportId} failed: ${statusRes.status} ${await statusRes.text()}`); + } + detail = await statusRes.json(); + if (TERMINAL_STATUSES.includes(detail.status)) { + break; + } + if (attempt < POLL_ATTEMPTS - 1) { + await sleep(POLL_DELAY_MS); + } + } + + const stats = detail.test_stats || {}; + const isComplete = detail.status === 'completed'; + const hasFailures = (stats.failed || 0) > 0; + const overallState = isComplete && !hasFailures ? 'success' : 'failure'; + + const summaryLines = [ + `### Test System IO — ${compositeIdentity.name}`, + '', + `**Status:** ${detail.status} · **Report:** [${reportId}](${reportUrl})`, + `**Tests:** ${stats.passed ?? '?'} passed, ${stats.failed ?? '?'} failed, ${stats.flaky ?? 0} flaky, ` + + `${stats.skipped ?? '?'} skipped (of ${stats.total ?? '?'})`, + '', + ]; + await core.summary.addRaw(summaryLines.join('\n')).write(); + + const description = `${stats.passed ?? 0}/${stats.total ?? 0} passed, ${stats.failed ?? 0} failed`.slice(0, 140); + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: compositeIdentity.commit_sha, + state: overallState, + context: commitStatusContext, + description, + target_url: reportUrl, + }); + + if (failOnTestFailures && overallState === 'failure') { + throw new Error(`TSIO report ${reportId} did not pass: status=${detail.status}, failed=${stats.failed || 0}`); + } + + return {reportUrl, status: detail.status, stats}; +} + +module.exports = reportTsioStatus; From 595cc30f7f6b0f45c154bc607b4eca4856d91646 Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Wed, 8 Jul 2026 05:49:14 +0530 Subject: [PATCH 08/37] just keep 1 status check --- .../compatibility-matrix-testing.yml | 1 + .github/workflows/e2e-functional-template.yml | 134 +------- .github/workflows/e2e-functional.yml | 147 +++------ .github/workflows/e2e-pr-trigger.yml | 1 + e2e/playwright.config.ts | 4 +- e2e/utils/analyze-flaky-test.js | 299 ------------------ e2e/utils/github-actions.js | 161 +--------- e2e/utils/tsio-report-status.js | 10 +- 8 files changed, 69 insertions(+), 688 deletions(-) delete mode 100644 e2e/utils/analyze-flaky-test.js diff --git a/.github/workflows/compatibility-matrix-testing.yml b/.github/workflows/compatibility-matrix-testing.yml index cdd6c5c6ba9..b9588c43b2f 100644 --- a/.github/workflows/compatibility-matrix-testing.yml +++ b/.github/workflows/compatibility-matrix-testing.yml @@ -170,6 +170,7 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ inputs.DESKTOP_VERSION }} + persist-credentials: false sparse-checkout: | e2e/utils/tsio-report-status.js sparse-checkout-cone-mode: false diff --git a/.github/workflows/e2e-functional-template.yml b/.github/workflows/e2e-functional-template.yml index 2af223538dc..e741efc637b 100644 --- a/.github/workflows/e2e-functional-template.yml +++ b/.github/workflows/e2e-functional-template.yml @@ -45,71 +45,6 @@ on: required: false type: string default: "" - outputs: - NEW_FAILURES_LINUX: - description: "The output to comment" - value: ${{ jobs.e2e.outputs.NEW_FAILURES_LINUX }} - NEW_FAILURES_MACOS: - description: "The output to comment" - value: ${{ jobs.e2e.outputs.NEW_FAILURES_MACOS }} - NEW_FAILURES_WINDOWS: - description: "The output to comment" - value: ${{ jobs.e2e.outputs.NEW_FAILURES_WINDOWS }} - REPORT_LINK_LINUX: - description: "Link to Linux report" - value: ${{ jobs.e2e.outputs.REPORT_LINK_LINUX }} - REPORT_LINK_MACOS: - description: "Link to MacOS report" - value: ${{ jobs.e2e.outputs.REPORT_LINK_MACOS }} - REPORT_LINK_WINDOWS: - description: "Link to Windows report" - value: ${{ jobs.e2e.outputs.REPORT_LINK_WINDOWS }} - STATUS_LINUX: - description: "The status of the linux test" - value: ${{ jobs.e2e.outputs.STATUS_LINUX }} - STATUS_MACOS: - description: "The status of the macOS test" - value: ${{ jobs.e2e.outputs.STATUS_MACOS }} - PASSED_LINUX: - description: "Number of passed tests on Linux" - value: ${{ jobs.e2e.outputs.PASSED_LINUX }} - PASSED_MACOS: - description: "Number of passed tests on macOS" - value: ${{ jobs.e2e.outputs.PASSED_MACOS }} - PASSED_WINDOWS: - description: "Number of passed tests on Windows" - value: ${{ jobs.e2e.outputs.PASSED_WINDOWS }} - TOTAL_LINUX: - description: "Total tests on Linux (passed + failed + skipped)" - value: ${{ jobs.e2e.outputs.TOTAL_LINUX }} - TOTAL_MACOS: - description: "Total tests on macOS (passed + failed + skipped)" - value: ${{ jobs.e2e.outputs.TOTAL_MACOS }} - TOTAL_WINDOWS: - description: "Total tests on Windows (passed + failed + skipped)" - value: ${{ jobs.e2e.outputs.TOTAL_WINDOWS }} - SKIPPED_LINUX: - description: "Number of skipped tests on Linux" - value: ${{ jobs.e2e.outputs.SKIPPED_LINUX }} - SKIPPED_MACOS: - description: "Number of skipped tests on macOS" - value: ${{ jobs.e2e.outputs.SKIPPED_MACOS }} - SKIPPED_WINDOWS: - description: "Number of skipped tests on Windows" - value: ${{ jobs.e2e.outputs.SKIPPED_WINDOWS }} - COLLECTION_FAILED_LINUX: - description: "Whether Playwright collected zero tests on Linux" - value: ${{ jobs.e2e.outputs.COLLECTION_FAILED_LINUX }} - COLLECTION_FAILED_MACOS: - description: "Whether Playwright collected zero tests on macOS" - value: ${{ jobs.e2e.outputs.COLLECTION_FAILED_MACOS }} - COLLECTION_FAILED_WINDOWS: - description: "Whether Playwright collected zero tests on Windows" - value: ${{ jobs.e2e.outputs.COLLECTION_FAILED_WINDOWS }} - STATUS_WINDOWS: - description: "The status of the windows test" - value: ${{ jobs.e2e.outputs.STATUS_WINDOWS }} - workflow_dispatch: inputs: MM_TEST_SERVER_URL: @@ -179,28 +114,6 @@ jobs: defaults: run: shell: bash - outputs: - NEW_FAILURES_LINUX: ${{ steps.analyze-flaky-tests.outputs.NEW_FAILURES_LINUX }} - NEW_FAILURES_MACOS: ${{ steps.analyze-flaky-tests.outputs.NEW_FAILURES_MACOS }} - NEW_FAILURES_WINDOWS: ${{ steps.analyze-flaky-tests.outputs.NEW_FAILURES_WINDOWS }} - REPORT_LINK_LINUX: ${{ steps.analyze-flaky-tests.outputs.REPORT_LINK_LINUX }} - REPORT_LINK_MACOS: ${{ steps.analyze-flaky-tests.outputs.REPORT_LINK_MACOS }} - REPORT_LINK_WINDOWS: ${{ steps.analyze-flaky-tests.outputs.REPORT_LINK_WINDOWS }} - STATUS_LINUX: ${{ steps.analyze-flaky-tests.outputs.STATUS_LINUX }} - STATUS_WINDOWS: ${{ steps.analyze-flaky-tests.outputs.STATUS_WINDOWS }} - STATUS_MACOS: ${{ steps.analyze-flaky-tests.outputs.STATUS_MACOS }} - PASSED_LINUX: ${{ steps.analyze-flaky-tests.outputs.PASSED_LINUX }} - PASSED_MACOS: ${{ steps.analyze-flaky-tests.outputs.PASSED_MACOS }} - PASSED_WINDOWS: ${{ steps.analyze-flaky-tests.outputs.PASSED_WINDOWS }} - TOTAL_LINUX: ${{ steps.analyze-flaky-tests.outputs.TOTAL_LINUX }} - TOTAL_MACOS: ${{ steps.analyze-flaky-tests.outputs.TOTAL_MACOS }} - TOTAL_WINDOWS: ${{ steps.analyze-flaky-tests.outputs.TOTAL_WINDOWS }} - SKIPPED_LINUX: ${{ steps.analyze-flaky-tests.outputs.SKIPPED_LINUX }} - SKIPPED_MACOS: ${{ steps.analyze-flaky-tests.outputs.SKIPPED_MACOS }} - SKIPPED_WINDOWS: ${{ steps.analyze-flaky-tests.outputs.SKIPPED_WINDOWS }} - COLLECTION_FAILED_LINUX: ${{ steps.analyze-flaky-tests.outputs.COLLECTION_FAILED_LINUX }} - COLLECTION_FAILED_MACOS: ${{ steps.analyze-flaky-tests.outputs.COLLECTION_FAILED_MACOS }} - COLLECTION_FAILED_WINDOWS: ${{ steps.analyze-flaky-tests.outputs.COLLECTION_FAILED_WINDOWS }} steps: - name: e2e/set-required-variables id: variables @@ -400,43 +313,6 @@ jobs: echo "report_url=${REPORT_URL}" >> "$GITHUB_OUTPUT" echo "Playwright report (${RUNNER_OS}) uploaded: ${REPORT_URL}" >> "$GITHUB_STEP_SUMMARY" - - name: e2e/analyze-flaky-tests - id: analyze-flaky-tests - if: always() - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - PER_OS_REPORT_URL: ${{ steps.upload-html-report-to-s3.outputs.report_url }} - JOB_STATUS: ${{ job.status }} - with: - script: | - process.chdir('./e2e'); - const { analyzeFlakyTests } = require('./utils/analyze-flaky-test.js'); - const { failureCount, passCount, skipCount, totalCount, os, testStatus, collectionFailed } = analyzeFlakyTests(); - const runUrl = `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`; - const reportUrl = process.env.PER_OS_REPORT_URL || runUrl; - const setOSOutputs = (suffix) => { - core.setOutput(`NEW_FAILURES_${suffix}`, String(failureCount)); - core.setOutput(`REPORT_LINK_${suffix}`, reportUrl); - core.setOutput(`STATUS_${suffix}`, testStatus); - core.setOutput(`PASSED_${suffix}`, String(passCount)); - core.setOutput(`SKIPPED_${suffix}`, String(skipCount)); - core.setOutput(`TOTAL_${suffix}`, String(totalCount)); - core.setOutput(`COLLECTION_FAILED_${suffix}`, String(collectionFailed)); - }; - switch (os) { - case 'linux': - setOSOutputs('LINUX'); - break; - case 'darwin': - setOSOutputs('MACOS'); - break; - case 'win32': - setOSOutputs('WINDOWS'); - break; - default: - throw new Error(`Unsupported OS: ${os}`); - } - # TSIO reporting: upload this leg's native Playwright JSON + screenshots. # Gated on tsio-config so callers that don't opt in (ad-hoc # workflow_dispatch without config) are untouched. Used by both plain @@ -457,3 +333,13 @@ jobs: json-path: e2e/test-results/results.json screenshots-dir: e2e/test-results + # Playwright's own exit code already accounts for retries — trust it + # instead of re-deriving pass/fail from the JUnit report. + - name: e2e/fail-on-test-failures + if: always() + run: | + if [ "${PLAYWRIGHT_EXIT_CODE:-1}" != "0" ]; then + echo "Playwright exited with code ${PLAYWRIGHT_EXIT_CODE:-unset}" >&2 + exit 1 + fi + diff --git a/.github/workflows/e2e-functional.yml b/.github/workflows/e2e-functional.yml index 92b61cef6c6..7fd1fe9fa58 100644 --- a/.github/workflows/e2e-functional.yml +++ b/.github/workflows/e2e-functional.yml @@ -57,7 +57,8 @@ jobs: echo "platforms=${platforms}" >> "$GITHUB_OUTPUT" # One composite identity + total-reports-expected for the whole run - # (every platform leg), so all legs land in a single TSIO report group. + # (every e2e-tests platform leg, plus the 2 fixed e2e-policy-tests legs), + # so all legs land in a single TSIO report group. - id: tsio-identity env: GITHUB_REPOSITORY: ${{ github.repository }} @@ -67,7 +68,7 @@ jobs: RUN_TYPE: ${{ inputs.run_type || 'PR' }} PLATFORMS: ${{ steps.generate.outputs.platforms }} run: | - TOTAL=$(echo "${PLATFORMS}" | jq -c 'length') + TOTAL=$(( $(echo "${PLATFORMS}" | jq -c 'length') + 2 )) echo "total-reports-expected=${TOTAL}" >> "$GITHUB_OUTPUT" NAME="desktop-$(echo "${RUN_TYPE}" | tr '[:upper:]' '[:lower:]')" if [ -n "${PR_NUMBER}" ]; then @@ -83,32 +84,9 @@ jobs: fi echo "composite-identity-json=${COMPOSITE_IDENTITY}" >> "$GITHUB_OUTPUT" - update-initial-status: - name: Update initial status - needs: prepare-matrix - runs-on: ubuntu-24.04 - permissions: - contents: read - statuses: write - steps: - - name: Checkout code - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - - name: Update initial status for all platforms - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - PLATFORMS: ${{ needs.prepare-matrix.outputs.platforms }} - with: - github-token: ${{ github.token }} - script: | - const { updateInitialStatus } = require('./e2e/utils/github-actions.js'); - const platforms = JSON.parse(process.env.PLATFORMS); - await updateInitialStatus({ github, context, platforms }); - e2e-tests: needs: - prepare-matrix - - update-initial-status name: ${{ matrix.platform }} strategy: matrix: @@ -140,21 +118,20 @@ jobs: tsio-config: ${{ format('{{"composite_identity":{0},"total_reports_expected":"{1}"}}', needs.prepare-matrix.outputs.tsio-composite-identity, needs.prepare-matrix.outputs.tsio-total-reports-expected) }} secrets: inherit - # Additive TSIO reporting, separate commit-status context from the existing - # per-platform status/label flow below (update-final-status / remove-e2e-label). - # Does not replace or affect that flow — required checks and the E2E label - # contract are untouched. + # The single E2E status check for PR/master (e2e-test/desktop-playwright), + # replacing the former 5 (e2e/linux, e2e/macos, e2e/windows, + # policy-test/macos, policy-test/windows). # # Uses e2e/utils/tsio-report-status.js, not test-system-io-summary — summary # only reads /api/v1/orchestration/status, which report-upload never - # populates (see that file's header comment). failOnTestFailures stays - # false: this job is additive reporting, not the required-check gate. + # populates (see that file's header comment). tsio-summary: name: TSIO summary runs-on: ubuntu-24.04 needs: - prepare-matrix - e2e-tests + - e2e-policy-tests if: always() permissions: contents: read @@ -165,6 +142,7 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ inputs.version_name }} + persist-credentials: false sparse-checkout: | e2e/utils/tsio-report-status.js sparse-checkout-cone-mode: false @@ -183,47 +161,10 @@ jobs: compositeIdentity: JSON.parse(process.env.TSIO_COMPOSITE_IDENTITY), totalReportsExpected: parseInt(process.env.TSIO_TOTAL_REPORTS_EXPECTED, 10), commitStatusContext: process.env.COMMIT_STATUS_CONTEXT, - failOnTestFailures: false, + failOnTestFailures: true, }); core.info(`TSIO report ${reportUrl}: ${status} (${JSON.stringify(stats)})`); - update-final-status: - name: Update final status - runs-on: ubuntu-24.04 - needs: - - prepare-matrix - - e2e-tests - if: always() - permissions: - contents: read - statuses: write - steps: - - name: Checkout code - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - - name: Update final status for all platforms - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - PR_NUMBER: ${{ inputs.pr_number }} - PLATFORMS: ${{ needs.prepare-matrix.outputs.platforms }} - OUTPUTS: ${{ toJSON(needs.e2e-tests.outputs) }} - E2E_TESTS_RESULT: ${{ needs.e2e-tests.result }} - with: - github-token: ${{ github.token }} - script: | - const { updateFinalStatus } = require('./e2e/utils/github-actions.js'); - const platforms = JSON.parse(process.env.PLATFORMS); - const outputs = JSON.parse(process.env.OUTPUTS); - const prNumber = parseInt(process.env.PR_NUMBER, 10) || null; - await updateFinalStatus({ - github, - context, - platforms, - outputs, - e2eTestsResult: process.env.E2E_TESTS_RESULT, - prNumber, - }); - remove-e2e-label: name: Remove E2E label from PR runs-on: ubuntu-22.04 @@ -233,7 +174,7 @@ jobs: needs: - e2e-tests - e2e-policy-tests - - update-final-status + - tsio-summary if: always() steps: - name: e2e/remove-label-from-pr @@ -290,11 +231,13 @@ jobs: e2e-policy-tests: name: policy-tests-${{ matrix.platform }} - # Runs untrusted PR code (npm ci / Playwright); grant only the commit-status write its - # check-for-failures step needs, never pull-requests: write. + needs: prepare-matrix + # Runs untrusted PR code (npm ci / Playwright); grant only what the TSIO + # upload needs, never pull-requests: write. permissions: contents: read - statuses: write + id-token: write + actions: read strategy: matrix: include: @@ -445,43 +388,29 @@ jobs: SERVER_VERSION: ${{ inputs.MM_SERVER_VERSION }} DESKTOP_VERSION: ${{ inputs.version_name }} - - name: e2e/check-for-failures - id: check-failures - if: always() - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + # Joins the same TSIO report group as e2e-tests — gh-job-name must match + # this job's rendered name (policy-tests-) above. + - name: e2e/upload-report-to-tsio + if: ${{ always() && needs.prepare-matrix.outputs.tsio-composite-identity != '' && hashFiles('e2e/test-results/results.json') != '' }} + uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-report-upload@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 0.10.0 / 2026-05-16 with: - script: | - process.chdir('./e2e'); - const { analyzeFlakyTests } = require('./utils/analyze-flaky-test.js'); - const { formatStatusDescription } = require('./utils/github-actions.js'); - const { newFailedTests, failureCount, passCount, skipCount, totalCount, collectionFailed } = analyzeFlakyTests(); - - const runUrl = `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`; - const platform = process.platform === 'darwin' ? 'macos' : 'windows'; - const hasFailed = newFailedTests && newFailedTests.length > 0; - const description = formatStatusDescription({ - passed: passCount, - failed: failureCount, - collectionFailed, - }); - - try { - await github.rest.repos.createCommitStatus({ - owner: context.repo.owner, - repo: context.repo.repo, - sha: context.sha, - state: hasFailed ? 'failure' : 'success', - context: `policy-test/${platform}`, - description, - target_url: runUrl, - }); - } catch (e) { - console.log(`Could not update commit status: ${e.message}`); - } - - if (hasFailed) { - core.setFailed(`${newFailedTests.length} policy test(s) failed:\n${newFailedTests.join('\n')}`); - } + composite-identity: ${{ needs.prepare-matrix.outputs.tsio-composite-identity }} + total-reports-expected: ${{ needs.prepare-matrix.outputs.tsio-total-reports-expected }} + framework: playwright + github-token: ${{ secrets.GITHUB_TOKEN }} + gh-job-name: policy-tests-${{ matrix.platform }} + json-path: e2e/test-results/results.json + screenshots-dir: e2e/test-results + + # Playwright's own exit code already accounts for retries — trust it + # instead of re-deriving pass/fail from the JUnit report. + - name: e2e/fail-on-test-failures + if: always() + run: | + if [ "${PLAYWRIGHT_EXIT_CODE:-1}" != "0" ]; then + echo "Playwright exited with code ${PLAYWRIGHT_EXIT_CODE:-unset}" >&2 + exit 1 + fi - name: Upload test results if: always() diff --git a/.github/workflows/e2e-pr-trigger.yml b/.github/workflows/e2e-pr-trigger.yml index d618bce10db..3782bacda8b 100644 --- a/.github/workflows/e2e-pr-trigger.yml +++ b/.github/workflows/e2e-pr-trigger.yml @@ -229,6 +229,7 @@ jobs: if: github.event.pull_request.head.repo.full_name == github.repository with: ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false sparse-checkout: | e2e/utils/github-actions.js sparse-checkout-cone-mode: false diff --git a/e2e/playwright.config.ts b/e2e/playwright.config.ts index 53360f4746c..e862a74a75a 100644 --- a/e2e/playwright.config.ts +++ b/e2e/playwright.config.ts @@ -80,11 +80,9 @@ function buildPlatformProjects(): Project[] { const reporters = process.env.CI ? [ ['blob', {outputDir: 'blob-report'}], ['line'], - ['junit', {outputFile: 'test-results/e2e-junit.xml'}], // Native Playwright JSON — required by test-system-io-report-upload's - // `framework: playwright` parser (it reads suites[].specs[].tests[].results[], - // not the JUnit shape analyze-flaky-test.js uses). + // `framework: playwright` parser (reads suites[].specs[].tests[].results[]). ['json', {outputFile: 'test-results/results.json'}], ] as const : [ ['html', {open: 'never', outputFolder: 'playwright-report'}], diff --git a/e2e/utils/analyze-flaky-test.js b/e2e/utils/analyze-flaky-test.js deleted file mode 100644 index 683aec879f7..00000000000 --- a/e2e/utils/analyze-flaky-test.js +++ /dev/null @@ -1,299 +0,0 @@ -// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. - -// CommonJS is required here: GitHub Actions workflows load this file via -// require() in actions/github-script, which does not support ES modules. - -const fs = require('fs'); -const path = require('path'); -const {createRequire} = require('module'); - -const JUNIT_REPORT_PATH = path.join(__dirname, '..', 'test-results', 'e2e-junit.xml'); - -function getXMLParserClass() { - const packageCandidates = [ - path.join(__dirname, '..', 'package.json'), - path.join(__dirname, '..', '..', 'package.json'), - ]; - - for (const packageJson of packageCandidates) { - try { - const {XMLParser} = createRequire(packageJson)('fast-xml-parser'); - return XMLParser; - } catch (error) { - const isModuleNotFound = - error && - (error.code === 'MODULE_NOT_FOUND' || error.code === 'ERR_MODULE_NOT_FOUND'); - if (!isModuleNotFound) { - throw error; - } - - // try the other package root (e2e/ vs repo root) - } - } - - throw new Error('fast-xml-parser is not installed. Run npm ci in the repo root and e2e/.'); -} - -function toNumber(value) { - const parsed = parseInt(value, 10); - return Number.isNaN(parsed) ? 0 : parsed; -} - -function asArray(value) { - if (!value) { - return []; - } - - return Array.isArray(value) ? value : [value]; -} - -function escapeRegex(value) { - return String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); -} - -function getSuiteFailureCount(suite) { - if (!suite || typeof suite !== 'object') { - return 0; - } - - // Short-circuit on the common "no failures" case: if the suite advertises - // 0 failures/errors AND has no testcase entries, there's nothing to walk. - const aggregatedFailures = toNumber(suite.failures) + toNumber(suite.errors); - const cases = asArray(suite.testcase); - if (aggregatedFailures === 0 && cases.length === 0) { - return 0; - } - - // If the run ultimately succeeded (exit code 0), the only failures present - // are retries that later passed — don't count any of them. - const exitCode = toNumber(process.env.PLAYWRIGHT_EXIT_CODE || '0'); - if (exitCode === 0) { - return 0; - } - - // Walk testcases and filter out failures that were later retried and passed. - // Playwright's JUnit aggregate `failures` attribute over-counts these, so - // never trust it as a final number. - const definitiveFailures = cases.filter((testcase) => { - // Empty self-closing elements () parse to "" — check for - // presence rather than truthiness. - if (testcase.failure === undefined && testcase.error === undefined) { - return false; - } - - // If this test name ends with a retry suffix like " (retry #1)", - // and the base test (without suffix) also appears as a passing case, - // this failure was retried and resolved — don't count it. - const name = testcase.name || ''; - const retryMatch = name.match(/^(.*) \(retry #\d+\)$/); - if (retryMatch) { - const baseName = retryMatch[1]; - const hasPassingRetry = cases.some( - (c) => c.name === baseName && c.failure === undefined && c.error === undefined, - ); - if (hasPassingRetry) { - return false; - } - } else { - const baseName = name; - const retryPattern = new RegExp(`^${escapeRegex(baseName)} \\(retry #\\d+\\)$`); - const hasPassingRetry = cases.some( - (c) => Boolean(c.name && retryPattern.test(c.name)) && - c.failure === undefined && c.error === undefined, - ); - if (hasPassingRetry) { - return false; - } - } - return true; - }); - - // If aggregate said failures>0 but we couldn't see any testcases (rare — - // summary-only reporter), fall back to the aggregate so we don't silently - // report 0 when something did fail. - if (definitiveFailures.length === 0 && cases.length === 0 && aggregatedFailures > 0) { - return aggregatedFailures; - } - - return definitiveFailures.length; -} - -function getFailureCountFromReport(report) { - if (!report || typeof report !== 'object') { - return 0; - } - - if (report.testsuites) { - const testsuites = report.testsuites; - - // Always walk the per-suite/testcase tree. The top-level aggregate - // `failures` / `errors` attributes include retried-and-passed tests, so - // returning them directly would over-count flaky tests as failures. - // getSuiteFailureCount() filters retries against passing reruns and - // honors PLAYWRIGHT_EXIT_CODE. - return asArray(testsuites.testsuite).reduce((total, suite) => total + getSuiteFailureCount(suite), 0); - } - - return getSuiteFailureCount(report.testsuite); -} - -/** - * Collect every testcase across every suite into a flat array. - */ -function collectAllCases(report) { - if (!report || typeof report !== 'object') { - return []; - } - const suites = report.testsuites ? - asArray(report.testsuites.testsuite) : - asArray(report.testsuite); - const all = []; - for (const suite of suites) { - if (!suite || typeof suite !== 'object') { - continue; - } - for (const tc of asArray(suite.testcase)) { - all.push(tc); - } - } - return all; -} - -/** - * Compute pass / fail / skip / total at the UNIQUE-TEST level (collapsing - * retries into a single outcome per test, mirroring how Playwright's HTML - * report reports stats). A test that failed once then passed on retry counts - * once as "passed" — not as both. - */ -function getOutcomeCounts(report) { - const cases = collectAllCases(report); - if (cases.length === 0) { - return {passed: 0, failed: 0, skipped: 0, total: 0}; - } - - // Group every case (including retries) by its base name. - const byBase = new Map(); - for (const tc of cases) { - const name = tc.name || ''; - const m = name.match(/^(.*) \(retry #\d+\)$/); - const base = m ? m[1] : name; - if (!byBase.has(base)) { - byBase.set(base, []); - } - byBase.get(base).push(tc); - } - - let passed = 0; - let failed = 0; - let skipped = 0; - for (const attempts of byBase.values()) { - // Empty self-closing tags parse to "" — check for property presence. - const anyPass = attempts.some( - (tc) => - tc.failure === undefined && - tc.error === undefined && - tc.skipped === undefined, - ); - const anyFail = attempts.some( - (tc) => tc.failure !== undefined || tc.error !== undefined, - ); - const allSkipped = attempts.every((tc) => tc.skipped !== undefined); - - if (anyPass) { - passed += 1; - } else if (anyFail) { - failed += 1; - } else if (allSkipped) { - skipped += 1; - } - } - - return {passed, failed, skipped, total: passed + failed + skipped}; -} - -function buildAnalysisResult({failureCount, passCount, skipCount, totalCount}) { - const collectedCount = passCount + failureCount + skipCount; - - // Playwright can exit 0 when test collection finds nothing (e.g. a broken - // import aborts discovery). Treat that as an infrastructure failure so the - // PR status check does not go green with "No tests ran". - if (collectedCount === 0) { - return { - failureCount: 1, - passCount: 0, - skipCount, - totalCount, - newFailedTests: ['no-tests-collected'], - os: process.platform, - testStatus: 'failure', - collectionFailed: true, - }; - } - - return { - failureCount, - passCount, - skipCount, - totalCount, - newFailedTests: new Array(failureCount).fill('failed'), - os: process.platform, - testStatus: failureCount > 0 ? 'failure' : 'success', - collectionFailed: false, - }; -} - -function analyzeFlakyTests() { - const hasJunit = fs.existsSync(JUNIT_REPORT_PATH); - - if (!hasJunit) { - if (process.env.JOB_STATUS === 'cancelled') { - return { - failureCount: 0, - passCount: 0, - skipCount: 0, - totalCount: 0, - newFailedTests: [], - os: process.platform, - testStatus: 'error', - collectionFailed: false, - }; - } - - return buildAnalysisResult({ - failureCount: 0, - passCount: 0, - skipCount: 0, - totalCount: 0, - }); - } - - const XMLParser = getXMLParserClass(); - const parser = new XMLParser({ - ignoreAttributes: false, - attributeNamePrefix: '', - }); - - const report = parser.parse(fs.readFileSync(JUNIT_REPORT_PATH, 'utf8')); - const failureCount = getFailureCountFromReport(report); - const outcomes = getOutcomeCounts(report); - - // `failureCount` is the authoritative number (it applies the retry-pass - // filter + honours PLAYWRIGHT_EXIT_CODE). When they disagree (rare — - // summary-only junit, or exit-code 0 with stale aggregates), trust - // `failureCount` and reconcile the rest. - const reconciledFailed = failureCount; - const reconciledPassed = Math.max(0, outcomes.total - reconciledFailed - outcomes.skipped); - - return buildAnalysisResult({ - failureCount: reconciledFailed, - passCount: reconciledPassed, - skipCount: outcomes.skipped, - totalCount: reconciledFailed + reconciledPassed + outcomes.skipped, - }); -} - -module.exports = { - analyzeFlakyTests, - buildAnalysisResult, -}; diff --git a/e2e/utils/github-actions.js b/e2e/utils/github-actions.js index f399f5cd41b..936c208a8e3 100644 --- a/e2e/utils/github-actions.js +++ b/e2e/utils/github-actions.js @@ -2,167 +2,37 @@ // See LICENSE.txt for license information. /* eslint-disable no-console -- Logging is intentional in CI utility scripts */ -const E2E_STATUS_CONTEXTS = [ - 'e2e/linux', - 'e2e/macos', - 'e2e/windows', - 'policy-test/macos', - 'policy-test/windows', -]; +// TSIO (tsio-summary job) is now the single source of truth for PR/master E2E +// pass/fail — this constant is only used to mark that one context as +// cancelled/skipped, not to post pending/final results (see +// e2e/utils/tsio-report-status.js for the real status logic). +const E2E_STATUS_CONTEXT = 'e2e-test/desktop-playwright'; const E2E_WORKFLOW_NAME = 'Electron Playwright Tests'; const ACTIVE_RUN_STATUSES = ['in_progress', 'queued', 'waiting']; const CANCELLED_STATUS_DESCRIPTION = 'E2E cancelled — tests skipped'; /** - * Update initial pending status for all platforms - * @param {Object} params - Parameters object - * @param {Object} params.github - GitHub API client from actions/github-script - * @param {Object} params.context - GitHub Actions context - * @param {Array} params.platforms - Array of platform objects from matrix - */ -async function updateInitialStatus({github, context, platforms}) { - const workflowUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; - - await Promise.all(platforms.map((platform) => - github.rest.repos.createCommitStatus({ - owner: context.repo.owner, - repo: context.repo.repo, - sha: context.sha, - state: 'pending', - context: `e2e/${platform.platform}`, - description: `E2E tests for Mattermost desktop app on ${platform.platform} have started...`, - target_url: workflowUrl, - }), - )); -} - -/** - * Build the short description shown in the PR status check. - * Only counts tests that actually ran on this platform (passed + failed). - * Skipped tests are omitted — they are cross-platform guards, not real - * failures, and inflate the denominator making results look worse. - * - * - all pass: "All 161 ran, 161 passed" - * - any failure: "161 ran, 157 passed, 4 failed" - */ -function formatStatusDescription({passed, failed, collectionFailed}) { - if (collectionFailed) { - return 'No tests ran (collection failed)'; - } - - const ran = passed + failed; - if (ran === 0) { - return failed > 0 ? `0 ran, ${failed} failed` : 'No tests ran'; - } - if (failed === 0) { - return `All ${ran} ran, ${passed} passed`; - } - return `${ran} ran, ${passed} passed, ${failed} failed`; -} - -async function resolveStatusSha({github, context, prNumber}) { - // Commit statuses must target the SHA this workflow run was dispatched for. - // PR HEAD moves when a new push cancels an in-flight run; using it would mark - // the new commit cancelled instead of the superseded one. - if (context.sha) { - return context.sha; - } - - if (prNumber) { - const {data: pr} = await github.rest.pulls.get({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: prNumber, - }); - return pr.head.sha; - } - - return context.payload.pull_request?.head?.sha; -} - -/** - * Update final status for all platforms based on test results - * @param {Object} params - Parameters object - * @param {Object} params.github - GitHub API client from actions/github-script - * @param {Object} params.context - GitHub Actions context - * @param {Array} params.platforms - Array of platform objects from matrix - * @param {Object} params.outputs - Test outputs from e2e-tests job - * @param {string} [params.e2eTestsResult] - needs.e2e-tests.result from the workflow - * @param {number} [params.prNumber] - PR number for status SHA lookup - */ -async function updateFinalStatus({github, context, platforms, outputs, e2eTestsResult, prNumber}) { - const workflowUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; - const sha = await resolveStatusSha({github, context, prNumber}); - const workflowCancelled = e2eTestsResult === 'cancelled'; - - await Promise.all(platforms.map((platform) => { - let osKey; - if (platform.runner.includes('ubuntu')) { - osKey = 'LINUX'; - } else if (platform.runner.includes('macos')) { - osKey = 'MACOS'; - } else { - osKey = 'WINDOWS'; - } - - const failed = Number(outputs[`NEW_FAILURES_${osKey}`] || 0); - const passed = Number(outputs[`PASSED_${osKey}`] || 0); - const collectionFailed = outputs[`COLLECTION_FAILED_${osKey}`] === 'true'; - const platformStatus = outputs[`STATUS_${osKey}`] || ''; - const reportLink = outputs[`REPORT_LINK_${osKey}`] || workflowUrl; - const ran = passed + failed; - - let state; - let description; - - if (platformStatus === 'error' || (workflowCancelled && ran === 0)) { - state = 'error'; - description = CANCELLED_STATUS_DESCRIPTION; - } else if (ran === 0 && (platformStatus === 'success' || platformStatus === '')) { - state = 'error'; - description = workflowCancelled ? CANCELLED_STATUS_DESCRIPTION : 'E2E incomplete — no tests ran'; - } else if (failed > 0 || platformStatus === 'failure') { - state = 'failure'; - description = formatStatusDescription({passed, failed, collectionFailed}); - } else { - state = 'success'; - description = formatStatusDescription({passed, failed, collectionFailed}); - } - - return github.rest.repos.createCommitStatus({ - owner: context.repo.owner, - repo: context.repo.repo, - sha, - state, - context: `e2e/${platform.platform}`, - description, - target_url: reportLink, - }); - })); -} - -/** - * Mark standard E2E commit statuses as cancelled/skipped on a SHA. + * Mark the E2E commit status as cancelled/skipped on a SHA. * GitHub commit statuses have no "skipped" state — `error` matches mobile E2E. */ async function markE2EStatusesCancelled({github, context, sha, reason = CANCELLED_STATUS_DESCRIPTION}) { const description = String(reason).substring(0, 140); const targetUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; - await Promise.all(E2E_STATUS_CONTEXTS.map((statusContext) => - github.rest.repos.createCommitStatus({ + try { + await github.rest.repos.createCommitStatus({ owner: context.repo.owner, repo: context.repo.repo, sha, state: 'error', - context: statusContext, + context: E2E_STATUS_CONTEXT, description, target_url: targetUrl, - }).catch((error) => { - console.log(`Could not update ${statusContext} on ${sha}: ${error.message}`); - }), - )); + }); + } catch (error) { + console.log(`Could not update ${E2E_STATUS_CONTEXT} on ${sha}: ${error.message}`); + } } /** @@ -308,12 +178,9 @@ async function removeE2ELabel({github, context}) { } module.exports = { - updateInitialStatus, - updateFinalStatus, removeE2ELabel, - formatStatusDescription, markE2EStatusesCancelled, cancelActiveE2ERuns, - E2E_STATUS_CONTEXTS, + E2E_STATUS_CONTEXT, CANCELLED_STATUS_DESCRIPTION, }; diff --git a/e2e/utils/tsio-report-status.js b/e2e/utils/tsio-report-status.js index e0f5e1cf83b..066c0245bed 100644 --- a/e2e/utils/tsio-report-status.js +++ b/e2e/utils/tsio-report-status.js @@ -2,11 +2,6 @@ // See LICENSE.txt for license information. /* eslint-disable no-console -- Logging is intentional in CI utility scripts */ -// Not test-system-io-summary: it reads /api/v1/orchestration/status, which only -// the dispatch-begin/dispatch-run flow populates. report-upload posts to a -// disjoint /api/v1/reports/* subsystem, so this re-opens the idempotent begin -// endpoint to recover the report id, then polls the public status endpoint. - const PRODUCTION_URL = 'https://test-io.test.mattermost.com'; const STAGING_URL = 'https://staging-test-io.test.mattermost.com'; @@ -69,7 +64,10 @@ async function reportTsioStatus({ throw new Error(`reports/begin failed: ${beginRes.status} ${await beginRes.text()}`); } const {report_id: reportId} = await beginRes.json(); - const reportUrl = `${baseUrl}/reports/${reportId}`; + + // /reports/{id} (no prefix) hits the frontend's repo-or-sha catch-all route, + // not the report detail page — it needs the g/ (group) prefix. + const reportUrl = `${baseUrl}/reports/g/${reportId}`; let detail; for (let attempt = 0; attempt < POLL_ATTEMPTS; attempt++) { From d7d43585e1ebe0012a0dafc2be6b730787aa03e4 Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Wed, 8 Jul 2026 07:03:46 +0530 Subject: [PATCH 09/37] just keep 1 status check --- .../compatibility-matrix-testing.yml | 8 + .github/workflows/e2e-functional-template.yml | 58 ++---- .github/workflows/e2e-functional.yml | 64 ++++++- .github/workflows/tsio-spike.yml | 172 ------------------ e2e/utils/tsio-report-status.js | 126 ++++++++----- 5 files changed, 165 insertions(+), 263 deletions(-) delete mode 100644 .github/workflows/tsio-spike.yml diff --git a/.github/workflows/compatibility-matrix-testing.yml b/.github/workflows/compatibility-matrix-testing.yml index b9588c43b2f..a770f22d7c2 100644 --- a/.github/workflows/compatibility-matrix-testing.yml +++ b/.github/workflows/compatibility-matrix-testing.yml @@ -78,7 +78,10 @@ jobs: needs: - calculate-commit-hash steps: + # Best-effort: a transient status-API hiccup here must never block the + # actual test matrix from running — the `e2e` job below needs this job. - uses: mattermost/actions/delivery/update-commit-status@218fd96a63451259dc100e1292a5e44af92fe15d + continue-on-error: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: @@ -182,6 +185,10 @@ jobs: TSIO_COMPOSITE_IDENTITY: ${{ needs.calculate-commit-hash.outputs.TSIO_COMPOSITE_IDENTITY }} TSIO_TOTAL_REPORTS_EXPECTED: ${{ needs.calculate-commit-hash.outputs.TSIO_TOTAL_REPORTS_EXPECTED }} COMMIT_STATUS_CONTEXT: e2e/compatibility-matrix-testing + # TSIO only sees test-case-level results — a job-level failure with no + # matching failed test (hung worker teardown, crashed runner, npm ci + # failure) would otherwise still read as "100% passed". + UPSTREAM_JOBS_SUCCEEDED: ${{ needs.e2e.result == 'success' }} with: script: | const {reportUrl, status, stats} = await require('./e2e/utils/tsio-report-status.js')({ @@ -189,6 +196,7 @@ jobs: compositeIdentity: JSON.parse(process.env.TSIO_COMPOSITE_IDENTITY), totalReportsExpected: parseInt(process.env.TSIO_TOTAL_REPORTS_EXPECTED, 10), commitStatusContext: process.env.COMMIT_STATUS_CONTEXT, + upstreamJobsSucceeded: process.env.UPSTREAM_JOBS_SUCCEEDED === 'true', failOnTestFailures: true, }); core.info(`TSIO report ${reportUrl}: ${status} (${JSON.stringify(stats)})`); diff --git a/.github/workflows/e2e-functional-template.yml b/.github/workflows/e2e-functional-template.yml index e741efc637b..c18894b26c8 100644 --- a/.github/workflows/e2e-functional-template.yml +++ b/.github/workflows/e2e-functional-template.yml @@ -278,41 +278,6 @@ jobs: DESKTOP_VERSION: ${{ inputs.DESKTOP_VERSION }} CI_ENVIRONMENT_NAME: ${{ env.CI_ENVIRONMENT_NAME }} - - name: e2e/generate-html-report - id: generate-html-report - if: always() - run: | - cd e2e - if [ -d blob-report ] && find blob-report -type f | grep -q .; then - npx playwright merge-reports --reporter=html blob-report - echo "html_report_ready=true" >> "$GITHUB_OUTPUT" - else - echo "No blob report produced for this OS — skipping HTML generation." - echo "html_report_ready=false" >> "$GITHUB_OUTPUT" - fi - - - name: e2e/upload-html-report-to-s3 - id: upload-html-report-to-s3 - if: always() && steps.generate-html-report.outputs.html_report_ready == 'true' - env: - AWS_ACCESS_KEY_ID: ${{ secrets.MM_DESKTOP_E2E_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.MM_DESKTOP_E2E_AWS_SECRET_ACCESS_KEY }} - AWS_REGION: us-east-1 - AWS_S3_BUCKET: mattermost-cypress-report - RUN_ID: ${{ github.run_id }} - RUN_ATTEMPT: ${{ github.run_attempt }} - run: | - # Per-OS report path so each platform's status check links to a clean, - # single-OS view — not an aggregated multi-OS report where counts and - # "skipped" totals are inflated by cross-OS tags. - S3_PREFIX="desktop-e2e/${RUN_ID}-${RUN_ATTEMPT}/${RUNNER_OS}" - aws s3 sync e2e/playwright-report/ "s3://${AWS_S3_BUCKET}/${S3_PREFIX}/" \ - --acl public-read \ - --cache-control "no-cache" - REPORT_URL="https://${AWS_S3_BUCKET}.s3.amazonaws.com/${S3_PREFIX}/index.html" - echo "report_url=${REPORT_URL}" >> "$GITHUB_OUTPUT" - echo "Playwright report (${RUNNER_OS}) uploaded: ${REPORT_URL}" >> "$GITHUB_STEP_SUMMARY" - # TSIO reporting: upload this leg's native Playwright JSON + screenshots. # Gated on tsio-config so callers that don't opt in (ad-hoc # workflow_dispatch without config) are untouched. Used by both plain @@ -333,13 +298,26 @@ jobs: json-path: e2e/test-results/results.json screenshots-dir: e2e/test-results - # Playwright's own exit code already accounts for retries — trust it - # instead of re-deriving pass/fail from the JUnit report. - - name: e2e/fail-on-test-failures + # Playwright's own exit code already accounts for retries, but a non-zero + # exit here isn't reliably a real test failure — an intermittent + # Electron/Playwright worker-teardown hang (playwright#29431) can also + # produce it with zero failed tests. When TSIO is tracking this run, + # warn instead of failing the job — TSIO's test_stats (from the JSON + # report, unaffected by this) becomes the source of truth for pass/fail. + # Without TSIO wired (tsio-config empty, e.g. an ad-hoc manual dispatch), + # there's no other failure signal at all, so this must still hard-fail. + - name: e2e/handle-nonzero-playwright-exit if: always() run: | - if [ "${PLAYWRIGHT_EXIT_CODE:-1}" != "0" ]; then - echo "Playwright exited with code ${PLAYWRIGHT_EXIT_CODE:-unset}" >&2 + if [ "${PLAYWRIGHT_EXIT_CODE:-1}" == "0" ]; then + exit 0 + fi + if [ -n "${TSIO_CONFIG}" ]; then + echo "::warning::Playwright exited with code ${PLAYWRIGHT_EXIT_CODE:-unset} — not failing the job on this alone, see TSIO report for actual test results" + else + echo "Playwright exited with code ${PLAYWRIGHT_EXIT_CODE:-unset} and no TSIO config is set for this run — failing the job since there is no other failure signal" >&2 exit 1 fi + env: + TSIO_CONFIG: ${{ inputs.tsio-config }} diff --git a/.github/workflows/e2e-functional.yml b/.github/workflows/e2e-functional.yml index 7fd1fe9fa58..fd0c9200916 100644 --- a/.github/workflows/e2e-functional.yml +++ b/.github/workflows/e2e-functional.yml @@ -84,9 +84,37 @@ jobs: fi echo "composite-identity-json=${COMPOSITE_IDENTITY}" >> "$GITHUB_OUTPUT" + # Pending status shown on the PR/commit while tests run — tsio-summary below + # flips it to success/failure once TSIO has (or hasn't) produced a report. + # Gated as a `needs` of e2e-tests/e2e-policy-tests so the pending row always + # lands before the final one (commit statuses have no ordering guarantee + # otherwise — last write wins by timestamp). + update-initial-status: + name: Set pending TSIO status + runs-on: ubuntu-22.04 + needs: + - prepare-matrix + permissions: + contents: read + statuses: write + steps: + # Best-effort: a transient status-API hiccup here must never block the + # actual test matrix from running. + - uses: mattermost/actions/delivery/update-commit-status@218fd96a63451259dc100e1292a5e44af92fe15d + continue-on-error: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + repository_full_name: ${{ github.repository }} + commit_sha: ${{ github.event.pull_request.head.sha || github.sha }} + context: e2e-test/desktop-playwright + description: "Running Electron Playwright E2E tests..." + status: pending + e2e-tests: needs: - prepare-matrix + - update-initial-status name: ${{ matrix.platform }} strategy: matrix: @@ -154,12 +182,17 @@ jobs: TSIO_COMPOSITE_IDENTITY: ${{ needs.prepare-matrix.outputs.tsio-composite-identity }} TSIO_TOTAL_REPORTS_EXPECTED: ${{ needs.prepare-matrix.outputs.tsio-total-reports-expected }} COMMIT_STATUS_CONTEXT: e2e-test/desktop-playwright + # TSIO only sees test-case-level results — a job-level failure with no + # matching failed test (hung worker teardown, crashed runner, npm ci + # failure) would otherwise still read as "100% passed". + UPSTREAM_JOBS_SUCCEEDED: ${{ needs.e2e-tests.result == 'success' && needs.e2e-policy-tests.result == 'success' }} with: script: | const {reportUrl, status, stats} = await require('./e2e/utils/tsio-report-status.js')({ core, context, github, compositeIdentity: JSON.parse(process.env.TSIO_COMPOSITE_IDENTITY), totalReportsExpected: parseInt(process.env.TSIO_TOTAL_REPORTS_EXPECTED, 10), + upstreamJobsSucceeded: process.env.UPSTREAM_JOBS_SUCCEEDED === 'true', commitStatusContext: process.env.COMMIT_STATUS_CONTEXT, failOnTestFailures: true, }); @@ -231,7 +264,9 @@ jobs: e2e-policy-tests: name: policy-tests-${{ matrix.platform }} - needs: prepare-matrix + needs: + - prepare-matrix + - update-initial-status # Runs untrusted PR code (npm ci / Playwright); grant only what the TSIO # upload needs, never pull-requests: write. permissions: @@ -255,14 +290,10 @@ jobs: run: shell: bash env: - AWS_S3_BUCKET: "mattermost-cypress-report" BRANCH: ${{ github.ref_name }} BUILD_TAG: ${{ github.sha }} MM_TEST_USER_NAME: ${{ inputs.MM_TEST_USER_NAME || secrets.MM_DESKTOP_E2E_USER_NAME }} MM_TEST_PASSWORD: ${{ inputs.MM_TEST_PASSWORD || secrets.MM_DESKTOP_E2E_USER_CREDENTIALS }} - AWS_ACCESS_KEY_ID: ${{ secrets.MM_DESKTOP_E2E_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.MM_DESKTOP_E2E_AWS_SECRET_ACCESS_KEY }} - AWS_REGION: "us-east-1" PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 NODE_ENV: "test" DEBUG_E2E: "true" @@ -402,15 +433,28 @@ jobs: json-path: e2e/test-results/results.json screenshots-dir: e2e/test-results - # Playwright's own exit code already accounts for retries — trust it - # instead of re-deriving pass/fail from the JUnit report. - - name: e2e/fail-on-test-failures + # Playwright's own exit code already accounts for retries, but a non-zero + # exit here isn't reliably a real test failure — an intermittent + # Electron/Playwright worker-teardown hang (playwright#29431) can also + # produce it with zero failed tests. When TSIO is tracking this run, + # warn instead of failing the job — TSIO's test_stats (from the JSON + # report, unaffected by this) becomes the source of truth for pass/fail. + # Without TSIO wired (composite-identity empty), there's no other + # failure signal at all, so this must still hard-fail. + - name: e2e/handle-nonzero-playwright-exit if: always() run: | - if [ "${PLAYWRIGHT_EXIT_CODE:-1}" != "0" ]; then - echo "Playwright exited with code ${PLAYWRIGHT_EXIT_CODE:-unset}" >&2 + if [ "${PLAYWRIGHT_EXIT_CODE:-1}" == "0" ]; then + exit 0 + fi + if [ -n "${TSIO_COMPOSITE_IDENTITY}" ]; then + echo "::warning::Playwright exited with code ${PLAYWRIGHT_EXIT_CODE:-unset} — not failing the job on this alone, see TSIO report for actual test results" + else + echo "Playwright exited with code ${PLAYWRIGHT_EXIT_CODE:-unset} and no TSIO config is set for this run — failing the job since there is no other failure signal" >&2 exit 1 fi + env: + TSIO_COMPOSITE_IDENTITY: ${{ needs.prepare-matrix.outputs.tsio-composite-identity }} - name: Upload test results if: always() diff --git a/.github/workflows/tsio-spike.yml b/.github/workflows/tsio-spike.yml deleted file mode 100644 index da1cb2d3e72..00000000000 --- a/.github/workflows/tsio-spike.yml +++ /dev/null @@ -1,172 +0,0 @@ -# Spike: validate the test-system-io-report-upload + test-system-io-summary chain -# on a throwaway Playwright run BEFORE wiring it into compatibility-matrix-testing.yml. -# -# Why standalone: production CMT needs Matterwick-provisioned servers + a built -# Electron bundle. This spike runs two trivial Playwright specs across 2 shards -# to exercise the only unproven part of the TSIO path — test-system-io-report-upload -# has zero production callers (only TSIO's own self-test). If group finalization + -# the consolidated summary render here, the desktop CMT wiring is safe to ship. -# -# Run: workflow_dispatch on a branch. No secrets required — auth is OIDC -# (permissions: id-token: write). Defaults to TSIO staging so we don't pollute prod. -# Safe to delete this file once the spike passes and the real CMT is wired to TSIO. -name: TSIO Spike - -on: - workflow_dispatch: - inputs: - use-staging: - description: "Target TSIO staging (true) instead of production" - required: false - default: "true" - type: boolean - fail-on-test-failures: - description: "Fail the workflow if any shard failed (leave false to see the summary on a red run)" - required: false - default: "false" - type: boolean - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -# Workflow-level default; jobs below narrow to least-privilege. -permissions: - contents: read - -jobs: - prepare: - runs-on: ubuntu-22.04 - outputs: - composite-identity-json: ${{ steps.identity.outputs.composite-identity-json }} - total-reports-expected: "2" - steps: - - name: Build composite identity - id: identity - env: - GITHUB_REPOSITORY: ${{ github.repository }} - MM_SHA: ${{ github.sha }} - MM_BRANCH: ${{ github.ref_name }} - run: | - # `name` groups the report on the TSIO dashboard. Distinct from any real - # CMT/PR context so the spike never collides with production report groups. - COMPOSITE_IDENTITY=$(jq -nc \ - --arg repo "${GITHUB_REPOSITORY}" \ - --arg sha "${MM_SHA}" \ - --arg run_id "${GITHUB_RUN_ID}" \ - --arg attempt "${GITHUB_RUN_ATTEMPT}" \ - --arg name "tsio-spike-desktop" \ - --arg branch "${MM_BRANCH}" \ - '{repository:$repo, commit_sha:$sha, gh_run_id:$run_id, name:$name, gh_run_attempt:$attempt, branch:$branch}') - echo "composite-identity-json=${COMPOSITE_IDENTITY}" >> "$GITHUB_OUTPUT" - echo "$COMPOSITE_IDENTITY" | jq . - - shards: - name: shard-${{ matrix.shard }} - needs: prepare - runs-on: ubuntu-22.04 - permissions: - contents: read - id-token: write # TSIO auth via OIDC - actions: read # report-upload resolves gh_job_id via the GitHub API - strategy: - fail-fast: false - matrix: - shard: [1, 2] - steps: - - name: Set up Node - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - node-version: '22.x' - - - name: Scaffold a tiny Playwright project - run: | - mkdir -p spike/tests - cat > spike/package.json <<'PKG' - { "name": "tsio-spike", "private": true, "type": "module" } - PKG - cat > spike/playwright.config.ts <<'CFG' - import {defineConfig, devices} from '@playwright/test'; - export default defineConfig({ - testDir: './tests', - fullyParallel: false, - workers: 1, - retries: 0, - reporter: [['line'], ['json', {outputFile: 'results.json'}], ['blob', {outputDir: 'blob-report'}]], - outputDir: './output', - use: {trace: 'off', screenshot: 'only-on-failure'}, - projects: [{name: 'chromium', use: {...devices['Desktop Chrome']}}], - }); - CFG - # Shard 1 gets a passing file; shard 2 gets pass + intentional fail + skip, - # so the consolidated summary has something real to render across legs. - cat > spike/tests/a.spec.ts <<'SPEC' - import {test, expect} from '@playwright/test'; - test('spike-pass-a', async () => { expect(1 + 1).toBe(2); }); - SPEC - cat > spike/tests/b.spec.ts <<'SPEC' - import {test, expect} from '@playwright/test'; - test('spike-pass-b', async () => { expect('ok').toBe('ok'); }); - test('spike-fail-b', async () => { expect(1).toBe(2); }); - test.skip('spike-skip-b', async () => {}); - SPEC - - - name: Install Playwright + browser - working-directory: spike - run: | - npm i -D @playwright/test - # No --with-deps: that runs apt-get, which flaked on ubuntu-22.04 - # (packages.microsoft.com NOSPLIT). Runner ships the needed libs. - npx playwright install chromium - - - name: Run tests (sharded) - continue-on-error: true - working-directory: spike - run: npx playwright test --shard=${{ matrix.shard }}/2 - - - name: Upload shard report to TSIO - if: ${{ always() && hashFiles('spike/results.json') != '' }} - uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-report-upload@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 0.10.0 / 2026-05-16 - with: - use-staging: ${{ inputs.use-staging }} - composite-identity: ${{ needs.prepare.outputs.composite-identity-json }} - total-reports-expected: ${{ needs.prepare.outputs.total-reports-expected }} - framework: playwright - github-token: ${{ secrets.GITHUB_TOKEN }} - # MUST match this job's `name:` field (shard-1 / shard-2). - gh-job-name: shard-${{ matrix.shard }} - json-path: spike/results.json - screenshots-dir: spike/output - - summary: - name: tsio-summary - needs: [prepare, shards] - if: always() - runs-on: ubuntu-22.04 - permissions: - contents: read - id-token: write - statuses: write # summary flips the pending→success/failure commit status - actions: read - steps: - - name: Render consolidated TSIO summary - id: summary - uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-summary@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 0.10.0 / 2026-05-16 - with: - use-staging: ${{ inputs.use-staging }} - composite-identity: ${{ needs.prepare.outputs.composite-identity-json }} - framework: playwright - commit-status-context: tsio-spike/desktop - fail-on-test-failures: ${{ inputs.fail-on-test-failures }} - github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Show outputs - if: always() - env: - DESC: ${{ steps.summary.outputs.commit_status_description }} - PAYLOAD: ${{ steps.summary.outputs.webhook_payload }} - run: | - echo "commit_status_description:" - echo "$DESC" - echo - echo "webhook_payload:" - echo "$PAYLOAD" \ No newline at end of file diff --git a/e2e/utils/tsio-report-status.js b/e2e/utils/tsio-report-status.js index 066c0245bed..12b7f85047e 100644 --- a/e2e/utils/tsio-report-status.js +++ b/e2e/utils/tsio-report-status.js @@ -25,6 +25,11 @@ const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); * @param {boolean} [params.failOnTestFailures] - When true (default), throw if the group didn't complete cleanly * @param {boolean} [params.useStaging] - Target TSIO staging instead of production * @param {string} [params.oidcAudience] - OIDC audience claim TSIO expects + * @param {boolean} [params.upstreamJobsSucceeded] - When false (default true), force the + * commit status to failure regardless of TSIO's test stats. TSIO only sees test-case-level + * results, so a job-level failure with no failing test attached to it (e.g. a hung worker + * teardown, a crashed runner, npm ci failing before any test ran) would otherwise still + * read as "100% passed" here even though the actual CI run failed. * @returns {Promise<{reportUrl: string, status: string, stats: Object}>} */ async function reportTsioStatus({ @@ -37,57 +42,89 @@ async function reportTsioStatus({ failOnTestFailures = true, useStaging = false, oidcAudience = 'mattermost-test-system-io', + upstreamJobsSucceeded = true, }) { const baseUrl = useStaging ? STAGING_URL : PRODUCTION_URL; - const idToken = await core.getIDToken(oidcAudience); - core.setSecret(idToken); - - const beginRes = await fetch(`${baseUrl}/api/v1/reports/begin`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${idToken}`, - }, - body: JSON.stringify({ - repository: compositeIdentity.repository, - commit: compositeIdentity.commit_sha, - gh_run_id: compositeIdentity.gh_run_id, - gh_run_attempt: compositeIdentity.gh_run_attempt, - framework: 'playwright', - name: compositeIdentity.name, - branch: compositeIdentity.branch, - total_reports_expected: totalReportsExpected, - ...(compositeIdentity.gh_pr_number ? {gh_pr_number: parseInt(compositeIdentity.gh_pr_number, 10)} : {}), - }), - }); - if (!beginRes.ok) { - throw new Error(`reports/begin failed: ${beginRes.status} ${await beginRes.text()}`); - } - const {report_id: reportId} = await beginRes.json(); - - // /reports/{id} (no prefix) hits the frontend's repo-or-sha catch-all route, - // not the report detail page — it needs the g/ (group) prefix. - const reportUrl = `${baseUrl}/reports/g/${reportId}`; + // Fallback target for the commit status when no TSIO report ever gets + // created (begin/poll failed outright) — the reviewer still needs + // somewhere to click instead of a stuck `pending` row. + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + let reportId; + let reportUrl; let detail; - for (let attempt = 0; attempt < POLL_ATTEMPTS; attempt++) { - const statusRes = await fetch(`${baseUrl}/api/v1/reports/${reportId}`); - if (!statusRes.ok) { - throw new Error(`reports/${reportId} failed: ${statusRes.status} ${await statusRes.text()}`); + try { + const idToken = await core.getIDToken(oidcAudience); + core.setSecret(idToken); + + const beginRes = await fetch(`${baseUrl}/api/v1/reports/begin`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${idToken}`, + }, + body: JSON.stringify({ + repository: compositeIdentity.repository, + commit: compositeIdentity.commit_sha, + gh_run_id: compositeIdentity.gh_run_id, + gh_run_attempt: compositeIdentity.gh_run_attempt, + framework: 'playwright', + name: compositeIdentity.name, + branch: compositeIdentity.branch, + total_reports_expected: totalReportsExpected, + ...(compositeIdentity.gh_pr_number ? {gh_pr_number: parseInt(compositeIdentity.gh_pr_number, 10)} : {}), + }), + }); + if (!beginRes.ok) { + throw new Error(`reports/begin failed: ${beginRes.status} ${await beginRes.text()}`); } - detail = await statusRes.json(); - if (TERMINAL_STATUSES.includes(detail.status)) { - break; + ({report_id: reportId} = await beginRes.json()); + + // /reports/{id} (no prefix) hits the frontend's repo-or-sha catch-all route, + // not the report detail page — it needs the g/ (group) prefix. + reportUrl = `${baseUrl}/reports/g/${reportId}`; + + for (let attempt = 0; attempt < POLL_ATTEMPTS; attempt++) { + const statusRes = await fetch(`${baseUrl}/api/v1/reports/${reportId}`); + if (!statusRes.ok) { + throw new Error(`reports/${reportId} failed: ${statusRes.status} ${await statusRes.text()}`); + } + detail = await statusRes.json(); + if (TERMINAL_STATUSES.includes(detail.status)) { + break; + } + if (attempt < POLL_ATTEMPTS - 1) { + await sleep(POLL_DELAY_MS); + } } - if (attempt < POLL_ATTEMPTS - 1) { - await sleep(POLL_DELAY_MS); + } catch (error) { + // Prefer the report link if begin already succeeded (the poll loop + // itself failed) — only fall back to the workflow run when no + // report exists at all. Swallow a secondary status-API failure so + // it doesn't mask the original error. The description is public + // (visible on the PR/commit), so keep the raw error out of it — + // log it instead and point readers at the workflow run for detail. + core.error(`TSIO reporting error: ${error.message}`); + try { + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: compositeIdentity.commit_sha, + state: 'failure', + context: commitStatusContext, + description: 'TSIO reporting error — see workflow run for details', + target_url: reportUrl || runUrl, + }); + } catch (statusError) { + core.warning(`Failed to create failure commit status: ${statusError.message}`); } + throw error; } const stats = detail.test_stats || {}; const isComplete = detail.status === 'completed'; const hasFailures = (stats.failed || 0) > 0; - const overallState = isComplete && !hasFailures ? 'success' : 'failure'; + const overallState = isComplete && !hasFailures && upstreamJobsSucceeded ? 'success' : 'failure'; const summaryLines = [ `### Test System IO — ${compositeIdentity.name}`, @@ -95,11 +132,15 @@ async function reportTsioStatus({ `**Status:** ${detail.status} · **Report:** [${reportId}](${reportUrl})`, `**Tests:** ${stats.passed ?? '?'} passed, ${stats.failed ?? '?'} failed, ${stats.flaky ?? 0} flaky, ` + `${stats.skipped ?? '?'} skipped (of ${stats.total ?? '?'})`, + ...(!upstreamJobsSucceeded ? + ['', ':warning: One or more CI jobs failed outside of any tracked test (e.g. a hung worker, a crashed runner) — forcing this status to failure even though the test stats above may show no failures.'] : + []), '', ]; await core.summary.addRaw(summaryLines.join('\n')).write(); - const description = `${stats.passed ?? 0}/${stats.total ?? 0} passed, ${stats.failed ?? 0} failed`.slice(0, 140); + const descriptionPrefix = !upstreamJobsSucceeded ? 'CI job failed (untracked by TSIO), ' : ''; + const description = `${descriptionPrefix}${stats.passed ?? 0}/${stats.total ?? 0} passed, ${stats.failed ?? 0} failed`.slice(0, 140); await github.rest.repos.createCommitStatus({ owner: context.repo.owner, repo: context.repo.repo, @@ -111,7 +152,10 @@ async function reportTsioStatus({ }); if (failOnTestFailures && overallState === 'failure') { - throw new Error(`TSIO report ${reportId} did not pass: status=${detail.status}, failed=${stats.failed || 0}`); + const reason = !upstreamJobsSucceeded && !hasFailures ? + 'an upstream CI job failed with no corresponding test failure' : + `status=${detail.status}, failed=${stats.failed || 0}`; + throw new Error(`TSIO report ${reportId} did not pass: ${reason}`); } return {reportUrl, status: detail.status, stats}; From 0dbac2ded815dc3ef5c061290391557dd4f99268 Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Wed, 8 Jul 2026 10:54:56 +0530 Subject: [PATCH 10/37] lint --- .github/workflows/e2e-pr-trigger.yml | 46 ++++++++-------------------- e2e/utils/tsio-report-status.js | 9 +++--- 2 files changed, 17 insertions(+), 38 deletions(-) diff --git a/.github/workflows/e2e-pr-trigger.yml b/.github/workflows/e2e-pr-trigger.yml index 3782bacda8b..75bde04b351 100644 --- a/.github/workflows/e2e-pr-trigger.yml +++ b/.github/workflows/e2e-pr-trigger.yml @@ -56,18 +56,13 @@ jobs: !github.event.pull_request.draft && contains(fromJSON('["opened", "reopened", "ready_for_review", "synchronize"]'), github.event.action) steps: + # Always the base ref's own reviewed copy of github-actions.js — this job + # holds write-scoped tokens (issues/pull-requests/actions/statuses), so it + # must never execute code sourced from the untrusted PR head. - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ github.event.pull_request.base.ref }} - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - if: github.event.pull_request.head.repo.full_name == github.repository - with: - ref: ${{ github.event.pull_request.head.sha }} - sparse-checkout: | - e2e/utils/github-actions.js - sparse-checkout-cone-mode: false - - name: Cancel running E2E tests and re-trigger uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: @@ -157,18 +152,13 @@ jobs: github.event.action == 'labeled' && github.event.label.name == 'E2E/Override' steps: + # Always the base ref's own reviewed copy of github-actions.js — this job + # holds write-scoped tokens (issues/pull-requests/actions/statuses), so it + # must never execute code sourced from the untrusted PR head. - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ github.event.pull_request.base.ref }} - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - if: github.event.pull_request.head.repo.full_name == github.repository - with: - ref: ${{ github.event.pull_request.head.sha }} - sparse-checkout: | - e2e/utils/github-actions.js - sparse-checkout-cone-mode: false - - name: Strip E2E/Run and cancel in-flight E2E uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: @@ -217,28 +207,16 @@ jobs: && github.event.label.name == 'E2E/Run' && github.event.sender.login != 'github-actions[bot]' steps: + # Always the base ref's own reviewed copy of github-actions.js — this job + # holds write-scoped tokens (actions/statuses), so it must never execute + # code sourced from the untrusted PR head. A single full checkout also + # sidesteps the previous bug where a second sparse checkout of head.sha + # replaced the base-ref tree, wiping out .github/actions/cancel-e2e-runs/ + # before this job's next step could use it. - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ github.event.pull_request.base.ref }} - # Sparse-checked into a separate path, not the default workspace: a plain - # second checkout of head.sha here would replace the base-ref tree with a - # sparse one containing only this file — wiping out - # .github/actions/cancel-e2e-runs/ before the step below can use it. - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - if: github.event.pull_request.head.repo.full_name == github.repository - with: - ref: ${{ github.event.pull_request.head.sha }} - persist-credentials: false - sparse-checkout: | - e2e/utils/github-actions.js - sparse-checkout-cone-mode: false - path: untrusted-head - - - name: Use PR's github-actions.js - if: github.event.pull_request.head.repo.full_name == github.repository - run: cp untrusted-head/e2e/utils/github-actions.js e2e/utils/github-actions.js - - name: Cancel E2E runs and mark statuses skipped uses: ./.github/actions/cancel-e2e-runs with: diff --git a/e2e/utils/tsio-report-status.js b/e2e/utils/tsio-report-status.js index 12b7f85047e..1b94cb86048 100644 --- a/e2e/utils/tsio-report-status.js +++ b/e2e/utils/tsio-report-status.js @@ -45,6 +45,7 @@ async function reportTsioStatus({ upstreamJobsSucceeded = true, }) { const baseUrl = useStaging ? STAGING_URL : PRODUCTION_URL; + // Fallback target for the commit status when no TSIO report ever gets // created (begin/poll failed outright) — the reviewer still needs // somewhere to click instead of a stuck `pending` row. @@ -132,14 +133,14 @@ async function reportTsioStatus({ `**Status:** ${detail.status} · **Report:** [${reportId}](${reportUrl})`, `**Tests:** ${stats.passed ?? '?'} passed, ${stats.failed ?? '?'} failed, ${stats.flaky ?? 0} flaky, ` + `${stats.skipped ?? '?'} skipped (of ${stats.total ?? '?'})`, - ...(!upstreamJobsSucceeded ? - ['', ':warning: One or more CI jobs failed outside of any tracked test (e.g. a hung worker, a crashed runner) — forcing this status to failure even though the test stats above may show no failures.'] : - []), + ...(upstreamJobsSucceeded ? + [] : + ['', ':warning: One or more CI jobs failed outside of any tracked test (e.g. a hung worker, a crashed runner) — forcing this status to failure even though the test stats above may show no failures.']), '', ]; await core.summary.addRaw(summaryLines.join('\n')).write(); - const descriptionPrefix = !upstreamJobsSucceeded ? 'CI job failed (untracked by TSIO), ' : ''; + const descriptionPrefix = upstreamJobsSucceeded ? '' : 'CI job failed (untracked by TSIO), '; const description = `${descriptionPrefix}${stats.passed ?? 0}/${stats.total ?? 0} passed, ${stats.failed ?? 0} failed`.slice(0, 140); await github.rest.repos.createCommitStatus({ owner: context.repo.owner, From dc32d513e540a17c18eeaf5d89479f49497ec846 Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Wed, 8 Jul 2026 12:24:51 +0530 Subject: [PATCH 11/37] fix status update --- .github/workflows/compatibility-matrix-testing.yml | 3 +++ .github/workflows/e2e-functional-template.yml | 9 ++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/compatibility-matrix-testing.yml b/.github/workflows/compatibility-matrix-testing.yml index a770f22d7c2..32551e2e792 100644 --- a/.github/workflows/compatibility-matrix-testing.yml +++ b/.github/workflows/compatibility-matrix-testing.yml @@ -77,6 +77,9 @@ jobs: runs-on: ubuntu-22.04 needs: - calculate-commit-hash + permissions: + contents: read + statuses: write steps: # Best-effort: a transient status-API hiccup here must never block the # actual test matrix from running — the `e2e` job below needs this job. diff --git a/.github/workflows/e2e-functional-template.yml b/.github/workflows/e2e-functional-template.yml index c18894b26c8..20493bd373e 100644 --- a/.github/workflows/e2e-functional-template.yml +++ b/.github/workflows/e2e-functional-template.yml @@ -102,7 +102,10 @@ env: jobs: e2e: - name: e2e-on-${{ inputs.runs-on }} + # Includes MM_SERVER_VERSION, not just runs-on: CMT tests multiple server + # versions against the same runner (e.g. macos-13 x 9.6.1 and macos-13 x + # 9.5.2) + name: e2e-on-${{ inputs.runs-on }}-${{ inputs.MM_SERVER_VERSION }} runs-on: ${{ inputs.runs-on }} # Runs untrusted PR code (npm ci / Playwright) with a read-only token. # id-token/actions:read are additive for TSIO reporting (OIDC auth + @@ -293,8 +296,8 @@ jobs: total-reports-expected: ${{ fromJSON(inputs.tsio-config).total_reports_expected }} framework: playwright github-token: ${{ secrets.GITHUB_TOKEN }} - # MUST match this job's `name:` field above (e2e-on-). - gh-job-name: e2e-on-${{ inputs.runs-on }} + # MUST match this job's `name:` field above. + gh-job-name: e2e-on-${{ inputs.runs-on }}-${{ inputs.MM_SERVER_VERSION }} json-path: e2e/test-results/results.json screenshots-dir: e2e/test-results From 7f92cadac974f7510e7e31b4337a425a6d00323e Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Wed, 8 Jul 2026 13:17:24 +0530 Subject: [PATCH 12/37] fix status update to include skipped --- e2e/helpers/userAttributes.ts | 7 ++++++- e2e/specs/system_tray_icon/tray_menu.test.ts | 2 +- e2e/utils/tsio-report-status.js | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/e2e/helpers/userAttributes.ts b/e2e/helpers/userAttributes.ts index 7c967493d19..8479194b6c1 100644 --- a/e2e/helpers/userAttributes.ts +++ b/e2e/helpers/userAttributes.ts @@ -305,7 +305,12 @@ export async function editTextCustomAttribute( await win.runInRenderer(` const fieldId = ${JSON.stringify(fieldId)}; const input = document.querySelector('#customAttribute_' + fieldId); - const row = input?.closest('.setting-list-item, .SettingsBlock, section, li, div') || document; + const row = input?.closest('.setting-list-item') || + input?.closest('.SettingsBlock') || + input?.closest('section') || + input?.closest('li') || + input?.closest('div') || + document; const saveBtn = Array.from(row.querySelectorAll('button')) .find((button) => (button.textContent || '').trim() === 'Save'); if (!(saveBtn instanceof HTMLButtonElement)) { diff --git a/e2e/specs/system_tray_icon/tray_menu.test.ts b/e2e/specs/system_tray_icon/tray_menu.test.ts index 0ae9e8aa1b9..ec02d50bf6b 100644 --- a/e2e/specs/system_tray_icon/tray_menu.test.ts +++ b/e2e/specs/system_tray_icon/tray_menu.test.ts @@ -19,7 +19,7 @@ test.describe('system_tray_icon/tray_menu', () => { test( 'TRAY-01 tray icon click restores hidden window when minimizeToTray is enabled', - {tag: ['@P0', '@all']}, + {tag: ['@P0', '@linux', '@win32']}, async ({electronApp}) => { await expect.poll( () => isMainWindowVisible(electronApp), diff --git a/e2e/utils/tsio-report-status.js b/e2e/utils/tsio-report-status.js index 1b94cb86048..ad22729fca9 100644 --- a/e2e/utils/tsio-report-status.js +++ b/e2e/utils/tsio-report-status.js @@ -141,7 +141,7 @@ async function reportTsioStatus({ await core.summary.addRaw(summaryLines.join('\n')).write(); const descriptionPrefix = upstreamJobsSucceeded ? '' : 'CI job failed (untracked by TSIO), '; - const description = `${descriptionPrefix}${stats.passed ?? 0}/${stats.total ?? 0} passed, ${stats.failed ?? 0} failed`.slice(0, 140); + const description = `${descriptionPrefix}${stats.passed ?? 0}/${stats.total ?? 0} passed, ${stats.failed ?? 0} failed, ${stats.skipped ?? 0} skipped`.slice(0, 140); await github.rest.repos.createCommitStatus({ owner: context.repo.owner, repo: context.repo.repo, From fcbdca42a9e48539130f9423f6f68f271d8ca7f5 Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Wed, 8 Jul 2026 16:43:31 +0530 Subject: [PATCH 13/37] clean up --- .../compatibility-matrix-testing.yml | 15 --- .github/workflows/e2e-functional-template.yml | 18 --- .github/workflows/e2e-functional.yml | 38 ------ e2e/helpers/serverContext.ts | 88 ++++++++----- e2e/helpers/userAttributes.ts | 31 +++-- e2e/specs/calls/calls_functionality.test.ts | 9 ++ e2e/specs/multi_window/multi_window.test.ts | 16 ++- e2e/specs/permissions/permissions_ipc.test.ts | 123 +++++++++++------- .../server_management/popout_windows.test.ts | 17 ++- e2e/utils/github-actions.js | 4 - e2e/utils/tsio-report-status.js | 12 +- 11 files changed, 194 insertions(+), 177 deletions(-) diff --git a/.github/workflows/compatibility-matrix-testing.yml b/.github/workflows/compatibility-matrix-testing.yml index 32551e2e792..83efe322175 100644 --- a/.github/workflows/compatibility-matrix-testing.yml +++ b/.github/workflows/compatibility-matrix-testing.yml @@ -130,12 +130,6 @@ jobs: strategy: fail-fast: false matrix: ${{ fromJson(inputs.CMT_MATRIX) }} - # A reusable workflow can only receive permissions the calling job itself - # already holds — the nested e2e job's TSIO reporting steps need - # actions:read + id-token:write (same fix as e2e-functional.yml's e2e-tests - # job; this workflow has no top-level `permissions:` default, so without - # this the nested job would hit the same "requesting actions:read, - # id-token:write but only allowed none" rejection e2e-functional.yml hit). permissions: contents: read actions: read @@ -152,15 +146,6 @@ jobs: # workflow_call caps reusable workflows at 10 inputs total. tsio-config: ${{ format('{{"composite_identity":{0},"total_reports_expected":"{1}"}}', needs.calculate-commit-hash.outputs.TSIO_COMPOSITE_IDENTITY, needs.calculate-commit-hash.outputs.TSIO_TOTAL_REPORTS_EXPECTED) }} - # Consolidated final status + rollup summary, via Test System IO instead of - # the hand-rolled artifact-based rollup. One TSIO report group covers every - # OS x server-version leg of the CMT run; commit status target_url deep-links - # that dashboard page. Context name unchanged (e2e/compatibility-matrix-testing). - # - # Uses e2e/utils/tsio-report-status.js, not test-system-io-summary — summary - # only reads /api/v1/orchestration/status, which report-upload never - # populates (see that file's header comment). - # https://mattermost.atlassian.net/browse/CLD-5815 update-final-status: runs-on: ubuntu-22.04 if: always() diff --git a/.github/workflows/e2e-functional-template.yml b/.github/workflows/e2e-functional-template.yml index 20493bd373e..926c53ba8f1 100644 --- a/.github/workflows/e2e-functional-template.yml +++ b/.github/workflows/e2e-functional-template.yml @@ -107,9 +107,6 @@ jobs: # 9.5.2) name: e2e-on-${{ inputs.runs-on }}-${{ inputs.MM_SERVER_VERSION }} runs-on: ${{ inputs.runs-on }} - # Runs untrusted PR code (npm ci / Playwright) with a read-only token. - # id-token/actions:read are additive for TSIO reporting (OIDC auth + - # gh_job_id lookup) — still no write access to repo contents. permissions: contents: read id-token: write @@ -281,13 +278,6 @@ jobs: DESKTOP_VERSION: ${{ inputs.DESKTOP_VERSION }} CI_ENVIRONMENT_NAME: ${{ env.CI_ENVIRONMENT_NAME }} - # TSIO reporting: upload this leg's native Playwright JSON + screenshots. - # Gated on tsio-config so callers that don't opt in (ad-hoc - # workflow_dispatch without config) are untouched. Used by both plain - # PR/master runs and CMT — the caller decides grouping via `name` inside - # composite_identity and computes total_reports_expected once for the - # whole run's matrix. Packed into one JSON input (see tsio-config above) - # to stay under workflow_call's 10-input cap. - name: e2e/upload-report-to-tsio if: ${{ always() && inputs.tsio-config != '' && hashFiles('e2e/test-results/results.json') != '' }} uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-report-upload@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 0.10.0 / 2026-05-16 @@ -301,14 +291,6 @@ jobs: json-path: e2e/test-results/results.json screenshots-dir: e2e/test-results - # Playwright's own exit code already accounts for retries, but a non-zero - # exit here isn't reliably a real test failure — an intermittent - # Electron/Playwright worker-teardown hang (playwright#29431) can also - # produce it with zero failed tests. When TSIO is tracking this run, - # warn instead of failing the job — TSIO's test_stats (from the JSON - # report, unaffected by this) becomes the source of truth for pass/fail. - # Without TSIO wired (tsio-config empty, e.g. an ad-hoc manual dispatch), - # there's no other failure signal at all, so this must still hard-fail. - name: e2e/handle-nonzero-playwright-exit if: always() run: | diff --git a/.github/workflows/e2e-functional.yml b/.github/workflows/e2e-functional.yml index fd0c9200916..1090b547e1e 100644 --- a/.github/workflows/e2e-functional.yml +++ b/.github/workflows/e2e-functional.yml @@ -56,9 +56,6 @@ jobs: platforms=$(echo "${INSTANCE_DETAILS}" | jq -c 'map(if .runner == "macos-latest" then .runner = "macos-26" else . end)') echo "platforms=${platforms}" >> "$GITHUB_OUTPUT" - # One composite identity + total-reports-expected for the whole run - # (every e2e-tests platform leg, plus the 2 fixed e2e-policy-tests legs), - # so all legs land in a single TSIO report group. - id: tsio-identity env: GITHUB_REPOSITORY: ${{ github.repository }} @@ -84,11 +81,6 @@ jobs: fi echo "composite-identity-json=${COMPOSITE_IDENTITY}" >> "$GITHUB_OUTPUT" - # Pending status shown on the PR/commit while tests run — tsio-summary below - # flips it to success/failure once TSIO has (or hasn't) produced a report. - # Gated as a `needs` of e2e-tests/e2e-policy-tests so the pending row always - # lands before the final one (commit statuses have no ordering guarantee - # otherwise — last write wins by timestamp). update-initial-status: name: Set pending TSIO status runs-on: ubuntu-22.04 @@ -120,10 +112,6 @@ jobs: matrix: include: ${{ fromJson(needs.prepare-matrix.outputs.platforms) }} fail-fast: false - # The reusable template runs untrusted PR code (npm ci / Playwright) with a read-only - # token; actions:read + id-token:write are additive, needed only to grant the nested - # e2e job's TSIO reporting steps (OIDC auth + gh_job_id lookup) — a reusable workflow - # can only receive permissions the calling job itself already holds. permissions: contents: read actions: read @@ -136,13 +124,7 @@ jobs: MM_SERVER_VERSION: ${{ inputs.MM_SERVER_VERSION }} MM_TEST_USER_NAME: ${{ inputs.MM_TEST_USER_NAME }} MM_TEST_PASSWORD: ${{ inputs.MM_TEST_PASSWORD }} - # When matterwick fires this workflow it sets run_type explicitly (MASTER for master - # pushes; PR-label runs are dispatched without run_type, defaulting to PR via the `|| 'PR'` - # below). The previous `startsWith(version_name, 'release-') && 'RELEASE'` fallback was - # for matterwick's release-branch push, which is no longer wired — removed. TYPE: ${{ inputs.run_type || 'PR' }} - # Packed as one JSON input (see e2e-functional-template.yml's tsio-config) — - # workflow_call caps reusable workflows at 10 inputs total. tsio-config: ${{ format('{{"composite_identity":{0},"total_reports_expected":"{1}"}}', needs.prepare-matrix.outputs.tsio-composite-identity, needs.prepare-matrix.outputs.tsio-total-reports-expected) }} secrets: inherit @@ -182,9 +164,6 @@ jobs: TSIO_COMPOSITE_IDENTITY: ${{ needs.prepare-matrix.outputs.tsio-composite-identity }} TSIO_TOTAL_REPORTS_EXPECTED: ${{ needs.prepare-matrix.outputs.tsio-total-reports-expected }} COMMIT_STATUS_CONTEXT: e2e-test/desktop-playwright - # TSIO only sees test-case-level results — a job-level failure with no - # matching failed test (hung worker teardown, crashed runner, npm ci - # failure) would otherwise still read as "100% passed". UPSTREAM_JOBS_SUCCEEDED: ${{ needs.e2e-tests.result == 'success' && needs.e2e-policy-tests.result == 'success' }} with: script: | @@ -324,8 +303,6 @@ jobs: with: node-version: '22.x' package-manager-cache: false - # node_modules is cached in e2e/cache-node-modules; disable setup-node's - # npm cache so actions/cache v5 does not hit legacy entries. - name: e2e/use-gnu-tar-macos if: runner.os == 'macOS' @@ -361,16 +338,9 @@ jobs: - name: e2e/setup-python uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 env: - # Bypass pip's on-disk HTTP cache entirely so a stale/corrupt entry on the - # runner image cannot trigger deserialization warnings (actions/setup-python#1317). PIP_NO_CACHE_DIR: "1" with: - # Use a version pre-installed on macOS-26 (3.13 is in its toolcache), so - # setup-python doesn't install a mismatched interpreter whose pip can't - # deserialize the image's pre-baked cache. PIP_NO_CACHE_DIR above is the - # remaining guard for non-macOS runners. python-version: "3.13" - # Omit `cache` — setup-python only accepts pip/pipenv/poetry, not `false`. - name: e2e/install-os-dependencies uses: ./.github/actions/install-os-dependencies @@ -433,14 +403,6 @@ jobs: json-path: e2e/test-results/results.json screenshots-dir: e2e/test-results - # Playwright's own exit code already accounts for retries, but a non-zero - # exit here isn't reliably a real test failure — an intermittent - # Electron/Playwright worker-teardown hang (playwright#29431) can also - # produce it with zero failed tests. When TSIO is tracking this run, - # warn instead of failing the job — TSIO's test_stats (from the JSON - # report, unaffected by this) becomes the source of truth for pass/fail. - # Without TSIO wired (composite-identity empty), there's no other - # failure signal at all, so this must still hard-fail. - name: e2e/handle-nonzero-playwright-exit if: always() run: | diff --git a/e2e/helpers/serverContext.ts b/e2e/helpers/serverContext.ts index abde6819864..cf5509a9de5 100644 --- a/e2e/helpers/serverContext.ts +++ b/e2e/helpers/serverContext.ts @@ -9,6 +9,8 @@ import {closeOverlayWindowsIfOpen} from './overlayWindows'; import type {ServerEntry, ServerMap} from './serverMap'; import {evaluateInMainProcessWithArg} from './testRefs'; +const NOT_YET_REGISTERED_PREFIX = 'No server view registered for webContentsId'; + /** * Make a specific server WebContentsView the active, focused target for automation * and application menu handlers (History, View, Find, etc.). @@ -23,40 +25,60 @@ export async function activateServerView( ): Promise { await closeOverlayWindowsIfOpen(app); - await evaluateInMainProcessWithArg(app, ({webContents, BrowserWindow}, id) => { - const refs = (global as any).__e2eTestRefs; - if (!refs) { - throw new Error('__e2eTestRefs is not available'); - } - - const mmView = refs.WebContentsManager.getViewByWebContentsId(id); - if (!mmView) { - throw new Error(`No server view registered for webContentsId ${id}`); - } - - const tabView = refs.ViewManager.getView(mmView.id); - if (tabView) { - refs.ServerManager.updateCurrentServer(tabView.serverId); - refs.TabManager.switchToTab(tabView.id); - } - - refs.TabManager.focusCurrentTab(); - - const wc = webContents.fromId(id); - if (!wc || wc.isDestroyed()) { - throw new Error(`webContents ${id} is not available`); + // WebContentsManager keeps two separate indexes: by internal view id (what + // buildServerMap reads to resolve this webContentsId in the first place) + // and by webContentsId (what this function needs). The second index can + // still be a beat behind the first right after a tab is created, so a + // caller that just got this id from buildServerMap can hit a real but + // transient "not registered yet" here. Poll specifically for that error; + // any other failure (e.g. the webContents was actually destroyed) still + // fails immediately. + const deadline = Date.now() + 10_000; + for (;;) { + try { + await evaluateInMainProcessWithArg(app, ({webContents, BrowserWindow}, id) => { + const refs = (global as any).__e2eTestRefs; + if (!refs) { + throw new Error('__e2eTestRefs is not available'); + } + + const mmView = refs.WebContentsManager.getViewByWebContentsId(id); + if (!mmView) { + throw new Error(`No server view registered for webContentsId ${id}`); + } + + const tabView = refs.ViewManager.getView(mmView.id); + if (tabView) { + refs.ServerManager.updateCurrentServer(tabView.serverId); + refs.TabManager.switchToTab(tabView.id); + } + + refs.TabManager.focusCurrentTab(); + + const wc = webContents.fromId(id); + if (!wc || wc.isDestroyed()) { + throw new Error(`webContents ${id} is not available`); + } + wc.focus(); + + // Menu handlers read this when the app menu blurs the webContents (macOS/Windows). + refs.WebContentsManager.focusedWebContentsView = mmView.id; + + const mainWindow = refs.MainWindow.get() ?? BrowserWindow.getAllWindows().find((win) => { + return !win.isDestroyed() && win.webContents.getURL().includes('index'); + }); + mainWindow?.show(); + mainWindow?.focus(); + }, webContentsId); + break; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (!message.includes(NOT_YET_REGISTERED_PREFIX) || Date.now() >= deadline) { + throw error; + } + await new Promise((resolve) => setTimeout(resolve, 100)); } - wc.focus(); - - // Menu handlers read this when the app menu blurs the webContents (macOS/Windows). - refs.WebContentsManager.focusedWebContentsView = mmView.id; - - const mainWindow = refs.MainWindow.get() ?? BrowserWindow.getAllWindows().find((win) => { - return !win.isDestroyed() && win.webContents.getURL().includes('index'); - }); - mainWindow?.show(); - mainWindow?.focus(); - }, webContentsId); + } const mainWindow = findMainWindow(app); if (mainWindow) { diff --git a/e2e/helpers/userAttributes.ts b/e2e/helpers/userAttributes.ts index 8479194b6c1..143ac35ca5a 100644 --- a/e2e/helpers/userAttributes.ts +++ b/e2e/helpers/userAttributes.ts @@ -47,6 +47,26 @@ export type CustomProfileAttributeDef = { const CPA_FIELDS_PATH = '/api/v4/custom_profile_attributes/fields'; +/** + * Renderer-side JS expression resolving a custom attribute row from an + * element within it (a Save/Edit button, or the input itself). A single + * `closest('a, b, c')` call matches whichever ancestor is nearest in the + * DOM, not the most specific selector — a wrapper div around some field + * types (e.g. phone-number formatting) can sit closer than the actual row + * container, silently excluding it (and whatever the caller needed from it, + * like the Save button). Try selectors in specificity order instead so the + * intended row container is always found first. + */ +function customAttributeRowJs(elExpr: string): string { + return `( + ${elExpr}?.closest('.setting-list-item') || + ${elExpr}?.closest('.SettingsBlock') || + ${elExpr}?.closest('section') || + ${elExpr}?.closest('li') || + ${elExpr}?.closest('div') + )`; +} + export const TEST_PHONE = '555-123-4567'; export const TEST_UPDATED_PHONE = '555-987-6543'; export const TEST_URL = 'https://example.com'; @@ -260,7 +280,7 @@ export async function getCustomAttributeLabelsInSettings(win: ServerView): Promi labels.push(nameEl.textContent.trim()); continue; } - const row = button.closest('.setting-list-item, .SettingsBlock, section, li, div'); + const row = ${customAttributeRowJs('button')}; const rowText = (row?.textContent || '') .replace(/Edit.*$/s, '') .replace(/Click 'Edit' to add your custom attribute/gi, '') @@ -305,12 +325,7 @@ export async function editTextCustomAttribute( await win.runInRenderer(` const fieldId = ${JSON.stringify(fieldId)}; const input = document.querySelector('#customAttribute_' + fieldId); - const row = input?.closest('.setting-list-item') || - input?.closest('.SettingsBlock') || - input?.closest('section') || - input?.closest('li') || - input?.closest('div') || - document; + const row = ${customAttributeRowJs('input')} || document; const saveBtn = Array.from(row.querySelectorAll('button')) .find((button) => (button.textContent || '').trim() === 'Save'); if (!(saveBtn instanceof HTMLButtonElement)) { @@ -347,7 +362,7 @@ export async function getCustomAttributeInputValue(win: ServerView, fieldId: str if (!editBtn) { return input instanceof HTMLInputElement ? input.value : ''; } - const row = editBtn.closest('.setting-list-item, .SettingsBlock, section, li, div'); + const row = ${customAttributeRowJs('editBtn')}; if (!row) { return ''; } diff --git a/e2e/specs/calls/calls_functionality.test.ts b/e2e/specs/calls/calls_functionality.test.ts index 22d96095dbf..9d092fd9827 100644 --- a/e2e/specs/calls/calls_functionality.test.ts +++ b/e2e/specs/calls/calls_functionality.test.ts @@ -6,6 +6,7 @@ import type {ElectronApplication} from 'playwright'; import {test, expect} from '../../fixtures/index'; import {findCallsWidgetWindow, waitForCallsWidgetWindow} from '../../helpers/callsWidget'; +import {waitForMattermostShellReady} from '../../helpers/channelReadiness'; import {demoMattermostConfig} from '../../helpers/config'; import {CALLS_LEAVE_CALL} from '../../helpers/ipcChannels'; import {loginToMattermost} from '../../helpers/login'; @@ -66,6 +67,14 @@ test.describe('calls/calls_functionality', () => { serverWin = serverEntry!.win; await loginToMattermost(serverWin); + + // loginToMattermost only waits for the generic app shell (post + // textbox / channel header / search bar) — the sidebar's channel + // list can still be hydrating a beat later, so clicking + // #sidebarItem_town-square immediately here raced it intermittently + // ("Element not found for click"). Same wait media_preview.test.ts + // already uses before the identical click. + await waitForMattermostShellReady(serverWin, {channelItem: '#sidebarItem_town-square'}); await serverWin.click('#sidebarItem_town-square'); await serverWin.waitForSelector('#channelHeaderTitle', {timeout: 10_000}); }); diff --git a/e2e/specs/multi_window/multi_window.test.ts b/e2e/specs/multi_window/multi_window.test.ts index 4592aadcb2a..e33060e823a 100644 --- a/e2e/specs/multi_window/multi_window.test.ts +++ b/e2e/specs/multi_window/multi_window.test.ts @@ -469,11 +469,23 @@ test.describe('multi_window/multi_window', () => { const browserWindow = await electronApp.browserWindow(popoutWindow); const initialBounds = await browserWindow.evaluate((w) => (w as Electron.BrowserWindow).getBounds()); + // A flat +200 request assumed the display always has 200px of headroom + // past the window's current position. CI macOS runners use a small + // virtual display, so `initial + 200` routinely exceeds the work + // area — macOS then clamps the actual bounds at the screen edge, + // which the test misread as "resize failed". Clamp the request to + // what the display can actually satisfy (see the identical fix in + // popout_windows.test.ts MM-TXXXX_2). + const workArea = await browserWindow.evaluate((w) => { + const {screen} = require('electron') as typeof import('electron'); + return screen.getDisplayMatching((w as Electron.BrowserWindow).getBounds()).workArea; + }); + const margin = 20; const resizedBounds = { x: initialBounds.x, y: initialBounds.y, - width: initialBounds.width + 200, - height: initialBounds.height + 200, + width: Math.min(initialBounds.width + 200, (workArea.x + workArea.width) - initialBounds.x - margin), + height: Math.min(initialBounds.height + 200, (workArea.y + workArea.height) - initialBounds.y - margin), }; await browserWindow.evaluate((w, bounds) => { diff --git a/e2e/specs/permissions/permissions_ipc.test.ts b/e2e/specs/permissions/permissions_ipc.test.ts index fed5302883f..f73a3be4fd2 100644 --- a/e2e/specs/permissions/permissions_ipc.test.ts +++ b/e2e/specs/permissions/permissions_ipc.test.ts @@ -6,6 +6,14 @@ import {SHOW_SETTINGS_WINDOW} from '../../helpers/ipcChannels'; type ElectronApplication = Awaited>; +/** + * Callers must also depend on the `mainWindow` fixture before calling this, + * even though it isn't used directly here. `electronApp` alone launches the + * app without waiting for it to become ready (see fixtures/index.ts) — if + * MainWindow.get() isn't populated yet, handleShowSettingsModal() silently + * no-ops on a missing main window, and the waitForEvent('window') below + * times out 15s later waiting for a window that was never created. + */ async function openSettingsWindow(electronApp: ElectronApplication) { const existingWindow = electronApp.windows().find((window) => window.url().includes('settings')); if (existingWindow) { @@ -39,54 +47,69 @@ async function openSettingsWindow(electronApp: ElectronApplication) { } test.describe('permissions/ipc', () => { - test('MM-T6163 should return a valid media access status via GET_MEDIA_ACCESS_STATUS IPC', {tag: ['@P2', '@darwin', '@win32']}, async ({electronApp}) => { - const settingsWindow = await openSettingsWindow(electronApp); - - const status = await settingsWindow.evaluate( - () => (window as any).desktop.getMediaAccessStatus('microphone'), - ); - expect(['granted', 'denied', 'not-determined', 'restricted', 'unknown']).toContain(status); - }); - - test('MM-T6164 should open ms-settings:privacy-webcam for camera preferences (Windows only)', {tag: ['@P2', '@win32']}, async ({electronApp}) => { - const settingsWindow = await openSettingsWindow(electronApp); - - await electronApp.evaluate(({shell}) => { - (global as any).__testCapturedExternalURL = null; - shell.openExternal = (url: string) => { - (global as any).__testCapturedExternalURL = url; - return Promise.resolve(); - }; - }); - - await settingsWindow.evaluate( - () => (window as any).desktop.openWindowsCameraPreferences(), - ); - - const capturedURL = await electronApp.evaluate( - () => (global as any).__testCapturedExternalURL, - ); - expect(capturedURL).toBe('ms-settings:privacy-webcam'); - }); - - test('MM-T6165 should open ms-settings:privacy-microphone for microphone preferences (Windows only)', {tag: ['@P2', '@win32']}, async ({electronApp}) => { - const settingsWindow = await openSettingsWindow(electronApp); - - await electronApp.evaluate(({shell}) => { - (global as any).__testCapturedExternalURL = null; - shell.openExternal = (url: string) => { - (global as any).__testCapturedExternalURL = url; - return Promise.resolve(); - }; - }); - - await settingsWindow.evaluate( - () => (window as any).desktop.openWindowsMicrophonePreferences(), - ); - - const capturedURL = await electronApp.evaluate( - () => (global as any).__testCapturedExternalURL, - ); - expect(capturedURL).toBe('ms-settings:privacy-microphone'); - }); + test( + 'MM-T6163 should return a valid media access status via GET_MEDIA_ACCESS_STATUS IPC', + {tag: ['@P2', '@darwin', '@win32']}, + // eslint-disable-next-line @typescript-eslint/no-unused-vars -- see openSettingsWindow's doc comment + async ({electronApp, mainWindow: _mainWindow}) => { + const settingsWindow = await openSettingsWindow(electronApp); + + const status = await settingsWindow.evaluate( + () => (window as any).desktop.getMediaAccessStatus('microphone'), + ); + expect(['granted', 'denied', 'not-determined', 'restricted', 'unknown']).toContain(status); + }, + ); + + test( + 'MM-T6164 should open ms-settings:privacy-webcam for camera preferences (Windows only)', + {tag: ['@P2', '@win32']}, + // eslint-disable-next-line @typescript-eslint/no-unused-vars -- see openSettingsWindow's doc comment + async ({electronApp, mainWindow: _mainWindow}) => { + const settingsWindow = await openSettingsWindow(electronApp); + + await electronApp.evaluate(({shell}) => { + (global as any).__testCapturedExternalURL = null; + shell.openExternal = (url: string) => { + (global as any).__testCapturedExternalURL = url; + return Promise.resolve(); + }; + }); + + await settingsWindow.evaluate( + () => (window as any).desktop.openWindowsCameraPreferences(), + ); + + const capturedURL = await electronApp.evaluate( + () => (global as any).__testCapturedExternalURL, + ); + expect(capturedURL).toBe('ms-settings:privacy-webcam'); + }, + ); + + test( + 'MM-T6165 should open ms-settings:privacy-microphone for microphone preferences (Windows only)', + {tag: ['@P2', '@win32']}, + // eslint-disable-next-line @typescript-eslint/no-unused-vars -- see openSettingsWindow's doc comment + async ({electronApp, mainWindow: _mainWindow}) => { + const settingsWindow = await openSettingsWindow(electronApp); + + await electronApp.evaluate(({shell}) => { + (global as any).__testCapturedExternalURL = null; + shell.openExternal = (url: string) => { + (global as any).__testCapturedExternalURL = url; + return Promise.resolve(); + }; + }); + + await settingsWindow.evaluate( + () => (window as any).desktop.openWindowsMicrophonePreferences(), + ); + + const capturedURL = await electronApp.evaluate( + () => (global as any).__testCapturedExternalURL, + ); + expect(capturedURL).toBe('ms-settings:privacy-microphone'); + }, + ); }); diff --git a/e2e/specs/server_management/popout_windows.test.ts b/e2e/specs/server_management/popout_windows.test.ts index b9f3dfbbcae..1bda07d47db 100644 --- a/e2e/specs/server_management/popout_windows.test.ts +++ b/e2e/specs/server_management/popout_windows.test.ts @@ -74,11 +74,24 @@ test.describe('server_management/popout_windows', () => { const browserWindow = await electronApp.browserWindow(popoutWindow); const initialBounds = await browserWindow.evaluate((w) => (w as Electron.BrowserWindow).getBounds()); + // A flat +200 request assumed the display always has 200px of + // headroom past the window's current position. CI macOS runners + // use a small virtual display, so `initial + 200` routinely + // exceeds the work area — macOS then clamps the actual bounds at + // the screen edge, which the test misread as "resize failed" by + // as much as 456px. Clamp the request to what the display can + // actually satisfy so the assertion checks the resize itself, + // not whether the display happened to have enough room. + const workArea = await browserWindow.evaluate((w) => { + const {screen} = require('electron') as typeof import('electron'); + return screen.getDisplayMatching((w as Electron.BrowserWindow).getBounds()).workArea; + }); + const margin = 20; const newBounds = { x: initialBounds.x, y: initialBounds.y, - width: initialBounds.width + 200, - height: initialBounds.height + 200, + width: Math.min(initialBounds.width + 200, (workArea.x + workArea.width) - initialBounds.x - margin), + height: Math.min(initialBounds.height + 200, (workArea.y + workArea.height) - initialBounds.y - margin), }; await browserWindow.evaluate((w, bounds) => { diff --git a/e2e/utils/github-actions.js b/e2e/utils/github-actions.js index 936c208a8e3..987a2f6ca9b 100644 --- a/e2e/utils/github-actions.js +++ b/e2e/utils/github-actions.js @@ -2,10 +2,6 @@ // See LICENSE.txt for license information. /* eslint-disable no-console -- Logging is intentional in CI utility scripts */ -// TSIO (tsio-summary job) is now the single source of truth for PR/master E2E -// pass/fail — this constant is only used to mark that one context as -// cancelled/skipped, not to post pending/final results (see -// e2e/utils/tsio-report-status.js for the real status logic). const E2E_STATUS_CONTEXT = 'e2e-test/desktop-playwright'; const E2E_WORKFLOW_NAME = 'Electron Playwright Tests'; diff --git a/e2e/utils/tsio-report-status.js b/e2e/utils/tsio-report-status.js index ad22729fca9..97673957561 100644 --- a/e2e/utils/tsio-report-status.js +++ b/e2e/utils/tsio-report-status.js @@ -99,12 +99,6 @@ async function reportTsioStatus({ } } } catch (error) { - // Prefer the report link if begin already succeeded (the poll loop - // itself failed) — only fall back to the workflow run when no - // report exists at all. Swallow a secondary status-API failure so - // it doesn't mask the original error. The description is public - // (visible on the PR/commit), so keep the raw error out of it — - // log it instead and point readers at the workflow run for detail. core.error(`TSIO reporting error: ${error.message}`); try { await github.rest.repos.createCommitStatus({ @@ -126,6 +120,7 @@ async function reportTsioStatus({ const isComplete = detail.status === 'completed'; const hasFailures = (stats.failed || 0) > 0; const overallState = isComplete && !hasFailures && upstreamJobsSucceeded ? 'success' : 'failure'; + const targetUrl = isComplete ? reportUrl : runUrl; const summaryLines = [ `### Test System IO — ${compositeIdentity.name}`, @@ -136,6 +131,9 @@ async function reportTsioStatus({ ...(upstreamJobsSucceeded ? [] : ['', ':warning: One or more CI jobs failed outside of any tracked test (e.g. a hung worker, a crashed runner) — forcing this status to failure even though the test stats above may show no failures.']), + ...(isComplete ? + [] : + ['', `:warning: Report never reached \`completed\` (stuck at \`${detail.status}\`) — see the [workflow run](${runUrl}) for the shard that didn't finish uploading.`]), '', ]; await core.summary.addRaw(summaryLines.join('\n')).write(); @@ -149,7 +147,7 @@ async function reportTsioStatus({ state: overallState, context: commitStatusContext, description, - target_url: reportUrl, + target_url: targetUrl, }); if (failOnTestFailures && overallState === 'failure') { From 29b47e7dcda38b98726f2abeafc4682bf945f440 Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Wed, 8 Jul 2026 17:57:27 +0530 Subject: [PATCH 14/37] test fixes --- e2e/helpers/ipcChannels.ts | 1 + e2e/helpers/userAttributes.ts | 36 ++++++++++++------- e2e/specs/menu_bar/help_menu.test.ts | 10 ++++-- e2e/specs/multi_window/multi_window.test.ts | 19 ++++------ .../server_management/popout_windows.test.ts | 18 ++++------ 5 files changed, 45 insertions(+), 39 deletions(-) diff --git a/e2e/helpers/ipcChannels.ts b/e2e/helpers/ipcChannels.ts index bab9c9199e4..5d299ec68f4 100644 --- a/e2e/helpers/ipcChannels.ts +++ b/e2e/helpers/ipcChannels.ts @@ -8,3 +8,4 @@ export const CLOSE_DOWNLOADS_DROPDOWN_MENU = 'close-downloads-dropdown-menu'; export const CALLS_LEAVE_CALL = 'calls-leave-call'; export const EMIT_CONFIGURATION = 'emit-configuration'; export const SHOW_NEW_SERVER_MODAL = 'show_new_server_modal'; +export const NOTIFICATION_CLICKED = 'notification-clicked'; diff --git a/e2e/helpers/userAttributes.ts b/e2e/helpers/userAttributes.ts index 143ac35ca5a..1c0f70a4b4c 100644 --- a/e2e/helpers/userAttributes.ts +++ b/e2e/helpers/userAttributes.ts @@ -47,19 +47,27 @@ export type CustomProfileAttributeDef = { const CPA_FIELDS_PATH = '/api/v4/custom_profile_attributes/fields'; +/** + * Renderer-side JS expression resolving the active custom-attribute edit + * section. Inputs and the Save/Cancel row are sibling `.setting-list-item` + * nodes under `.setting-list`, so scoping to the input's list-item misses + * `#saveSetting`. + */ +function customAttributeEditScopeJs(elExpr: string): string { + return `( + ${elExpr}?.closest('.setting-list') || + ${elExpr}?.closest('section.section-max') || + ${elExpr}?.closest('section') + )`; +} + /** * Renderer-side JS expression resolving a custom attribute row from an - * element within it (a Save/Edit button, or the input itself). A single - * `closest('a, b, c')` call matches whichever ancestor is nearest in the - * DOM, not the most specific selector — a wrapper div around some field - * types (e.g. phone-number formatting) can sit closer than the actual row - * container, silently excluding it (and whatever the caller needed from it, - * like the Save button). Try selectors in specificity order instead so the - * intended row container is always found first. + * element within it (e.g. the Edit button in display mode). */ function customAttributeRowJs(elExpr: string): string { return `( - ${elExpr}?.closest('.setting-list-item') || + ${elExpr}?.closest('.setting-list') || ${elExpr}?.closest('.SettingsBlock') || ${elExpr}?.closest('section') || ${elExpr}?.closest('li') || @@ -322,14 +330,18 @@ export async function editTextCustomAttribute( await win.fill(`#customAttribute_${fieldId}`, newValue); } if (save) { + await win.waitForSelector('#saveSetting', {timeout: 10_000}); + await expect.poll(async () => win.runInRenderer(` + const saveBtn = document.querySelector('#saveSetting'); + return saveBtn instanceof HTMLButtonElement && !saveBtn.disabled; + `), {timeout: 10_000, message: 'Save button must be enabled before saving custom attribute'}).toBe(true); await win.runInRenderer(` const fieldId = ${JSON.stringify(fieldId)}; const input = document.querySelector('#customAttribute_' + fieldId); - const row = ${customAttributeRowJs('input')} || document; - const saveBtn = Array.from(row.querySelectorAll('button')) - .find((button) => (button.textContent || '').trim() === 'Save'); + const scope = ${customAttributeEditScopeJs('input')} || document; + const saveBtn = scope.querySelector('#saveSetting') || document.querySelector('#saveSetting'); if (!(saveBtn instanceof HTMLButtonElement)) { - throw new Error('Save button not found for custom attribute row'); + throw new Error('Save button not found for custom attribute edit section'); } saveBtn.click(); `); diff --git a/e2e/specs/menu_bar/help_menu.test.ts b/e2e/specs/menu_bar/help_menu.test.ts index 31202263e6c..cd150fd4e5b 100644 --- a/e2e/specs/menu_bar/help_menu.test.ts +++ b/e2e/specs/menu_bar/help_menu.test.ts @@ -87,7 +87,9 @@ test.describe('menu_bar/help_menu', () => { 'MM-T6151 Show logs menu item opens the log file location', {tag: ['@P1', '@all']}, async ({electronApp}) => { - await electronApp.evaluate(({shell}) => { + await waitForAppReady(electronApp); + + await evaluateInMainProcess(electronApp, ({shell}) => { (global as any).__e2eShownInFolder = [] as string[]; (global as any).__e2eOriginalShowItemInFolder = shell.showItemInFolder.bind(shell); shell.showItemInFolder = (fullPath: string) => { @@ -100,10 +102,12 @@ test.describe('menu_bar/help_menu', () => { await clickApplicationMenuItem(electronApp, 'help', {id: 'Show logs'}); await expect.poll(async () => { - return electronApp.evaluate(() => ((global as any).__e2eShownInFolder as string[] | undefined)?.length ?? 0); + return evaluateInMainProcess(electronApp, () => { + return ((global as any).__e2eShownInFolder as string[] | undefined)?.length ?? 0; + }); }, {timeout: 10_000}).toBeGreaterThan(0); } finally { - await electronApp.evaluate(({shell}) => { + await evaluateInMainProcess(electronApp, ({shell}) => { const original = (global as any).__e2eOriginalShowItemInFolder; if (original) { shell.showItemInFolder = original; diff --git a/e2e/specs/multi_window/multi_window.test.ts b/e2e/specs/multi_window/multi_window.test.ts index e33060e823a..ae3a8cbf41d 100644 --- a/e2e/specs/multi_window/multi_window.test.ts +++ b/e2e/specs/multi_window/multi_window.test.ts @@ -12,6 +12,7 @@ import {demoMattermostConfig} from '../../helpers/config'; import {closeDownloadsDropdownIfOpen} from '../../helpers/downloadsDropdown'; import {launchDirectTestApp} from '../../helpers/directLaunch'; import {closeElectronAppFast, waitForWindow} from '../../helpers/electronApp'; +import {NOTIFICATION_CLICKED} from '../../helpers/ipcChannels'; import {loginToMattermost} from '../../helpers/login'; import {waitForMainWindowFocused} from '../../helpers/mainWindowFocus'; import {POST_TEXTBOX_SELECTOR, waitForChannelPostListLoaded, waitForMattermostShellReady} from '../../helpers/mattermostShell'; @@ -32,12 +33,12 @@ import {prepareMattermostServerView} from '../../helpers/prepareServerView'; import {resolvedChannelPath, resolveChannelByName} from '../../helpers/server_api/channel'; import {seedThreadInChannel} from '../../helpers/server_api/post'; import {buildServerMap} from '../../helpers/serverMap'; +import {evaluateInMainProcessWithArg} from '../../helpers/testRefs'; import { clickOpenInNewWindowFromRhsThreadMenu, clickOpenInNewWindowFromThreadsListMenu, clickOpenInNewWindowMenuItem, } from '../../helpers/webappMenu'; -import {NOTIFICATION_CLICKED} from '../../../src/common/communication'; const config = { ...demoMattermostConfig, @@ -469,17 +470,11 @@ test.describe('multi_window/multi_window', () => { const browserWindow = await electronApp.browserWindow(popoutWindow); const initialBounds = await browserWindow.evaluate((w) => (w as Electron.BrowserWindow).getBounds()); - // A flat +200 request assumed the display always has 200px of headroom - // past the window's current position. CI macOS runners use a small - // virtual display, so `initial + 200` routinely exceeds the work - // area — macOS then clamps the actual bounds at the screen edge, - // which the test misread as "resize failed". Clamp the request to - // what the display can actually satisfy (see the identical fix in - // popout_windows.test.ts MM-TXXXX_2). - const workArea = await browserWindow.evaluate((w) => { - const {screen} = require('electron') as typeof import('electron'); - return screen.getDisplayMatching((w as Electron.BrowserWindow).getBounds()).workArea; - }); + const workArea = await evaluateInMainProcessWithArg( + electronApp, + (electron, bounds) => electron.screen.getDisplayMatching(bounds).workArea, + initialBounds, + ); const margin = 20; const resizedBounds = { x: initialBounds.x, diff --git a/e2e/specs/server_management/popout_windows.test.ts b/e2e/specs/server_management/popout_windows.test.ts index 1bda07d47db..a8eb38f9888 100644 --- a/e2e/specs/server_management/popout_windows.test.ts +++ b/e2e/specs/server_management/popout_windows.test.ts @@ -16,6 +16,7 @@ import { openPopoutWindow, } from '../../helpers/popoutWindow'; import {buildServerMap} from '../../helpers/serverMap'; +import {evaluateInMainProcessWithArg} from '../../helpers/testRefs'; const config = { ...demoMattermostConfig, @@ -74,18 +75,11 @@ test.describe('server_management/popout_windows', () => { const browserWindow = await electronApp.browserWindow(popoutWindow); const initialBounds = await browserWindow.evaluate((w) => (w as Electron.BrowserWindow).getBounds()); - // A flat +200 request assumed the display always has 200px of - // headroom past the window's current position. CI macOS runners - // use a small virtual display, so `initial + 200` routinely - // exceeds the work area — macOS then clamps the actual bounds at - // the screen edge, which the test misread as "resize failed" by - // as much as 456px. Clamp the request to what the display can - // actually satisfy so the assertion checks the resize itself, - // not whether the display happened to have enough room. - const workArea = await browserWindow.evaluate((w) => { - const {screen} = require('electron') as typeof import('electron'); - return screen.getDisplayMatching((w as Electron.BrowserWindow).getBounds()).workArea; - }); + const workArea = await evaluateInMainProcessWithArg( + electronApp, + (electron, bounds) => electron.screen.getDisplayMatching(bounds).workArea, + initialBounds, + ); const margin = 20; const newBounds = { x: initialBounds.x, From 02669f92bd7f8ef7e9f0db7dabc094455f43ab02 Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Thu, 9 Jul 2026 22:06:52 +0530 Subject: [PATCH 15/37] coderabbit review --- .../compatibility-matrix-testing.yml | 1 - .github/workflows/e2e-functional-template.yml | 52 ++++++-- .github/workflows/e2e-functional.yml | 125 +++++++++++++++--- e2e/utils/github-actions.js | 1 + e2e/utils/tsio-report-status.js | 21 ++- 5 files changed, 162 insertions(+), 38 deletions(-) diff --git a/.github/workflows/compatibility-matrix-testing.yml b/.github/workflows/compatibility-matrix-testing.yml index 83efe322175..f1536433876 100644 --- a/.github/workflows/compatibility-matrix-testing.yml +++ b/.github/workflows/compatibility-matrix-testing.yml @@ -133,7 +133,6 @@ jobs: permissions: contents: read actions: read - id-token: write secrets: inherit with: runs-on: ${{ matrix.environment.runner }} diff --git a/.github/workflows/e2e-functional-template.yml b/.github/workflows/e2e-functional-template.yml index 926c53ba8f1..f47204f03ae 100644 --- a/.github/workflows/e2e-functional-template.yml +++ b/.github/workflows/e2e-functional-template.yml @@ -107,10 +107,9 @@ jobs: # 9.5.2) name: e2e-on-${{ inputs.runs-on }}-${{ inputs.MM_SERVER_VERSION }} runs-on: ${{ inputs.runs-on }} + # Runs untrusted PR code (npm ci / Playwright) with a read-only token. permissions: contents: read - id-token: write - actions: read defaults: run: shell: bash @@ -278,18 +277,15 @@ jobs: DESKTOP_VERSION: ${{ inputs.DESKTOP_VERSION }} CI_ENVIRONMENT_NAME: ${{ env.CI_ENVIRONMENT_NAME }} - - name: e2e/upload-report-to-tsio - if: ${{ always() && inputs.tsio-config != '' && hashFiles('e2e/test-results/results.json') != '' }} - uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-report-upload@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 0.10.0 / 2026-05-16 + - name: e2e/upload-tsio-test-results + if: always() && inputs.tsio-config != '' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - composite-identity: ${{ toJSON(fromJSON(inputs.tsio-config).composite_identity) }} - total-reports-expected: ${{ fromJSON(inputs.tsio-config).total_reports_expected }} - framework: playwright - github-token: ${{ secrets.GITHUB_TOKEN }} - # MUST match this job's `name:` field above. - gh-job-name: e2e-on-${{ inputs.runs-on }}-${{ inputs.MM_SERVER_VERSION }} - json-path: e2e/test-results/results.json - screenshots-dir: e2e/test-results + name: tsio-${{ inputs.runs-on }}-${{ inputs.MM_SERVER_VERSION }} + path: | + e2e/test-results/results.json + e2e/test-results + if-no-files-found: ignore - name: e2e/handle-nonzero-playwright-exit if: always() @@ -306,3 +302,33 @@ jobs: env: TSIO_CONFIG: ${{ inputs.tsio-config }} + tsio-upload: + name: tsio-upload-${{ inputs.runs-on }}-${{ inputs.MM_SERVER_VERSION }} + needs: e2e + if: always() && inputs.tsio-config != '' + runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write + actions: read + steps: + - name: e2e/download-tsio-test-results + id: download-tsio-test-results + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + continue-on-error: true + with: + name: tsio-${{ inputs.runs-on }}-${{ inputs.MM_SERVER_VERSION }} + + - name: e2e/upload-report-to-tsio + if: steps.download-tsio-test-results.outcome == 'success' && hashFiles('e2e/test-results/results.json') != '' + uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-report-upload@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 0.10.0 / 2026-05-16 + with: + composite-identity: ${{ toJSON(fromJSON(inputs.tsio-config).composite_identity) }} + total-reports-expected: ${{ fromJSON(inputs.tsio-config).total_reports_expected }} + framework: playwright + github-token: ${{ secrets.GITHUB_TOKEN }} + # MUST match the e2e job's `name:` field above. + gh-job-name: e2e-on-${{ inputs.runs-on }}-${{ inputs.MM_SERVER_VERSION }} + json-path: e2e/test-results/results.json + screenshots-dir: e2e/test-results + diff --git a/.github/workflows/e2e-functional.yml b/.github/workflows/e2e-functional.yml index 1090b547e1e..1ad612c8dff 100644 --- a/.github/workflows/e2e-functional.yml +++ b/.github/workflows/e2e-functional.yml @@ -115,7 +115,6 @@ jobs: permissions: contents: read actions: read - id-token: write uses: ./.github/workflows/e2e-functional-template.yml with: runs-on: ${{ matrix.runner }} @@ -148,10 +147,11 @@ jobs: id-token: write statuses: write steps: - # tsio-report-status.js is repo code, not fetched by github-script itself. + # Always master — this job holds id-token/statuses write and must not + # execute repo code from the untrusted PR head (inputs.version_name). - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: - ref: ${{ inputs.version_name }} + ref: master persist-credentials: false sparse-checkout: | e2e/utils/tsio-report-status.js @@ -167,12 +167,60 @@ jobs: UPSTREAM_JOBS_SUCCEEDED: ${{ needs.e2e-tests.result == 'success' && needs.e2e-policy-tests.result == 'success' }} with: script: | + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const commitStatusContext = process.env.COMMIT_STATUS_CONTEXT; + const identityRaw = process.env.TSIO_COMPOSITE_IDENTITY; + let compositeIdentity; + let totalReportsExpected; + + if (!identityRaw) { + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: context.sha, + state: 'failure', + context: commitStatusContext, + description: 'TSIO summary failed — missing composite identity', + target_url: runUrl, + }); + throw new Error('TSIO_COMPOSITE_IDENTITY is missing (prepare-matrix failed or was skipped)'); + } + + try { + compositeIdentity = JSON.parse(identityRaw); + totalReportsExpected = parseInt(process.env.TSIO_TOTAL_REPORTS_EXPECTED, 10); + } catch (parseError) { + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: context.sha, + state: 'failure', + context: commitStatusContext, + description: 'TSIO summary failed — invalid composite identity', + target_url: runUrl, + }); + throw parseError; + } + + if (!compositeIdentity?.commit_sha || !Number.isFinite(totalReportsExpected)) { + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: compositeIdentity?.commit_sha || context.sha, + state: 'failure', + context: commitStatusContext, + description: 'TSIO summary failed — incomplete composite identity', + target_url: runUrl, + }); + throw new Error('TSIO composite identity or totalReportsExpected is incomplete'); + } + const {reportUrl, status, stats} = await require('./e2e/utils/tsio-report-status.js')({ core, context, github, - compositeIdentity: JSON.parse(process.env.TSIO_COMPOSITE_IDENTITY), - totalReportsExpected: parseInt(process.env.TSIO_TOTAL_REPORTS_EXPECTED, 10), + compositeIdentity, + totalReportsExpected, upstreamJobsSucceeded: process.env.UPSTREAM_JOBS_SUCCEEDED === 'true', - commitStatusContext: process.env.COMMIT_STATUS_CONTEXT, + commitStatusContext, failOnTestFailures: true, }); core.info(`TSIO report ${reportUrl}: ${status} (${JSON.stringify(stats)})`); @@ -246,12 +294,10 @@ jobs: needs: - prepare-matrix - update-initial-status - # Runs untrusted PR code (npm ci / Playwright); grant only what the TSIO - # upload needs, never pull-requests: write. + # Runs untrusted PR code (npm ci / Playwright); no OIDC here — TSIO upload + # is handled by e2e-policy-tsio-upload after artifacts are saved. permissions: contents: read - id-token: write - actions: read strategy: matrix: include: @@ -389,19 +435,17 @@ jobs: SERVER_VERSION: ${{ inputs.MM_SERVER_VERSION }} DESKTOP_VERSION: ${{ inputs.version_name }} - # Joins the same TSIO report group as e2e-tests — gh-job-name must match - # this job's rendered name (policy-tests-) above. - - name: e2e/upload-report-to-tsio - if: ${{ always() && needs.prepare-matrix.outputs.tsio-composite-identity != '' && hashFiles('e2e/test-results/results.json') != '' }} - uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-report-upload@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 0.10.0 / 2026-05-16 + # Joins the same TSIO report group as e2e-tests — gh-job-name in the + # upload job must match this job's rendered name (policy-tests-). + - name: e2e/upload-tsio-test-results + if: always() && needs.prepare-matrix.outputs.tsio-composite-identity != '' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - composite-identity: ${{ needs.prepare-matrix.outputs.tsio-composite-identity }} - total-reports-expected: ${{ needs.prepare-matrix.outputs.tsio-total-reports-expected }} - framework: playwright - github-token: ${{ secrets.GITHUB_TOKEN }} - gh-job-name: policy-tests-${{ matrix.platform }} - json-path: e2e/test-results/results.json - screenshots-dir: e2e/test-results + name: policy-tsio-${{ matrix.platform }} + path: | + e2e/test-results/results.json + e2e/test-results + if-no-files-found: ignore - name: e2e/handle-nonzero-playwright-exit if: always() @@ -426,3 +470,40 @@ jobs: path: e2e/playwright-report if-no-files-found: ignore retention-days: 7 + + e2e-policy-tsio-upload: + name: policy-tsio-upload-${{ matrix.platform }} + needs: + - prepare-matrix + - e2e-policy-tests + if: always() && needs.prepare-matrix.outputs.tsio-composite-identity != '' + strategy: + matrix: + include: + - platform: macos + - platform: windows + fail-fast: false + runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write + actions: read + steps: + - name: e2e/download-tsio-test-results + id: download-tsio-test-results + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + continue-on-error: true + with: + name: policy-tsio-${{ matrix.platform }} + + - name: e2e/upload-report-to-tsio + if: steps.download-tsio-test-results.outcome == 'success' && hashFiles('e2e/test-results/results.json') != '' + uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-report-upload@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 0.10.0 / 2026-05-16 + with: + composite-identity: ${{ needs.prepare-matrix.outputs.tsio-composite-identity }} + total-reports-expected: ${{ needs.prepare-matrix.outputs.tsio-total-reports-expected }} + framework: playwright + github-token: ${{ secrets.GITHUB_TOKEN }} + gh-job-name: policy-tests-${{ matrix.platform }} + json-path: e2e/test-results/results.json + screenshots-dir: e2e/test-results diff --git a/e2e/utils/github-actions.js b/e2e/utils/github-actions.js index 987a2f6ca9b..94ba8386e80 100644 --- a/e2e/utils/github-actions.js +++ b/e2e/utils/github-actions.js @@ -28,6 +28,7 @@ async function markE2EStatusesCancelled({github, context, sha, reason = CANCELLE }); } catch (error) { console.log(`Could not update ${E2E_STATUS_CONTEXT} on ${sha}: ${error.message}`); + throw error; } } diff --git a/e2e/utils/tsio-report-status.js b/e2e/utils/tsio-report-status.js index 97673957561..1a4c342fc7e 100644 --- a/e2e/utils/tsio-report-status.js +++ b/e2e/utils/tsio-report-status.js @@ -8,9 +8,26 @@ const STAGING_URL = 'https://staging-test-io.test.mattermost.com'; const TERMINAL_STATUSES = ['completed', 'incomplete']; const POLL_ATTEMPTS = 6; const POLL_DELAY_MS = 5000; +const FETCH_TIMEOUT_MS = 30_000; const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +async function fetchWithTimeout(url, init = {}, timeoutMs = FETCH_TIMEOUT_MS) { + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), timeoutMs); + + try { + return await fetch(url, {...init, signal: controller.signal}); + } catch (error) { + if (controller.signal.aborted) { + throw new Error(`Request timed out after ${timeoutMs}ms: ${url}`); + } + throw error; + } finally { + clearTimeout(timeoutId); + } +} + /** * Recover a report group's id via the idempotent begin endpoint, poll the * public status endpoint until the group leaves in_progress, render a step @@ -58,7 +75,7 @@ async function reportTsioStatus({ const idToken = await core.getIDToken(oidcAudience); core.setSecret(idToken); - const beginRes = await fetch(`${baseUrl}/api/v1/reports/begin`, { + const beginRes = await fetchWithTimeout(`${baseUrl}/api/v1/reports/begin`, { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -86,7 +103,7 @@ async function reportTsioStatus({ reportUrl = `${baseUrl}/reports/g/${reportId}`; for (let attempt = 0; attempt < POLL_ATTEMPTS; attempt++) { - const statusRes = await fetch(`${baseUrl}/api/v1/reports/${reportId}`); + const statusRes = await fetchWithTimeout(`${baseUrl}/api/v1/reports/${reportId}`); if (!statusRes.ok) { throw new Error(`reports/${reportId} failed: ${statusRes.status} ${await statusRes.text()}`); } From 63bba56fe445fe5c2aa4d6c1694965b38439e0d3 Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Fri, 10 Jul 2026 00:05:28 +0530 Subject: [PATCH 16/37] fix label context --- .github/workflows/e2e-functional-template.yml | 29 ---------- .github/workflows/e2e-functional.yml | 39 +++++++++++++ e2e/utils/tsio-report-status.js | 57 ++++++++++--------- 3 files changed, 69 insertions(+), 56 deletions(-) diff --git a/.github/workflows/e2e-functional-template.yml b/.github/workflows/e2e-functional-template.yml index f47204f03ae..f13061c0eef 100644 --- a/.github/workflows/e2e-functional-template.yml +++ b/.github/workflows/e2e-functional-template.yml @@ -302,33 +302,4 @@ jobs: env: TSIO_CONFIG: ${{ inputs.tsio-config }} - tsio-upload: - name: tsio-upload-${{ inputs.runs-on }}-${{ inputs.MM_SERVER_VERSION }} - needs: e2e - if: always() && inputs.tsio-config != '' - runs-on: ubuntu-24.04 - permissions: - contents: read - id-token: write - actions: read - steps: - - name: e2e/download-tsio-test-results - id: download-tsio-test-results - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - continue-on-error: true - with: - name: tsio-${{ inputs.runs-on }}-${{ inputs.MM_SERVER_VERSION }} - - - name: e2e/upload-report-to-tsio - if: steps.download-tsio-test-results.outcome == 'success' && hashFiles('e2e/test-results/results.json') != '' - uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-report-upload@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 0.10.0 / 2026-05-16 - with: - composite-identity: ${{ toJSON(fromJSON(inputs.tsio-config).composite_identity) }} - total-reports-expected: ${{ fromJSON(inputs.tsio-config).total_reports_expected }} - framework: playwright - github-token: ${{ secrets.GITHUB_TOKEN }} - # MUST match the e2e job's `name:` field above. - gh-job-name: e2e-on-${{ inputs.runs-on }}-${{ inputs.MM_SERVER_VERSION }} - json-path: e2e/test-results/results.json - screenshots-dir: e2e/test-results diff --git a/.github/workflows/e2e-functional.yml b/.github/workflows/e2e-functional.yml index 1ad612c8dff..1c3b5e8c041 100644 --- a/.github/workflows/e2e-functional.yml +++ b/.github/workflows/e2e-functional.yml @@ -141,6 +141,8 @@ jobs: - prepare-matrix - e2e-tests - e2e-policy-tests + - e2e-tsio-upload + - e2e-policy-tsio-upload if: always() permissions: contents: read @@ -471,6 +473,43 @@ jobs: if-no-files-found: ignore retention-days: 7 + e2e-tsio-upload: + name: tsio-upload-${{ matrix.platform }} + needs: + - prepare-matrix + - e2e-tests + if: always() && needs.prepare-matrix.outputs.tsio-composite-identity != '' + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.prepare-matrix.outputs.platforms) }} + runs-on: ubuntu-24.04 + permissions: + contents: read + id-token: write + actions: read + steps: + - name: e2e/download-tsio-test-results + id: download-tsio-test-results + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + continue-on-error: true + with: + # Matches the artifact uploaded by e2e-functional-template.yml's e2e job. + name: tsio-${{ matrix.runner }}-${{ inputs.MM_SERVER_VERSION }} + + - name: e2e/upload-report-to-tsio + if: steps.download-tsio-test-results.outcome == 'success' && hashFiles('e2e/test-results/results.json') != '' + uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-report-upload@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 0.10.0 / 2026-05-16 + with: + composite-identity: ${{ needs.prepare-matrix.outputs.tsio-composite-identity }} + total-reports-expected: ${{ needs.prepare-matrix.outputs.tsio-total-reports-expected }} + framework: playwright + github-token: ${{ secrets.GITHUB_TOKEN }} + # Matches the e2e-tests job's rendered `name: ${{ matrix.platform }}`. + gh-job-name: ${{ matrix.platform }} + json-path: e2e/test-results/results.json + screenshots-dir: e2e/test-results + e2e-policy-tsio-upload: name: policy-tsio-upload-${{ matrix.platform }} needs: diff --git a/e2e/utils/tsio-report-status.js b/e2e/utils/tsio-report-status.js index 1a4c342fc7e..c8a94c17f14 100644 --- a/e2e/utils/tsio-report-status.js +++ b/e2e/utils/tsio-report-status.js @@ -12,12 +12,17 @@ const FETCH_TIMEOUT_MS = 30_000; const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); -async function fetchWithTimeout(url, init = {}, timeoutMs = FETCH_TIMEOUT_MS) { +async function fetchJsonWithTimeout(url, {init = {}, label, timeoutMs = FETCH_TIMEOUT_MS} = {}) { const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), timeoutMs); try { - return await fetch(url, {...init, signal: controller.signal}); + const res = await fetch(url, {...init, signal: controller.signal}); + if (!res.ok) { + const text = await res.text(); + throw new Error(`${label} failed: ${res.status} ${text}`); + } + return await res.json(); } catch (error) { if (controller.signal.aborted) { throw new Error(`Request timed out after ${timeoutMs}ms: ${url}`); @@ -75,39 +80,37 @@ async function reportTsioStatus({ const idToken = await core.getIDToken(oidcAudience); core.setSecret(idToken); - const beginRes = await fetchWithTimeout(`${baseUrl}/api/v1/reports/begin`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${idToken}`, + const beginJson = await fetchJsonWithTimeout(`${baseUrl}/api/v1/reports/begin`, { + init: { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${idToken}`, + }, + body: JSON.stringify({ + repository: compositeIdentity.repository, + commit: compositeIdentity.commit_sha, + gh_run_id: compositeIdentity.gh_run_id, + gh_run_attempt: compositeIdentity.gh_run_attempt, + framework: 'playwright', + name: compositeIdentity.name, + branch: compositeIdentity.branch, + total_reports_expected: totalReportsExpected, + ...(compositeIdentity.gh_pr_number ? {gh_pr_number: parseInt(compositeIdentity.gh_pr_number, 10)} : {}), + }), }, - body: JSON.stringify({ - repository: compositeIdentity.repository, - commit: compositeIdentity.commit_sha, - gh_run_id: compositeIdentity.gh_run_id, - gh_run_attempt: compositeIdentity.gh_run_attempt, - framework: 'playwright', - name: compositeIdentity.name, - branch: compositeIdentity.branch, - total_reports_expected: totalReportsExpected, - ...(compositeIdentity.gh_pr_number ? {gh_pr_number: parseInt(compositeIdentity.gh_pr_number, 10)} : {}), - }), + label: 'reports/begin', }); - if (!beginRes.ok) { - throw new Error(`reports/begin failed: ${beginRes.status} ${await beginRes.text()}`); - } - ({report_id: reportId} = await beginRes.json()); + ({report_id: reportId} = beginJson); // /reports/{id} (no prefix) hits the frontend's repo-or-sha catch-all route, // not the report detail page — it needs the g/ (group) prefix. reportUrl = `${baseUrl}/reports/g/${reportId}`; for (let attempt = 0; attempt < POLL_ATTEMPTS; attempt++) { - const statusRes = await fetchWithTimeout(`${baseUrl}/api/v1/reports/${reportId}`); - if (!statusRes.ok) { - throw new Error(`reports/${reportId} failed: ${statusRes.status} ${await statusRes.text()}`); - } - detail = await statusRes.json(); + detail = await fetchJsonWithTimeout(`${baseUrl}/api/v1/reports/${reportId}`, { + label: `reports/${reportId}`, + }); if (TERMINAL_STATUSES.includes(detail.status)) { break; } From 0ed634018d7e7dec9bed715841f8eddc4df6e6ff Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Fri, 10 Jul 2026 07:03:19 +0530 Subject: [PATCH 17/37] fix label context --- .github/workflows/e2e-functional.yml | 135 +++++++++++++++++++++++++-- 1 file changed, 125 insertions(+), 10 deletions(-) diff --git a/.github/workflows/e2e-functional.yml b/.github/workflows/e2e-functional.yml index 1c3b5e8c041..921fa3652e7 100644 --- a/.github/workflows/e2e-functional.yml +++ b/.github/workflows/e2e-functional.yml @@ -149,15 +149,13 @@ jobs: id-token: write statuses: write steps: - # Always master — this job holds id-token/statuses write and must not - # execute repo code from the untrusted PR head (inputs.version_name). - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - ref: master - persist-credentials: false - sparse-checkout: | - e2e/utils/tsio-report-status.js - sparse-checkout-cone-mode: false + # No checkout / no require(): this job holds id-token + statuses: write, + # so it must never load a file from the untrusted PR head. The helper can't + # be loaded from `ref: master` either — e2e/utils/tsio-report-status.js is + # new in this PR and isn't on master until this merges. So the logic is + # INLINED below (the "inlining the logic instead" alternative CodeRabbit + # suggested for the trusted-source review comment). The file copy is kept + # for compatibility-matrix-testing.yml, which runs on trusted release refs. - name: Render TSIO summary + flip commit status id: summary @@ -217,7 +215,124 @@ jobs: throw new Error('TSIO composite identity or totalReportsExpected is incomplete'); } - const {reportUrl, status, stats} = await require('./e2e/utils/tsio-report-status.js')({ + // Inlined from e2e/utils/tsio-report-status.js (see comment above). + const PRODUCTION_URL = 'https://test-io.test.mattermost.com'; + const STAGING_URL = 'https://staging-test-io.test.mattermost.com'; + const TERMINAL_STATUSES = ['completed', 'incomplete']; + const POLL_ATTEMPTS = 6; + const POLL_DELAY_MS = 5000; + const FETCH_TIMEOUT_MS = 30_000; + const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + + async function fetchJsonWithTimeout(url, {init = {}, label, timeoutMs = FETCH_TIMEOUT_MS} = {}) { + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), timeoutMs); + try { + const res = await fetch(url, {...init, signal: controller.signal}); + if (!res.ok) { + const text = await res.text(); + throw new Error(`${label} failed: ${res.status} ${text}`); + } + return await res.json(); + } catch (error) { + if (controller.signal.aborted) { + throw new Error(`Request timed out after ${timeoutMs}ms: ${url}`); + } + throw error; + } finally { + clearTimeout(timeoutId); + } + } + + async function reportTsioStatus({ + core, context, github, compositeIdentity, totalReportsExpected, + commitStatusContext, failOnTestFailures = true, useStaging = false, + oidcAudience = 'mattermost-test-system-io', upstreamJobsSucceeded = true, + }) { + const baseUrl = useStaging ? STAGING_URL : PRODUCTION_URL; + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + let reportId; + let reportUrl; + let detail; + try { + const idToken = await core.getIDToken(oidcAudience); + core.setSecret(idToken); + const beginJson = await fetchJsonWithTimeout(`${baseUrl}/api/v1/reports/begin`, { + init: { + method: 'POST', + headers: {'Content-Type': 'application/json', Authorization: `Bearer ${idToken}`}, + body: JSON.stringify({ + repository: compositeIdentity.repository, + commit: compositeIdentity.commit_sha, + gh_run_id: compositeIdentity.gh_run_id, + gh_run_attempt: compositeIdentity.gh_run_attempt, + framework: 'playwright', + name: compositeIdentity.name, + branch: compositeIdentity.branch, + total_reports_expected: totalReportsExpected, + ...(compositeIdentity.gh_pr_number ? {gh_pr_number: parseInt(compositeIdentity.gh_pr_number, 10)} : {}), + }), + }, + label: 'reports/begin', + }); + ({report_id: reportId} = beginJson); + reportUrl = `${baseUrl}/reports/g/${reportId}`; + for (let attempt = 0; attempt < POLL_ATTEMPTS; attempt++) { + detail = await fetchJsonWithTimeout(`${baseUrl}/api/v1/reports/${reportId}`, {label: `reports/${reportId}`}); + if (TERMINAL_STATUSES.includes(detail.status)) { + break; + } + if (attempt < POLL_ATTEMPTS - 1) { + await sleep(POLL_DELAY_MS); + } + } + } catch (error) { + core.error(`TSIO reporting error: ${error.message}`); + try { + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, repo: context.repo.repo, + sha: compositeIdentity.commit_sha, state: 'failure', + context: commitStatusContext, + description: 'TSIO reporting error — see workflow run for details', + target_url: reportUrl || runUrl, + }); + } catch (statusError) { + core.warning(`Failed to create failure commit status: ${statusError.message}`); + } + throw error; + } + const stats = detail.test_stats || {}; + const isComplete = detail.status === 'completed'; + const hasFailures = (stats.failed || 0) > 0; + const overallState = isComplete && !hasFailures && upstreamJobsSucceeded ? 'success' : 'failure'; + const targetUrl = isComplete ? reportUrl : runUrl; + const summaryLines = [ + `### Test System IO — ${compositeIdentity.name}`, + '', + `**Status:** ${detail.status} · **Report:** [${reportId}](${reportUrl})`, + `**Tests:** ${stats.passed ?? '?'} passed, ${stats.failed ?? '?'} failed, ${stats.flaky ?? 0} flaky, ${stats.skipped ?? '?'} skipped (of ${stats.total ?? '?'})`, + ...(upstreamJobsSucceeded ? [] : ['', ':warning: One or more CI jobs failed outside of any tracked test — forcing this status to failure even though the test stats above may show no failures.']), + ...(isComplete ? [] : ['', `:warning: Report never reached \`completed\` (stuck at \`${detail.status}\`) — see the [workflow run](${runUrl}) for the shard that didn't finish uploading.`]), + '', + ]; + await core.summary.addRaw(summaryLines.join('\n')).write(); + const descriptionPrefix = upstreamJobsSucceeded ? '' : 'CI job failed (untracked by TSIO), '; + const description = `${descriptionPrefix}${stats.passed ?? 0}/${stats.total ?? 0} passed, ${stats.failed ?? 0} failed, ${stats.skipped ?? 0} skipped`.slice(0, 140); + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, repo: context.repo.repo, + sha: compositeIdentity.commit_sha, state: overallState, + context: commitStatusContext, description, target_url: targetUrl, + }); + if (failOnTestFailures && overallState === 'failure') { + const reason = !upstreamJobsSucceeded && !hasFailures ? + 'an upstream CI job failed with no corresponding test failure' : + `status=${detail.status}, failed=${stats.failed || 0}`; + throw new Error(`TSIO report ${reportId} did not pass: ${reason}`); + } + return {reportUrl, status: detail.status, stats}; + } + + const {reportUrl, status, stats} = await reportTsioStatus({ core, context, github, compositeIdentity, totalReportsExpected, From 6fa1afdfab63cd3b8851bfacd952a66b0828fca4 Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Fri, 10 Jul 2026 07:55:24 +0530 Subject: [PATCH 18/37] fix context --- .github/workflows/e2e-functional.yml | 30 ++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e-functional.yml b/.github/workflows/e2e-functional.yml index 921fa3652e7..8105d12e07a 100644 --- a/.github/workflows/e2e-functional.yml +++ b/.github/workflows/e2e-functional.yml @@ -612,16 +612,27 @@ jobs: # Matches the artifact uploaded by e2e-functional-template.yml's e2e job. name: tsio-${{ matrix.runner }}-${{ inputs.MM_SERVER_VERSION }} + - name: e2e/check-tsio-results-present + id: tsio-results + if: steps.download-tsio-test-results.outcome == 'success' + run: | + if [ -f e2e/test-results/results.json ]; then + echo "found=true" >> "$GITHUB_OUTPUT" + else + echo "found=false" >> "$GITHUB_OUTPUT" + echo "::warning::Downloaded TSIO artifact is missing e2e/test-results/results.json — skipping upload" + fi + - name: e2e/upload-report-to-tsio - if: steps.download-tsio-test-results.outcome == 'success' && hashFiles('e2e/test-results/results.json') != '' + if: steps.tsio-results.outputs.found == 'true' uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-report-upload@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 0.10.0 / 2026-05-16 with: composite-identity: ${{ needs.prepare-matrix.outputs.tsio-composite-identity }} total-reports-expected: ${{ needs.prepare-matrix.outputs.tsio-total-reports-expected }} framework: playwright github-token: ${{ secrets.GITHUB_TOKEN }} - # Matches the e2e-tests job's rendered `name: ${{ matrix.platform }}`. - gh-job-name: ${{ matrix.platform }} + # MUST match e2e-functional-template.yml's e2e job `name:` field. + gh-job-name: e2e-on-${{ matrix.runner }}-${{ inputs.MM_SERVER_VERSION }} json-path: e2e/test-results/results.json screenshots-dir: e2e/test-results @@ -650,8 +661,19 @@ jobs: with: name: policy-tsio-${{ matrix.platform }} + - name: e2e/check-tsio-results-present + id: tsio-results + if: steps.download-tsio-test-results.outcome == 'success' + run: | + if [ -f e2e/test-results/results.json ]; then + echo "found=true" >> "$GITHUB_OUTPUT" + else + echo "found=false" >> "$GITHUB_OUTPUT" + echo "::warning::Downloaded TSIO artifact is missing e2e/test-results/results.json — skipping upload" + fi + - name: e2e/upload-report-to-tsio - if: steps.download-tsio-test-results.outcome == 'success' && hashFiles('e2e/test-results/results.json') != '' + if: steps.tsio-results.outputs.found == 'true' uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-report-upload@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 0.10.0 / 2026-05-16 with: composite-identity: ${{ needs.prepare-matrix.outputs.tsio-composite-identity }} From 3acb1182450e8cdace53aa2ac4687088708e0031 Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Fri, 10 Jul 2026 08:29:45 +0530 Subject: [PATCH 19/37] fix context --- .github/scripts/prepare-tsio-artifacts.sh | 38 +++++++++++ .github/scripts/restore-tsio-artifacts.sh | 34 ++++++++++ .github/workflows/e2e-functional-template.yml | 20 ++++-- .github/workflows/e2e-functional.yml | 66 +++++++++++++------ e2e/package.json | 2 +- e2e/playwright.report-merge.config.ts | 13 ++++ 6 files changed, 148 insertions(+), 25 deletions(-) create mode 100755 .github/scripts/prepare-tsio-artifacts.sh create mode 100755 .github/scripts/restore-tsio-artifacts.sh create mode 100644 e2e/playwright.report-merge.config.ts diff --git a/.github/scripts/prepare-tsio-artifacts.sh b/.github/scripts/prepare-tsio-artifacts.sh new file mode 100755 index 00000000000..adbf919ba46 --- /dev/null +++ b/.github/scripts/prepare-tsio-artifacts.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. +# See LICENSE.txt for license information. +# +# Stage the minimal files TSIO needs (JSON report + failure screenshots). +# Playwright also writes trace zips and per-test output under test-results/; +# those are useful for local debugging but bloat the OIDC-split CI artifact. +set -euo pipefail + +SRC="${1:-e2e/test-results}" +DEST="${2:-tsio-artifact/e2e/test-results}" +GENERATE_FROM_BLOB="${3:-false}" + +rm -rf "$(dirname "$DEST")" +mkdir -p "$DEST" + +json="$SRC/results.json" + +if [ ! -f "$json" ] && [ "$GENERATE_FROM_BLOB" = "true" ] && [ -d e2e/blob-report ]; then + echo "Synthesizing results.json from blob-report..." + (cd e2e && npx playwright merge-reports --config playwright.report-merge.config.ts blob-report) +fi + +if [ ! -f "$json" ]; then + echo "::warning::No Playwright results.json at ${json} — TSIO upload will be skipped" + echo "has_results=false" + exit 0 +fi + +cp "$json" "$DEST/" + +while IFS= read -r -d '' png; do + rel="${png#"${SRC}"/}" + mkdir -p "$DEST/$(dirname "$rel")" + cp "$png" "$DEST/$rel" +done < <(find "$SRC" -name '*.png' -type f -print0) + +echo "has_results=true" diff --git a/.github/scripts/restore-tsio-artifacts.sh b/.github/scripts/restore-tsio-artifacts.sh new file mode 100755 index 00000000000..3b7ba79875d --- /dev/null +++ b/.github/scripts/restore-tsio-artifacts.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. +# See LICENSE.txt for license information. +# +# Materialize e2e/test-results/ from a downloaded TSIO artifact, tolerating +# upload-artifact root-directory differences across versions. +set -euo pipefail + +mkdir -p e2e/test-results +rm -rf e2e/test-results/* + +json="" +for candidate in \ + tsio-artifact/e2e/test-results/results.json \ + e2e/test-results-tsio/results.json \ + e2e/test-results/results.json; do + if [ -f "$candidate" ]; then + json="$candidate" + break + fi +done + +if [ -z "$json" ]; then + json="$(find . -path '*/test-results/results.json' -type f 2>/dev/null | head -1 || true)" +fi + +if [ -z "$json" ]; then + echo "::warning::Downloaded TSIO artifact is missing test-results/results.json — skipping upload" + echo "found=false" + exit 0 +fi + +cp -a "$(dirname "$json")/." e2e/test-results/ +echo "found=true" diff --git a/.github/workflows/e2e-functional-template.yml b/.github/workflows/e2e-functional-template.yml index f13061c0eef..4165da33f5d 100644 --- a/.github/workflows/e2e-functional-template.yml +++ b/.github/workflows/e2e-functional-template.yml @@ -277,15 +277,25 @@ jobs: DESKTOP_VERSION: ${{ inputs.DESKTOP_VERSION }} CI_ENVIRONMENT_NAME: ${{ env.CI_ENVIRONMENT_NAME }} - - name: e2e/upload-tsio-test-results + - name: e2e/prepare-tsio-artifacts + id: prepare-tsio if: always() && inputs.tsio-config != '' + run: | + while IFS= read -r line; do + if [[ "$line" == has_results=* ]]; then + echo "$line" >> "$GITHUB_OUTPUT" + else + echo "$line" + fi + done < <(bash .github/scripts/prepare-tsio-artifacts.sh e2e/test-results tsio-artifact/e2e/test-results false) + + - name: e2e/upload-tsio-test-results + if: always() && inputs.tsio-config != '' && steps.prepare-tsio.outputs.has_results == 'true' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: tsio-${{ inputs.runs-on }}-${{ inputs.MM_SERVER_VERSION }} - path: | - e2e/test-results/results.json - e2e/test-results - if-no-files-found: ignore + path: tsio-artifact + if-no-files-found: error - name: e2e/handle-nonzero-playwright-exit if: always() diff --git a/.github/workflows/e2e-functional.yml b/.github/workflows/e2e-functional.yml index 8105d12e07a..a3c83fb7824 100644 --- a/.github/workflows/e2e-functional.yml +++ b/.github/workflows/e2e-functional.yml @@ -537,6 +537,7 @@ jobs: echo "PLAYWRIGHT_EXIT_CODE=$?" >> $GITHUB_ENV npm run send-report || true env: + CI: true SERVER_VERSION: ${{ inputs.MM_SERVER_VERSION }} DESKTOP_VERSION: ${{ inputs.version_name }} @@ -549,20 +550,31 @@ jobs: echo "PLAYWRIGHT_EXIT_CODE=$?" >> $GITHUB_ENV npm run send-report || true env: + CI: true SERVER_VERSION: ${{ inputs.MM_SERVER_VERSION }} DESKTOP_VERSION: ${{ inputs.version_name }} # Joins the same TSIO report group as e2e-tests — gh-job-name in the # upload job must match this job's rendered name (policy-tests-). - - name: e2e/upload-tsio-test-results + - name: e2e/prepare-tsio-artifacts + id: prepare-tsio if: always() && needs.prepare-matrix.outputs.tsio-composite-identity != '' + run: | + while IFS= read -r line; do + if [[ "$line" == has_results=* ]]; then + echo "$line" >> "$GITHUB_OUTPUT" + else + echo "$line" + fi + done < <(bash .github/scripts/prepare-tsio-artifacts.sh e2e/test-results tsio-artifact/e2e/test-results true) + + - name: e2e/upload-tsio-test-results + if: always() && needs.prepare-matrix.outputs.tsio-composite-identity != '' && steps.prepare-tsio.outputs.has_results == 'true' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: policy-tsio-${{ matrix.platform }} - path: | - e2e/test-results/results.json - e2e/test-results - if-no-files-found: ignore + path: tsio-artifact + if-no-files-found: error - name: e2e/handle-nonzero-playwright-exit if: always() @@ -604,6 +616,13 @@ jobs: id-token: write actions: read steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + sparse-checkout: | + .github/scripts/restore-tsio-artifacts.sh + sparse-checkout-cone-mode: false + - name: e2e/download-tsio-test-results id: download-tsio-test-results uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 @@ -612,16 +631,17 @@ jobs: # Matches the artifact uploaded by e2e-functional-template.yml's e2e job. name: tsio-${{ matrix.runner }}-${{ inputs.MM_SERVER_VERSION }} - - name: e2e/check-tsio-results-present + - name: e2e/restore-tsio-artifacts id: tsio-results if: steps.download-tsio-test-results.outcome == 'success' run: | - if [ -f e2e/test-results/results.json ]; then - echo "found=true" >> "$GITHUB_OUTPUT" - else - echo "found=false" >> "$GITHUB_OUTPUT" - echo "::warning::Downloaded TSIO artifact is missing e2e/test-results/results.json — skipping upload" - fi + while IFS= read -r line; do + if [[ "$line" == found=* ]]; then + echo "$line" >> "$GITHUB_OUTPUT" + else + echo "$line" + fi + done < <(bash .github/scripts/restore-tsio-artifacts.sh) - name: e2e/upload-report-to-tsio if: steps.tsio-results.outputs.found == 'true' @@ -654,6 +674,13 @@ jobs: id-token: write actions: read steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + sparse-checkout: | + .github/scripts/restore-tsio-artifacts.sh + sparse-checkout-cone-mode: false + - name: e2e/download-tsio-test-results id: download-tsio-test-results uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 @@ -661,16 +688,17 @@ jobs: with: name: policy-tsio-${{ matrix.platform }} - - name: e2e/check-tsio-results-present + - name: e2e/restore-tsio-artifacts id: tsio-results if: steps.download-tsio-test-results.outcome == 'success' run: | - if [ -f e2e/test-results/results.json ]; then - echo "found=true" >> "$GITHUB_OUTPUT" - else - echo "found=false" >> "$GITHUB_OUTPUT" - echo "::warning::Downloaded TSIO artifact is missing e2e/test-results/results.json — skipping upload" - fi + while IFS= read -r line; do + if [[ "$line" == found=* ]]; then + echo "$line" >> "$GITHUB_OUTPUT" + else + echo "$line" + fi + done < <(bash .github/scripts/restore-tsio-artifacts.sh) - name: e2e/upload-report-to-tsio if: steps.tsio-results.outputs.found == 'true' diff --git a/e2e/package.json b/e2e/package.json index af1295dd104..f3877c0965e 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -7,7 +7,7 @@ "test": "playwright test", "enable-public-links": "tsx scripts/enable-public-links.ts", "run:policy": "cross-env RUN_POLICY_E2E=true playwright test specs/policy/policy.test.ts", - "send-report": "playwright merge-reports --reporter=html blob-report" + "send-report": "playwright merge-reports --config playwright.report-merge.config.ts blob-report" }, "repository": { "type": "git", diff --git a/e2e/playwright.report-merge.config.ts b/e2e/playwright.report-merge.config.ts new file mode 100644 index 00000000000..875c0372f43 --- /dev/null +++ b/e2e/playwright.report-merge.config.ts @@ -0,0 +1,13 @@ +// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +import {defineConfig} from '@playwright/test'; + +// Used by `npm run send-report` after policy (and other blob-based) runs. +// Merges blob shards once into both the HTML artifact and the JSON TSIO needs. +export default defineConfig({ + reporter: [ + ['html', {open: 'never', outputFolder: 'playwright-report'}], + ['json', {outputFile: 'test-results/results.json'}], + ], +}); From 316541a1ac4b882ee1da90877f9e92d9046d15f6 Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Fri, 10 Jul 2026 09:07:11 +0530 Subject: [PATCH 20/37] fix context --- .github/scripts/prepare-tsio-artifacts.sh | 38 -- .github/scripts/restore-tsio-artifacts.sh | 34 -- .../compatibility-matrix-testing.yml | 5 + .github/workflows/e2e-functional-template.yml | 33 +- .github/workflows/e2e-functional.yml | 363 +++--------------- e2e/package.json | 2 +- e2e/playwright.report-merge.config.ts | 13 - e2e/utils/github-actions.js | 1 - e2e/utils/tsio-report-status.js | 70 ++-- 9 files changed, 91 insertions(+), 468 deletions(-) delete mode 100755 .github/scripts/prepare-tsio-artifacts.sh delete mode 100755 .github/scripts/restore-tsio-artifacts.sh delete mode 100644 e2e/playwright.report-merge.config.ts diff --git a/.github/scripts/prepare-tsio-artifacts.sh b/.github/scripts/prepare-tsio-artifacts.sh deleted file mode 100755 index adbf919ba46..00000000000 --- a/.github/scripts/prepare-tsio-artifacts.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. -# See LICENSE.txt for license information. -# -# Stage the minimal files TSIO needs (JSON report + failure screenshots). -# Playwright also writes trace zips and per-test output under test-results/; -# those are useful for local debugging but bloat the OIDC-split CI artifact. -set -euo pipefail - -SRC="${1:-e2e/test-results}" -DEST="${2:-tsio-artifact/e2e/test-results}" -GENERATE_FROM_BLOB="${3:-false}" - -rm -rf "$(dirname "$DEST")" -mkdir -p "$DEST" - -json="$SRC/results.json" - -if [ ! -f "$json" ] && [ "$GENERATE_FROM_BLOB" = "true" ] && [ -d e2e/blob-report ]; then - echo "Synthesizing results.json from blob-report..." - (cd e2e && npx playwright merge-reports --config playwright.report-merge.config.ts blob-report) -fi - -if [ ! -f "$json" ]; then - echo "::warning::No Playwright results.json at ${json} — TSIO upload will be skipped" - echo "has_results=false" - exit 0 -fi - -cp "$json" "$DEST/" - -while IFS= read -r -d '' png; do - rel="${png#"${SRC}"/}" - mkdir -p "$DEST/$(dirname "$rel")" - cp "$png" "$DEST/$rel" -done < <(find "$SRC" -name '*.png' -type f -print0) - -echo "has_results=true" diff --git a/.github/scripts/restore-tsio-artifacts.sh b/.github/scripts/restore-tsio-artifacts.sh deleted file mode 100755 index 3b7ba79875d..00000000000 --- a/.github/scripts/restore-tsio-artifacts.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. -# See LICENSE.txt for license information. -# -# Materialize e2e/test-results/ from a downloaded TSIO artifact, tolerating -# upload-artifact root-directory differences across versions. -set -euo pipefail - -mkdir -p e2e/test-results -rm -rf e2e/test-results/* - -json="" -for candidate in \ - tsio-artifact/e2e/test-results/results.json \ - e2e/test-results-tsio/results.json \ - e2e/test-results/results.json; do - if [ -f "$candidate" ]; then - json="$candidate" - break - fi -done - -if [ -z "$json" ]; then - json="$(find . -path '*/test-results/results.json' -type f 2>/dev/null | head -1 || true)" -fi - -if [ -z "$json" ]; then - echo "::warning::Downloaded TSIO artifact is missing test-results/results.json — skipping upload" - echo "found=false" - exit 0 -fi - -cp -a "$(dirname "$json")/." e2e/test-results/ -echo "found=true" diff --git a/.github/workflows/compatibility-matrix-testing.yml b/.github/workflows/compatibility-matrix-testing.yml index f1536433876..d70c90e26bd 100644 --- a/.github/workflows/compatibility-matrix-testing.yml +++ b/.github/workflows/compatibility-matrix-testing.yml @@ -20,6 +20,10 @@ concurrency: cancel-in-progress: true jobs: + # TSIO reporting model (CMT): + # 1 workflow run → 1 RC cut (DESKTOP_VERSION) → 1 resolved SHA + # → 1 composite_identity (name=cmt-desktop) → 1 TSIO report covering + # every (OS × server-version) matrix leg. ## This is picked up after the finish for cleanup upload-cmt-server-detals: runs-on: ubuntu-22.04 @@ -133,6 +137,7 @@ jobs: permissions: contents: read actions: read + id-token: write secrets: inherit with: runs-on: ${{ matrix.environment.runner }} diff --git a/.github/workflows/e2e-functional-template.yml b/.github/workflows/e2e-functional-template.yml index 4165da33f5d..926c53ba8f1 100644 --- a/.github/workflows/e2e-functional-template.yml +++ b/.github/workflows/e2e-functional-template.yml @@ -107,9 +107,10 @@ jobs: # 9.5.2) name: e2e-on-${{ inputs.runs-on }}-${{ inputs.MM_SERVER_VERSION }} runs-on: ${{ inputs.runs-on }} - # Runs untrusted PR code (npm ci / Playwright) with a read-only token. permissions: contents: read + id-token: write + actions: read defaults: run: shell: bash @@ -277,25 +278,18 @@ jobs: DESKTOP_VERSION: ${{ inputs.DESKTOP_VERSION }} CI_ENVIRONMENT_NAME: ${{ env.CI_ENVIRONMENT_NAME }} - - name: e2e/prepare-tsio-artifacts - id: prepare-tsio - if: always() && inputs.tsio-config != '' - run: | - while IFS= read -r line; do - if [[ "$line" == has_results=* ]]; then - echo "$line" >> "$GITHUB_OUTPUT" - else - echo "$line" - fi - done < <(bash .github/scripts/prepare-tsio-artifacts.sh e2e/test-results tsio-artifact/e2e/test-results false) - - - name: e2e/upload-tsio-test-results - if: always() && inputs.tsio-config != '' && steps.prepare-tsio.outputs.has_results == 'true' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + - name: e2e/upload-report-to-tsio + if: ${{ always() && inputs.tsio-config != '' && hashFiles('e2e/test-results/results.json') != '' }} + uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-report-upload@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 0.10.0 / 2026-05-16 with: - name: tsio-${{ inputs.runs-on }}-${{ inputs.MM_SERVER_VERSION }} - path: tsio-artifact - if-no-files-found: error + composite-identity: ${{ toJSON(fromJSON(inputs.tsio-config).composite_identity) }} + total-reports-expected: ${{ fromJSON(inputs.tsio-config).total_reports_expected }} + framework: playwright + github-token: ${{ secrets.GITHUB_TOKEN }} + # MUST match this job's `name:` field above. + gh-job-name: e2e-on-${{ inputs.runs-on }}-${{ inputs.MM_SERVER_VERSION }} + json-path: e2e/test-results/results.json + screenshots-dir: e2e/test-results - name: e2e/handle-nonzero-playwright-exit if: always() @@ -312,4 +306,3 @@ jobs: env: TSIO_CONFIG: ${{ inputs.tsio-config }} - diff --git a/.github/workflows/e2e-functional.yml b/.github/workflows/e2e-functional.yml index a3c83fb7824..c91fd35fb0a 100644 --- a/.github/workflows/e2e-functional.yml +++ b/.github/workflows/e2e-functional.yml @@ -39,11 +39,16 @@ permissions: contents: read jobs: + # TSIO reporting model (PR / master / release): + # 1 workflow run → 1 composite_identity (gh_run_id + resolved commit SHA + name) + # → 1 TSIO report group aggregating every platform leg + both policy legs. + # Matterwick sets run_type=MASTER for master pushes; PR runs default to desktop-pr. prepare-matrix: if: ${{ github.event_name == 'workflow_dispatch' && inputs.instance_details != '' }} runs-on: ubuntu-latest outputs: platforms: ${{ steps.generate.outputs.platforms }} + desktop-sha: ${{ steps.resolve-sha.outputs.sha }} tsio-composite-identity: ${{ steps.tsio-identity.outputs.composite-identity-json }} tsio-total-reports-expected: ${{ steps.tsio-identity.outputs.total-reports-expected }} steps: @@ -56,15 +61,26 @@ jobs: platforms=$(echo "${INSTANCE_DETAILS}" | jq -c 'map(if .runner == "macos-latest" then .runner = "macos-26" else . end)') echo "platforms=${platforms}" >> "$GITHUB_OUTPUT" + # Resolve the SHA under test from version_name (branch/tag/SHA), not github.sha + # from the dispatch ref — same approach as compatibility-matrix-testing.yml. + - name: e2e/checkout-desktop-version + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ inputs.version_name }} + + - id: resolve-sha + run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + - id: tsio-identity env: GITHUB_REPOSITORY: ${{ github.repository }} - MM_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + MM_SHA: ${{ steps.resolve-sha.outputs.sha }} MM_BRANCH: ${{ inputs.version_name }} PR_NUMBER: ${{ inputs.pr_number }} RUN_TYPE: ${{ inputs.run_type || 'PR' }} PLATFORMS: ${{ steps.generate.outputs.platforms }} run: | + # platform legs (linux/macos/windows) + policy-tests-macos + policy-tests-windows TOTAL=$(( $(echo "${PLATFORMS}" | jq -c 'length') + 2 )) echo "total-reports-expected=${TOTAL}" >> "$GITHUB_OUTPUT" NAME="desktop-$(echo "${RUN_TYPE}" | tr '[:upper:]' '[:lower:]')" @@ -98,7 +114,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: repository_full_name: ${{ github.repository }} - commit_sha: ${{ github.event.pull_request.head.sha || github.sha }} + commit_sha: ${{ needs.prepare-matrix.outputs.desktop-sha }} context: e2e-test/desktop-playwright description: "Running Electron Playwright E2E tests..." status: pending @@ -115,6 +131,7 @@ jobs: permissions: contents: read actions: read + id-token: write uses: ./.github/workflows/e2e-functional-template.yml with: runs-on: ${{ matrix.runner }} @@ -141,21 +158,20 @@ jobs: - prepare-matrix - e2e-tests - e2e-policy-tests - - e2e-tsio-upload - - e2e-policy-tsio-upload if: always() permissions: contents: read id-token: write statuses: write steps: - # No checkout / no require(): this job holds id-token + statuses: write, - # so it must never load a file from the untrusted PR head. The helper can't - # be loaded from `ref: master` either — e2e/utils/tsio-report-status.js is - # new in this PR and isn't on master until this merges. So the logic is - # INLINED below (the "inlining the logic instead" alternative CodeRabbit - # suggested for the trusted-source review comment). The file copy is kept - # for compatibility-matrix-testing.yml, which runs on trusted release refs. + # tsio-report-status.js is repo code, not fetched by github-script itself. + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ inputs.version_name }} + persist-credentials: false + sparse-checkout: | + e2e/utils/tsio-report-status.js + sparse-checkout-cone-mode: false - name: Render TSIO summary + flip commit status id: summary @@ -167,177 +183,12 @@ jobs: UPSTREAM_JOBS_SUCCEEDED: ${{ needs.e2e-tests.result == 'success' && needs.e2e-policy-tests.result == 'success' }} with: script: | - const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; - const commitStatusContext = process.env.COMMIT_STATUS_CONTEXT; - const identityRaw = process.env.TSIO_COMPOSITE_IDENTITY; - let compositeIdentity; - let totalReportsExpected; - - if (!identityRaw) { - await github.rest.repos.createCommitStatus({ - owner: context.repo.owner, - repo: context.repo.repo, - sha: context.sha, - state: 'failure', - context: commitStatusContext, - description: 'TSIO summary failed — missing composite identity', - target_url: runUrl, - }); - throw new Error('TSIO_COMPOSITE_IDENTITY is missing (prepare-matrix failed or was skipped)'); - } - - try { - compositeIdentity = JSON.parse(identityRaw); - totalReportsExpected = parseInt(process.env.TSIO_TOTAL_REPORTS_EXPECTED, 10); - } catch (parseError) { - await github.rest.repos.createCommitStatus({ - owner: context.repo.owner, - repo: context.repo.repo, - sha: context.sha, - state: 'failure', - context: commitStatusContext, - description: 'TSIO summary failed — invalid composite identity', - target_url: runUrl, - }); - throw parseError; - } - - if (!compositeIdentity?.commit_sha || !Number.isFinite(totalReportsExpected)) { - await github.rest.repos.createCommitStatus({ - owner: context.repo.owner, - repo: context.repo.repo, - sha: compositeIdentity?.commit_sha || context.sha, - state: 'failure', - context: commitStatusContext, - description: 'TSIO summary failed — incomplete composite identity', - target_url: runUrl, - }); - throw new Error('TSIO composite identity or totalReportsExpected is incomplete'); - } - - // Inlined from e2e/utils/tsio-report-status.js (see comment above). - const PRODUCTION_URL = 'https://test-io.test.mattermost.com'; - const STAGING_URL = 'https://staging-test-io.test.mattermost.com'; - const TERMINAL_STATUSES = ['completed', 'incomplete']; - const POLL_ATTEMPTS = 6; - const POLL_DELAY_MS = 5000; - const FETCH_TIMEOUT_MS = 30_000; - const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); - - async function fetchJsonWithTimeout(url, {init = {}, label, timeoutMs = FETCH_TIMEOUT_MS} = {}) { - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), timeoutMs); - try { - const res = await fetch(url, {...init, signal: controller.signal}); - if (!res.ok) { - const text = await res.text(); - throw new Error(`${label} failed: ${res.status} ${text}`); - } - return await res.json(); - } catch (error) { - if (controller.signal.aborted) { - throw new Error(`Request timed out after ${timeoutMs}ms: ${url}`); - } - throw error; - } finally { - clearTimeout(timeoutId); - } - } - - async function reportTsioStatus({ - core, context, github, compositeIdentity, totalReportsExpected, - commitStatusContext, failOnTestFailures = true, useStaging = false, - oidcAudience = 'mattermost-test-system-io', upstreamJobsSucceeded = true, - }) { - const baseUrl = useStaging ? STAGING_URL : PRODUCTION_URL; - const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; - let reportId; - let reportUrl; - let detail; - try { - const idToken = await core.getIDToken(oidcAudience); - core.setSecret(idToken); - const beginJson = await fetchJsonWithTimeout(`${baseUrl}/api/v1/reports/begin`, { - init: { - method: 'POST', - headers: {'Content-Type': 'application/json', Authorization: `Bearer ${idToken}`}, - body: JSON.stringify({ - repository: compositeIdentity.repository, - commit: compositeIdentity.commit_sha, - gh_run_id: compositeIdentity.gh_run_id, - gh_run_attempt: compositeIdentity.gh_run_attempt, - framework: 'playwright', - name: compositeIdentity.name, - branch: compositeIdentity.branch, - total_reports_expected: totalReportsExpected, - ...(compositeIdentity.gh_pr_number ? {gh_pr_number: parseInt(compositeIdentity.gh_pr_number, 10)} : {}), - }), - }, - label: 'reports/begin', - }); - ({report_id: reportId} = beginJson); - reportUrl = `${baseUrl}/reports/g/${reportId}`; - for (let attempt = 0; attempt < POLL_ATTEMPTS; attempt++) { - detail = await fetchJsonWithTimeout(`${baseUrl}/api/v1/reports/${reportId}`, {label: `reports/${reportId}`}); - if (TERMINAL_STATUSES.includes(detail.status)) { - break; - } - if (attempt < POLL_ATTEMPTS - 1) { - await sleep(POLL_DELAY_MS); - } - } - } catch (error) { - core.error(`TSIO reporting error: ${error.message}`); - try { - await github.rest.repos.createCommitStatus({ - owner: context.repo.owner, repo: context.repo.repo, - sha: compositeIdentity.commit_sha, state: 'failure', - context: commitStatusContext, - description: 'TSIO reporting error — see workflow run for details', - target_url: reportUrl || runUrl, - }); - } catch (statusError) { - core.warning(`Failed to create failure commit status: ${statusError.message}`); - } - throw error; - } - const stats = detail.test_stats || {}; - const isComplete = detail.status === 'completed'; - const hasFailures = (stats.failed || 0) > 0; - const overallState = isComplete && !hasFailures && upstreamJobsSucceeded ? 'success' : 'failure'; - const targetUrl = isComplete ? reportUrl : runUrl; - const summaryLines = [ - `### Test System IO — ${compositeIdentity.name}`, - '', - `**Status:** ${detail.status} · **Report:** [${reportId}](${reportUrl})`, - `**Tests:** ${stats.passed ?? '?'} passed, ${stats.failed ?? '?'} failed, ${stats.flaky ?? 0} flaky, ${stats.skipped ?? '?'} skipped (of ${stats.total ?? '?'})`, - ...(upstreamJobsSucceeded ? [] : ['', ':warning: One or more CI jobs failed outside of any tracked test — forcing this status to failure even though the test stats above may show no failures.']), - ...(isComplete ? [] : ['', `:warning: Report never reached \`completed\` (stuck at \`${detail.status}\`) — see the [workflow run](${runUrl}) for the shard that didn't finish uploading.`]), - '', - ]; - await core.summary.addRaw(summaryLines.join('\n')).write(); - const descriptionPrefix = upstreamJobsSucceeded ? '' : 'CI job failed (untracked by TSIO), '; - const description = `${descriptionPrefix}${stats.passed ?? 0}/${stats.total ?? 0} passed, ${stats.failed ?? 0} failed, ${stats.skipped ?? 0} skipped`.slice(0, 140); - await github.rest.repos.createCommitStatus({ - owner: context.repo.owner, repo: context.repo.repo, - sha: compositeIdentity.commit_sha, state: overallState, - context: commitStatusContext, description, target_url: targetUrl, - }); - if (failOnTestFailures && overallState === 'failure') { - const reason = !upstreamJobsSucceeded && !hasFailures ? - 'an upstream CI job failed with no corresponding test failure' : - `status=${detail.status}, failed=${stats.failed || 0}`; - throw new Error(`TSIO report ${reportId} did not pass: ${reason}`); - } - return {reportUrl, status: detail.status, stats}; - } - - const {reportUrl, status, stats} = await reportTsioStatus({ + const {reportUrl, status, stats} = await require('./e2e/utils/tsio-report-status.js')({ core, context, github, - compositeIdentity, - totalReportsExpected, + compositeIdentity: JSON.parse(process.env.TSIO_COMPOSITE_IDENTITY), + totalReportsExpected: parseInt(process.env.TSIO_TOTAL_REPORTS_EXPECTED, 10), upstreamJobsSucceeded: process.env.UPSTREAM_JOBS_SUCCEEDED === 'true', - commitStatusContext, + commitStatusContext: process.env.COMMIT_STATUS_CONTEXT, failOnTestFailures: true, }); core.info(`TSIO report ${reportUrl}: ${status} (${JSON.stringify(stats)})`); @@ -411,10 +262,12 @@ jobs: needs: - prepare-matrix - update-initial-status - # Runs untrusted PR code (npm ci / Playwright); no OIDC here — TSIO upload - # is handled by e2e-policy-tsio-upload after artifacts are saved. + # Runs untrusted PR code (npm ci / Playwright); grant only what the TSIO + # upload needs, never pull-requests: write. permissions: contents: read + id-token: write + actions: read strategy: matrix: include: @@ -537,7 +390,6 @@ jobs: echo "PLAYWRIGHT_EXIT_CODE=$?" >> $GITHUB_ENV npm run send-report || true env: - CI: true SERVER_VERSION: ${{ inputs.MM_SERVER_VERSION }} DESKTOP_VERSION: ${{ inputs.version_name }} @@ -550,31 +402,22 @@ jobs: echo "PLAYWRIGHT_EXIT_CODE=$?" >> $GITHUB_ENV npm run send-report || true env: - CI: true SERVER_VERSION: ${{ inputs.MM_SERVER_VERSION }} DESKTOP_VERSION: ${{ inputs.version_name }} - # Joins the same TSIO report group as e2e-tests — gh-job-name in the - # upload job must match this job's rendered name (policy-tests-). - - name: e2e/prepare-tsio-artifacts - id: prepare-tsio - if: always() && needs.prepare-matrix.outputs.tsio-composite-identity != '' - run: | - while IFS= read -r line; do - if [[ "$line" == has_results=* ]]; then - echo "$line" >> "$GITHUB_OUTPUT" - else - echo "$line" - fi - done < <(bash .github/scripts/prepare-tsio-artifacts.sh e2e/test-results tsio-artifact/e2e/test-results true) - - - name: e2e/upload-tsio-test-results - if: always() && needs.prepare-matrix.outputs.tsio-composite-identity != '' && steps.prepare-tsio.outputs.has_results == 'true' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + # Joins the same TSIO report group as e2e-tests — gh-job-name must match + # this job's rendered name (policy-tests-) above. + - name: e2e/upload-report-to-tsio + if: ${{ always() && needs.prepare-matrix.outputs.tsio-composite-identity != '' && hashFiles('e2e/test-results/results.json') != '' }} + uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-report-upload@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 0.10.0 / 2026-05-16 with: - name: policy-tsio-${{ matrix.platform }} - path: tsio-artifact - if-no-files-found: error + composite-identity: ${{ needs.prepare-matrix.outputs.tsio-composite-identity }} + total-reports-expected: ${{ needs.prepare-matrix.outputs.tsio-total-reports-expected }} + framework: playwright + github-token: ${{ secrets.GITHUB_TOKEN }} + gh-job-name: policy-tests-${{ matrix.platform }} + json-path: e2e/test-results/results.json + screenshots-dir: e2e/test-results - name: e2e/handle-nonzero-playwright-exit if: always() @@ -599,115 +442,3 @@ jobs: path: e2e/playwright-report if-no-files-found: ignore retention-days: 7 - - e2e-tsio-upload: - name: tsio-upload-${{ matrix.platform }} - needs: - - prepare-matrix - - e2e-tests - if: always() && needs.prepare-matrix.outputs.tsio-composite-identity != '' - strategy: - fail-fast: false - matrix: - include: ${{ fromJson(needs.prepare-matrix.outputs.platforms) }} - runs-on: ubuntu-24.04 - permissions: - contents: read - id-token: write - actions: read - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - persist-credentials: false - sparse-checkout: | - .github/scripts/restore-tsio-artifacts.sh - sparse-checkout-cone-mode: false - - - name: e2e/download-tsio-test-results - id: download-tsio-test-results - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - continue-on-error: true - with: - # Matches the artifact uploaded by e2e-functional-template.yml's e2e job. - name: tsio-${{ matrix.runner }}-${{ inputs.MM_SERVER_VERSION }} - - - name: e2e/restore-tsio-artifacts - id: tsio-results - if: steps.download-tsio-test-results.outcome == 'success' - run: | - while IFS= read -r line; do - if [[ "$line" == found=* ]]; then - echo "$line" >> "$GITHUB_OUTPUT" - else - echo "$line" - fi - done < <(bash .github/scripts/restore-tsio-artifacts.sh) - - - name: e2e/upload-report-to-tsio - if: steps.tsio-results.outputs.found == 'true' - uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-report-upload@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 0.10.0 / 2026-05-16 - with: - composite-identity: ${{ needs.prepare-matrix.outputs.tsio-composite-identity }} - total-reports-expected: ${{ needs.prepare-matrix.outputs.tsio-total-reports-expected }} - framework: playwright - github-token: ${{ secrets.GITHUB_TOKEN }} - # MUST match e2e-functional-template.yml's e2e job `name:` field. - gh-job-name: e2e-on-${{ matrix.runner }}-${{ inputs.MM_SERVER_VERSION }} - json-path: e2e/test-results/results.json - screenshots-dir: e2e/test-results - - e2e-policy-tsio-upload: - name: policy-tsio-upload-${{ matrix.platform }} - needs: - - prepare-matrix - - e2e-policy-tests - if: always() && needs.prepare-matrix.outputs.tsio-composite-identity != '' - strategy: - matrix: - include: - - platform: macos - - platform: windows - fail-fast: false - runs-on: ubuntu-24.04 - permissions: - contents: read - id-token: write - actions: read - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - persist-credentials: false - sparse-checkout: | - .github/scripts/restore-tsio-artifacts.sh - sparse-checkout-cone-mode: false - - - name: e2e/download-tsio-test-results - id: download-tsio-test-results - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - continue-on-error: true - with: - name: policy-tsio-${{ matrix.platform }} - - - name: e2e/restore-tsio-artifacts - id: tsio-results - if: steps.download-tsio-test-results.outcome == 'success' - run: | - while IFS= read -r line; do - if [[ "$line" == found=* ]]; then - echo "$line" >> "$GITHUB_OUTPUT" - else - echo "$line" - fi - done < <(bash .github/scripts/restore-tsio-artifacts.sh) - - - name: e2e/upload-report-to-tsio - if: steps.tsio-results.outputs.found == 'true' - uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-report-upload@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 0.10.0 / 2026-05-16 - with: - composite-identity: ${{ needs.prepare-matrix.outputs.tsio-composite-identity }} - total-reports-expected: ${{ needs.prepare-matrix.outputs.tsio-total-reports-expected }} - framework: playwright - github-token: ${{ secrets.GITHUB_TOKEN }} - gh-job-name: policy-tests-${{ matrix.platform }} - json-path: e2e/test-results/results.json - screenshots-dir: e2e/test-results diff --git a/e2e/package.json b/e2e/package.json index f3877c0965e..af1295dd104 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -7,7 +7,7 @@ "test": "playwright test", "enable-public-links": "tsx scripts/enable-public-links.ts", "run:policy": "cross-env RUN_POLICY_E2E=true playwright test specs/policy/policy.test.ts", - "send-report": "playwright merge-reports --config playwright.report-merge.config.ts blob-report" + "send-report": "playwright merge-reports --reporter=html blob-report" }, "repository": { "type": "git", diff --git a/e2e/playwright.report-merge.config.ts b/e2e/playwright.report-merge.config.ts deleted file mode 100644 index 875c0372f43..00000000000 --- a/e2e/playwright.report-merge.config.ts +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. - -import {defineConfig} from '@playwright/test'; - -// Used by `npm run send-report` after policy (and other blob-based) runs. -// Merges blob shards once into both the HTML artifact and the JSON TSIO needs. -export default defineConfig({ - reporter: [ - ['html', {open: 'never', outputFolder: 'playwright-report'}], - ['json', {outputFile: 'test-results/results.json'}], - ], -}); diff --git a/e2e/utils/github-actions.js b/e2e/utils/github-actions.js index 94ba8386e80..987a2f6ca9b 100644 --- a/e2e/utils/github-actions.js +++ b/e2e/utils/github-actions.js @@ -28,7 +28,6 @@ async function markE2EStatusesCancelled({github, context, sha, reason = CANCELLE }); } catch (error) { console.log(`Could not update ${E2E_STATUS_CONTEXT} on ${sha}: ${error.message}`); - throw error; } } diff --git a/e2e/utils/tsio-report-status.js b/e2e/utils/tsio-report-status.js index c8a94c17f14..97673957561 100644 --- a/e2e/utils/tsio-report-status.js +++ b/e2e/utils/tsio-report-status.js @@ -8,31 +8,9 @@ const STAGING_URL = 'https://staging-test-io.test.mattermost.com'; const TERMINAL_STATUSES = ['completed', 'incomplete']; const POLL_ATTEMPTS = 6; const POLL_DELAY_MS = 5000; -const FETCH_TIMEOUT_MS = 30_000; const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); -async function fetchJsonWithTimeout(url, {init = {}, label, timeoutMs = FETCH_TIMEOUT_MS} = {}) { - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), timeoutMs); - - try { - const res = await fetch(url, {...init, signal: controller.signal}); - if (!res.ok) { - const text = await res.text(); - throw new Error(`${label} failed: ${res.status} ${text}`); - } - return await res.json(); - } catch (error) { - if (controller.signal.aborted) { - throw new Error(`Request timed out after ${timeoutMs}ms: ${url}`); - } - throw error; - } finally { - clearTimeout(timeoutId); - } -} - /** * Recover a report group's id via the idempotent begin endpoint, poll the * public status endpoint until the group leaves in_progress, render a step @@ -80,37 +58,39 @@ async function reportTsioStatus({ const idToken = await core.getIDToken(oidcAudience); core.setSecret(idToken); - const beginJson = await fetchJsonWithTimeout(`${baseUrl}/api/v1/reports/begin`, { - init: { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${idToken}`, - }, - body: JSON.stringify({ - repository: compositeIdentity.repository, - commit: compositeIdentity.commit_sha, - gh_run_id: compositeIdentity.gh_run_id, - gh_run_attempt: compositeIdentity.gh_run_attempt, - framework: 'playwright', - name: compositeIdentity.name, - branch: compositeIdentity.branch, - total_reports_expected: totalReportsExpected, - ...(compositeIdentity.gh_pr_number ? {gh_pr_number: parseInt(compositeIdentity.gh_pr_number, 10)} : {}), - }), + const beginRes = await fetch(`${baseUrl}/api/v1/reports/begin`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${idToken}`, }, - label: 'reports/begin', + body: JSON.stringify({ + repository: compositeIdentity.repository, + commit: compositeIdentity.commit_sha, + gh_run_id: compositeIdentity.gh_run_id, + gh_run_attempt: compositeIdentity.gh_run_attempt, + framework: 'playwright', + name: compositeIdentity.name, + branch: compositeIdentity.branch, + total_reports_expected: totalReportsExpected, + ...(compositeIdentity.gh_pr_number ? {gh_pr_number: parseInt(compositeIdentity.gh_pr_number, 10)} : {}), + }), }); - ({report_id: reportId} = beginJson); + if (!beginRes.ok) { + throw new Error(`reports/begin failed: ${beginRes.status} ${await beginRes.text()}`); + } + ({report_id: reportId} = await beginRes.json()); // /reports/{id} (no prefix) hits the frontend's repo-or-sha catch-all route, // not the report detail page — it needs the g/ (group) prefix. reportUrl = `${baseUrl}/reports/g/${reportId}`; for (let attempt = 0; attempt < POLL_ATTEMPTS; attempt++) { - detail = await fetchJsonWithTimeout(`${baseUrl}/api/v1/reports/${reportId}`, { - label: `reports/${reportId}`, - }); + const statusRes = await fetch(`${baseUrl}/api/v1/reports/${reportId}`); + if (!statusRes.ok) { + throw new Error(`reports/${reportId} failed: ${statusRes.status} ${await statusRes.text()}`); + } + detail = await statusRes.json(); if (TERMINAL_STATUSES.includes(detail.status)) { break; } From 535721bfb1bf797a9ccb40422583825f1b14146b Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Fri, 10 Jul 2026 09:56:43 +0530 Subject: [PATCH 21/37] report uploading --- .../compatibility-matrix-testing.yml | 2 + .github/workflows/e2e-functional-template.yml | 7 +++ .github/workflows/e2e-functional.yml | 9 +++ e2e/utils/tsio-report-status.js | 56 ++++++++++++++++--- e2e/utils/write-tsio-failure-stub.mjs | 49 ++++++++++++++++ 5 files changed, 114 insertions(+), 9 deletions(-) create mode 100644 e2e/utils/write-tsio-failure-stub.mjs diff --git a/.github/workflows/compatibility-matrix-testing.yml b/.github/workflows/compatibility-matrix-testing.yml index d70c90e26bd..b548a9b5479 100644 --- a/.github/workflows/compatibility-matrix-testing.yml +++ b/.github/workflows/compatibility-matrix-testing.yml @@ -176,6 +176,8 @@ jobs: env: TSIO_COMPOSITE_IDENTITY: ${{ needs.calculate-commit-hash.outputs.TSIO_COMPOSITE_IDENTITY }} TSIO_TOTAL_REPORTS_EXPECTED: ${{ needs.calculate-commit-hash.outputs.TSIO_TOTAL_REPORTS_EXPECTED }} + TSIO_POLL_ATTEMPTS: 24 + TSIO_POLL_DELAY_MS: 5000 COMMIT_STATUS_CONTEXT: e2e/compatibility-matrix-testing # TSIO only sees test-case-level results — a job-level failure with no # matching failed test (hung worker teardown, crashed runner, npm ci diff --git a/.github/workflows/e2e-functional-template.yml b/.github/workflows/e2e-functional-template.yml index 926c53ba8f1..d2e23843218 100644 --- a/.github/workflows/e2e-functional-template.yml +++ b/.github/workflows/e2e-functional-template.yml @@ -278,6 +278,13 @@ jobs: DESKTOP_VERSION: ${{ inputs.DESKTOP_VERSION }} CI_ENVIRONMENT_NAME: ${{ env.CI_ENVIRONMENT_NAME }} + - name: e2e/write-tsio-failure-stub + if: ${{ always() && inputs.tsio-config != '' && hashFiles('e2e/test-results/results.json') == '' && (failure() || cancelled() || (env.PLAYWRIGHT_EXIT_CODE != '0' && env.PLAYWRIGHT_EXIT_CODE != '')) }} + run: node e2e/utils/write-tsio-failure-stub.mjs + env: + TSIO_GH_JOB_NAME: e2e-on-${{ inputs.runs-on }}-${{ inputs.MM_SERVER_VERSION }} + PLAYWRIGHT_EXIT_CODE: ${{ env.PLAYWRIGHT_EXIT_CODE }} + - name: e2e/upload-report-to-tsio if: ${{ always() && inputs.tsio-config != '' && hashFiles('e2e/test-results/results.json') != '' }} uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-report-upload@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 0.10.0 / 2026-05-16 diff --git a/.github/workflows/e2e-functional.yml b/.github/workflows/e2e-functional.yml index c91fd35fb0a..f69cd8c0db1 100644 --- a/.github/workflows/e2e-functional.yml +++ b/.github/workflows/e2e-functional.yml @@ -179,6 +179,8 @@ jobs: env: TSIO_COMPOSITE_IDENTITY: ${{ needs.prepare-matrix.outputs.tsio-composite-identity }} TSIO_TOTAL_REPORTS_EXPECTED: ${{ needs.prepare-matrix.outputs.tsio-total-reports-expected }} + TSIO_POLL_ATTEMPTS: 12 + TSIO_POLL_DELAY_MS: 5000 COMMIT_STATUS_CONTEXT: e2e-test/desktop-playwright UPSTREAM_JOBS_SUCCEEDED: ${{ needs.e2e-tests.result == 'success' && needs.e2e-policy-tests.result == 'success' }} with: @@ -407,6 +409,13 @@ jobs: # Joins the same TSIO report group as e2e-tests — gh-job-name must match # this job's rendered name (policy-tests-) above. + - name: e2e/write-tsio-failure-stub + if: ${{ always() && needs.prepare-matrix.outputs.tsio-composite-identity != '' && hashFiles('e2e/test-results/results.json') == '' && (failure() || cancelled() || (env.PLAYWRIGHT_EXIT_CODE != '0' && env.PLAYWRIGHT_EXIT_CODE != '')) }} + run: node e2e/utils/write-tsio-failure-stub.mjs + env: + TSIO_GH_JOB_NAME: policy-tests-${{ matrix.platform }} + PLAYWRIGHT_EXIT_CODE: ${{ env.PLAYWRIGHT_EXIT_CODE }} + - name: e2e/upload-report-to-tsio if: ${{ always() && needs.prepare-matrix.outputs.tsio-composite-identity != '' && hashFiles('e2e/test-results/results.json') != '' }} uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-report-upload@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 0.10.0 / 2026-05-16 diff --git a/e2e/utils/tsio-report-status.js b/e2e/utils/tsio-report-status.js index 97673957561..08cf5ca4480 100644 --- a/e2e/utils/tsio-report-status.js +++ b/e2e/utils/tsio-report-status.js @@ -6,11 +6,23 @@ const PRODUCTION_URL = 'https://test-io.test.mattermost.com'; const STAGING_URL = 'https://staging-test-io.test.mattermost.com'; const TERMINAL_STATUSES = ['completed', 'incomplete']; -const POLL_ATTEMPTS = 6; -const POLL_DELAY_MS = 5000; const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +function intEnv(name, fallback) { + const raw = process.env[name]; + if (raw === undefined || raw === '') { + return fallback; + } + const n = Number(raw); + return Number.isInteger(n) && n > 0 ? n : fallback; +} + +function positiveInt(value, fallback) { + const n = Number(value); + return Number.isInteger(n) && n > 0 ? n : fallback; +} + /** * Recover a report group's id via the idempotent begin endpoint, poll the * public status endpoint until the group leaves in_progress, render a step @@ -30,6 +42,10 @@ const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); * results, so a job-level failure with no failing test attached to it (e.g. a hung worker * teardown, a crashed runner, npm ci failing before any test ran) would otherwise still * read as "100% passed" here even though the actual CI run failed. + * @param {number} [params.pollAttempts] - How many times to poll report group status (default + * 12, or TSIO_POLL_ATTEMPTS env). CMT runs with many legs should pass a higher value. + * @param {number} [params.pollDelayMs] - Delay between polls in ms (default 5000, or + * TSIO_POLL_DELAY_MS env). * @returns {Promise<{reportUrl: string, status: string, stats: Object}>} */ async function reportTsioStatus({ @@ -43,7 +59,18 @@ async function reportTsioStatus({ useStaging = false, oidcAudience = 'mattermost-test-system-io', upstreamJobsSucceeded = true, + pollAttempts, + pollDelayMs, }) { + const resolvedPollAttempts = positiveInt( + pollAttempts ?? intEnv('TSIO_POLL_ATTEMPTS', 12), + 12, + ); + const resolvedPollDelayMs = positiveInt( + pollDelayMs ?? intEnv('TSIO_POLL_DELAY_MS', 5000), + 5000, + ); + const baseUrl = useStaging ? STAGING_URL : PRODUCTION_URL; // Fallback target for the commit status when no TSIO report ever gets @@ -85,7 +112,7 @@ async function reportTsioStatus({ // not the report detail page — it needs the g/ (group) prefix. reportUrl = `${baseUrl}/reports/g/${reportId}`; - for (let attempt = 0; attempt < POLL_ATTEMPTS; attempt++) { + for (let attempt = 0; attempt < resolvedPollAttempts; attempt++) { const statusRes = await fetch(`${baseUrl}/api/v1/reports/${reportId}`); if (!statusRes.ok) { throw new Error(`reports/${reportId} failed: ${statusRes.status} ${await statusRes.text()}`); @@ -94,8 +121,8 @@ async function reportTsioStatus({ if (TERMINAL_STATUSES.includes(detail.status)) { break; } - if (attempt < POLL_ATTEMPTS - 1) { - await sleep(POLL_DELAY_MS); + if (attempt < resolvedPollAttempts - 1) { + await sleep(resolvedPollDelayMs); } } } catch (error) { @@ -116,11 +143,17 @@ async function reportTsioStatus({ throw error; } + if (!detail) { + throw new Error('TSIO report status never returned after polling'); + } + const stats = detail.test_stats || {}; const isComplete = detail.status === 'completed'; + const isIncomplete = detail.status === 'incomplete'; + const uploadedShards = Array.isArray(detail.reports) ? detail.reports.length : 0; const hasFailures = (stats.failed || 0) > 0; const overallState = isComplete && !hasFailures && upstreamJobsSucceeded ? 'success' : 'failure'; - const targetUrl = isComplete ? reportUrl : runUrl; + const targetUrl = isComplete || isIncomplete ? reportUrl : runUrl; const summaryLines = [ `### Test System IO — ${compositeIdentity.name}`, @@ -128,12 +161,17 @@ async function reportTsioStatus({ `**Status:** ${detail.status} · **Report:** [${reportId}](${reportUrl})`, `**Tests:** ${stats.passed ?? '?'} passed, ${stats.failed ?? '?'} failed, ${stats.flaky ?? 0} flaky, ` + `${stats.skipped ?? '?'} skipped (of ${stats.total ?? '?'})`, + ...(uploadedShards > 0 ? + [`**Shards uploaded:** ${uploadedShards}/${totalReportsExpected}`] : + []), ...(upstreamJobsSucceeded ? [] : ['', ':warning: One or more CI jobs failed outside of any tracked test (e.g. a hung worker, a crashed runner) — forcing this status to failure even though the test stats above may show no failures.']), - ...(isComplete ? - [] : - ['', `:warning: Report never reached \`completed\` (stuck at \`${detail.status}\`) — see the [workflow run](${runUrl}) for the shard that didn't finish uploading.`]), + ...(isIncomplete ? + ['', `:warning: Report finalized as \`incomplete\` (${uploadedShards}/${totalReportsExpected} shards) — partial results are in the [TSIO report](${reportUrl}); see the [workflow run](${runUrl}) for missing legs.`] : + isComplete ? + [] : + ['', `:warning: Report never reached a terminal state (stuck at \`${detail.status}\`) after ${resolvedPollAttempts} polls — see the [workflow run](${runUrl}).`]), '', ]; await core.summary.addRaw(summaryLines.join('\n')).write(); diff --git a/e2e/utils/write-tsio-failure-stub.mjs b/e2e/utils/write-tsio-failure-stub.mjs new file mode 100644 index 00000000000..468d635b41a --- /dev/null +++ b/e2e/utils/write-tsio-failure-stub.mjs @@ -0,0 +1,49 @@ +// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +const OUTPUT = path.join('e2e', 'test-results', 'results.json'); + +const reason = process.env.TSIO_STUB_REASON || + (process.env.PLAYWRIGHT_EXIT_CODE && process.env.PLAYWRIGHT_EXIT_CODE !== '0' ? + `Playwright exited with code ${process.env.PLAYWRIGHT_EXIT_CODE} and no results.json was written` : + 'CI job failed before Playwright JSON results were available'); + +const jobLabel = process.env.TSIO_GH_JOB_NAME || 'unknown-job'; + +const report = { + config: { + rootDir: path.resolve('e2e'), + version: '', + }, + suites: [{ + title: '', + file: 'ci/tsio-shard-failure.ts', + column: 0, + line: 0, + specs: [{ + title: `[${jobLabel}] CI shard failure`, + ok: false, + tags: ['@tsio-stub'], + tests: [{ + timeout: 0, + annotations: [], + expectedStatus: 'passed', + projectName: jobLabel, + results: [{ + workerIndex: 0, + status: 'failed', + duration: 0, + error: {message: reason}, + }], + status: 'failed', + }], + }], + }], +}; + +fs.mkdirSync(path.dirname(OUTPUT), {recursive: true}); +fs.writeFileSync(OUTPUT, JSON.stringify(report)); +console.log(`Wrote TSIO failure stub to ${OUTPUT}: ${reason}`); From c7030971be9ea45810143331ecbc7566861e23ef Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Fri, 10 Jul 2026 15:41:58 +0530 Subject: [PATCH 22/37] fix report link --- e2e/utils/tsio-report-status.js | 88 ++++++++++++++++++++++++--------- 1 file changed, 64 insertions(+), 24 deletions(-) diff --git a/e2e/utils/tsio-report-status.js b/e2e/utils/tsio-report-status.js index 08cf5ca4480..ec0f69b4a14 100644 --- a/e2e/utils/tsio-report-status.js +++ b/e2e/utils/tsio-report-status.js @@ -23,6 +23,20 @@ function positiveInt(value, fallback) { return Number.isInteger(n) && n > 0 ? n : fallback; } +/** + * Dashboard URL keyed by display identity (repo / branch / short SHA / name). + * Lists every TSIO run for that commit+name — matches test-system-io-summary + * and test-system-io-dispatch-begin. Use for commit-status target_url. + */ +function buildDisplayReportUrl(baseUrl, compositeIdentity) { + const repoTrailing = (compositeIdentity.repository || '').split('/').pop() || compositeIdentity.repository; + const repo = encodeURIComponent(repoTrailing); + const branch = encodeURIComponent(compositeIdentity.branch || 'main'); + const shortSha = (compositeIdentity.commit_sha || '').slice(0, 7); + const name = encodeURIComponent(compositeIdentity.name); + return `${baseUrl}/reports/${repo}/${branch}/${shortSha}/${name}`; +} + /** * Recover a report group's id via the idempotent begin endpoint, poll the * public status endpoint until the group leaves in_progress, render a step @@ -79,7 +93,8 @@ async function reportTsioStatus({ const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; let reportId; - let reportUrl; + let displayReportUrl; + let groupReportUrl; let detail; try { const idToken = await core.getIDToken(oidcAudience); @@ -108,9 +123,10 @@ async function reportTsioStatus({ } ({report_id: reportId} = await beginRes.json()); - // /reports/{id} (no prefix) hits the frontend's repo-or-sha catch-all route, - // not the report detail page — it needs the g/ (group) prefix. - reportUrl = `${baseUrl}/reports/g/${reportId}`; + displayReportUrl = buildDisplayReportUrl(baseUrl, compositeIdentity); + + // Direct link to this run's merged shard group (job summary only). + groupReportUrl = `${baseUrl}/reports/g/${reportId}`; for (let attempt = 0; attempt < resolvedPollAttempts; attempt++) { const statusRes = await fetch(`${baseUrl}/api/v1/reports/${reportId}`); @@ -135,7 +151,7 @@ async function reportTsioStatus({ state: 'failure', context: commitStatusContext, description: 'TSIO reporting error — see workflow run for details', - target_url: reportUrl || runUrl, + target_url: displayReportUrl || runUrl, }); } catch (statusError) { core.warning(`Failed to create failure commit status: ${statusError.message}`); @@ -152,28 +168,49 @@ async function reportTsioStatus({ const isIncomplete = detail.status === 'incomplete'; const uploadedShards = Array.isArray(detail.reports) ? detail.reports.length : 0; const hasFailures = (stats.failed || 0) > 0; - const overallState = isComplete && !hasFailures && upstreamJobsSucceeded ? 'success' : 'failure'; - const targetUrl = isComplete || isIncomplete ? reportUrl : runUrl; + + let overallState = 'failure'; + if (isComplete && !hasFailures && upstreamJobsSucceeded) { + overallState = 'success'; + } + + let targetUrl = runUrl; + if (isComplete || isIncomplete) { + targetUrl = displayReportUrl; + } const summaryLines = [ `### Test System IO — ${compositeIdentity.name}`, '', - `**Status:** ${detail.status} · **Report:** [${reportId}](${reportUrl})`, + `**Status:** ${detail.status} · **Report:** [${compositeIdentity.name}](${displayReportUrl}) · [this run](${groupReportUrl})`, `**Tests:** ${stats.passed ?? '?'} passed, ${stats.failed ?? '?'} failed, ${stats.flaky ?? 0} flaky, ` + `${stats.skipped ?? '?'} skipped (of ${stats.total ?? '?'})`, - ...(uploadedShards > 0 ? - [`**Shards uploaded:** ${uploadedShards}/${totalReportsExpected}`] : - []), - ...(upstreamJobsSucceeded ? - [] : - ['', ':warning: One or more CI jobs failed outside of any tracked test (e.g. a hung worker, a crashed runner) — forcing this status to failure even though the test stats above may show no failures.']), - ...(isIncomplete ? - ['', `:warning: Report finalized as \`incomplete\` (${uploadedShards}/${totalReportsExpected} shards) — partial results are in the [TSIO report](${reportUrl}); see the [workflow run](${runUrl}) for missing legs.`] : - isComplete ? - [] : - ['', `:warning: Report never reached a terminal state (stuck at \`${detail.status}\`) after ${resolvedPollAttempts} polls — see the [workflow run](${runUrl}).`]), - '', ]; + + if (uploadedShards > 0) { + summaryLines.push(`**Shards uploaded:** ${uploadedShards}/${totalReportsExpected}`); + } + + if (!upstreamJobsSucceeded) { + summaryLines.push( + '', + ':warning: One or more CI jobs failed outside of any tracked test (e.g. a hung worker, a crashed runner) — forcing this status to failure even though the test stats above may show no failures.', + ); + } + + if (isIncomplete) { + summaryLines.push( + '', + `:warning: Report finalized as \`incomplete\` (${uploadedShards}/${totalReportsExpected} shards) — partial results are in the [TSIO report](${displayReportUrl}); see the [workflow run](${runUrl}) for missing legs.`, + ); + } else if (!isComplete) { + summaryLines.push( + '', + `:warning: Report never reached a terminal state (stuck at \`${detail.status}\`) after ${resolvedPollAttempts} polls — see the [workflow run](${runUrl}).`, + ); + } + + summaryLines.push(''); await core.summary.addRaw(summaryLines.join('\n')).write(); const descriptionPrefix = upstreamJobsSucceeded ? '' : 'CI job failed (untracked by TSIO), '; @@ -189,13 +226,16 @@ async function reportTsioStatus({ }); if (failOnTestFailures && overallState === 'failure') { - const reason = !upstreamJobsSucceeded && !hasFailures ? - 'an upstream CI job failed with no corresponding test failure' : - `status=${detail.status}, failed=${stats.failed || 0}`; + let reason; + if (!upstreamJobsSucceeded && !hasFailures) { + reason = 'an upstream CI job failed with no corresponding test failure'; + } else { + reason = `status=${detail.status}, failed=${stats.failed || 0}`; + } throw new Error(`TSIO report ${reportId} did not pass: ${reason}`); } - return {reportUrl, status: detail.status, stats}; + return {reportUrl: displayReportUrl, status: detail.status, stats}; } module.exports = reportTsioStatus; From 688ba42f989f28ebefffb518868608d2ec24c98c Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Fri, 10 Jul 2026 18:18:56 +0530 Subject: [PATCH 23/37] clean up PR status --- .github/workflows/ci.yaml | 1 + .github/workflows/e2e-pr-trigger.yml | 189 +++++++++++---------------- 2 files changed, 75 insertions(+), 115 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7e36c7b4ce6..9a888b9386f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -214,6 +214,7 @@ jobs: - name: ci/publish-results uses: EnricoMi/publish-unit-test-result-action@d0a4676d0e0b938bc201470d88276b7c74c712b3 # v2.24.0 with: + check_name: Unit Tests (Jest) comment_mode: failures compare_to_earlier_commit: false junit_files: "**/*.xml" diff --git a/.github/workflows/e2e-pr-trigger.yml b/.github/workflows/e2e-pr-trigger.yml index 75bde04b351..a5abbe1d369 100644 --- a/.github/workflows/e2e-pr-trigger.yml +++ b/.github/workflows/e2e-pr-trigger.yml @@ -1,29 +1,9 @@ name: E2E PR Trigger -# Automatically adds the E2E/Run label to non-draft PRs targeting master. -# Adding the label signals Matterwick to provision cloud servers and dispatch -# the Electron Playwright Tests (e2e-functional.yml) workflow. -# After tests complete, e2e-functional.yml / e2e-label-cleanup.yml remove the label. -# -# E2E/Override (same contract as mattermost-mobile): when present on a PR, -# opened/synchronize events do not add E2E/Run, and applying the override label -# strips E2E/Run and cancels in-flight E2E runs. -# -# On synchronize: in-progress Electron Playwright Tests runs for this PR are -# cancelled before re-adding the label so stale runs for the old commit do not -# block the new one. Runs on other PR branches are left running. -# -# The concurrency group ensures rapid pushes to the same PR don't queue multiple -# label operations: only the most recent push proceeds. -# -# Group key is bucketed by event, not just PR number (PR #3891 regression: a -# same-second, unrelated `labeled` event from a release-notes bot cancelled the -# `opened` event's in-progress run via cancel-in-progress before it could add -# E2E/Run, and the winning run's own job conditions didn't match that label — -# so E2E/Run never got added at all). opened/reopened/ready_for_review/synchronize -# still share one group (preserves "only the most recent push proceeds" for -# rapid pushes); every other event (any other label add/remove) gets its own -# action+label-keyed group so it can't cancel a real trigger run. +# Adds E2E/Run on non-draft PRs to master (Matterwick dispatches e2e-functional.yml). +# E2E/Override skips/cancels E2E; manual E2E/Run removal cancels in-flight runs. +# One job keeps the PR checks list to a single row. Concurrency is keyed by +# event type so unrelated label bots cannot cancel an in-progress trigger (PR #3891). on: pull_request: types: @@ -44,8 +24,8 @@ concurrency: cancel-in-progress: true jobs: - add-e2e-label: - name: Add E2E/Run label + e2e-label-orchestration: + name: E2E label runs-on: ubuntu-22.04 permissions: issues: write @@ -53,17 +33,79 @@ jobs: actions: write statuses: write if: >- - !github.event.pull_request.draft - && contains(fromJSON('["opened", "reopened", "ready_for_review", "synchronize"]'), github.event.action) + ( + !github.event.pull_request.draft + && contains(fromJSON('["opened", "reopened", "ready_for_review", "synchronize"]'), github.event.action) + ) + || ( + github.event.action == 'labeled' + && github.event.label.name == 'E2E/Override' + ) + || ( + github.event.action == 'unlabeled' + && github.event.label.name == 'E2E/Run' + && github.event.sender.login != 'github-actions[bot]' + ) steps: - # Always the base ref's own reviewed copy of github-actions.js — this job - # holds write-scoped tokens (issues/pull-requests/actions/statuses), so it - # must never execute code sourced from the untrusted PR head. + # Always the base ref's own reviewed copy — this job holds write-scoped + # tokens, so it must never execute code from the untrusted PR head. - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ github.event.pull_request.base.ref }} - - name: Cancel running E2E tests and re-trigger + - name: Cancel E2E on manual label removal + if: >- + github.event.action == 'unlabeled' + && github.event.label.name == 'E2E/Run' + && github.event.sender.login != 'github-actions[bot]' + uses: ./.github/actions/cancel-e2e-runs + with: + pr_number: ${{ github.event.pull_request.number }} + reason: E2E cancelled (E2E/Run label removed) + + - name: Honor E2E/Override + if: >- + github.event.action == 'labeled' + && github.event.label.name == 'E2E/Override' + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + github-token: ${{ github.token }} + script: | + const {cancelActiveE2ERuns, markE2EStatusesCancelled} = require('./e2e/utils/github-actions.js'); + const { owner, repo } = context.repo; + const issue_number = context.issue.number; + const pr = context.payload.pull_request; + + try { + await github.rest.issues.removeLabel({ + owner, + repo, + issue_number, + name: 'E2E/Run', + }); + } catch (error) { + if (error.status !== 404) { + throw error; + } + } + + await cancelActiveE2ERuns({ + github, + context, + prNumber: issue_number, + headBranch: pr.head.ref, + }); + await markE2EStatusesCancelled({ + github, + context, + sha: pr.head.sha, + reason: 'E2E cancelled (E2E/Override label applied)', + }); + + - name: Refresh E2E/Run label + if: >- + !github.event.pull_request.draft + && contains(fromJSON('["opened", "reopened", "ready_for_review", "synchronize"]'), github.event.action) uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: github-token: ${{ github.token }} @@ -139,86 +181,3 @@ jobs: issue_number, labels: ['E2E/Run'], }); - - honor-e2e-override: - name: Honor E2E/Override - runs-on: ubuntu-22.04 - permissions: - issues: write - pull-requests: write - actions: write - statuses: write - if: >- - github.event.action == 'labeled' - && github.event.label.name == 'E2E/Override' - steps: - # Always the base ref's own reviewed copy of github-actions.js — this job - # holds write-scoped tokens (issues/pull-requests/actions/statuses), so it - # must never execute code sourced from the untrusted PR head. - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - ref: ${{ github.event.pull_request.base.ref }} - - - name: Strip E2E/Run and cancel in-flight E2E - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - github-token: ${{ github.token }} - script: | - const {cancelActiveE2ERuns, markE2EStatusesCancelled} = require('./e2e/utils/github-actions.js'); - const { owner, repo } = context.repo; - const issue_number = context.issue.number; - const pr = context.payload.pull_request; - - try { - await github.rest.issues.removeLabel({ - owner, - repo, - issue_number, - name: 'E2E/Run', - }); - } catch (error) { - if (error.status !== 404) { - throw error; - } - } - - await cancelActiveE2ERuns({ - github, - context, - prNumber: issue_number, - headBranch: pr.head.ref, - }); - await markE2EStatusesCancelled({ - github, - context, - sha: pr.head.sha, - reason: 'E2E cancelled (E2E/Override label applied)', - }); - - cancel-on-manual-unlabel: - name: Cancel E2E on manual label removal - runs-on: ubuntu-22.04 - permissions: - actions: write - statuses: write - pull-requests: read - if: >- - github.event.action == 'unlabeled' - && github.event.label.name == 'E2E/Run' - && github.event.sender.login != 'github-actions[bot]' - steps: - # Always the base ref's own reviewed copy of github-actions.js — this job - # holds write-scoped tokens (actions/statuses), so it must never execute - # code sourced from the untrusted PR head. A single full checkout also - # sidesteps the previous bug where a second sparse checkout of head.sha - # replaced the base-ref tree, wiping out .github/actions/cancel-e2e-runs/ - # before this job's next step could use it. - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - ref: ${{ github.event.pull_request.base.ref }} - - - name: Cancel E2E runs and mark statuses skipped - uses: ./.github/actions/cancel-e2e-runs - with: - pr_number: ${{ github.event.pull_request.number }} - reason: E2E cancelled (E2E/Run label removed) From 0e92d65e6d99bb4c96bc85b42a9f44ebabc3319f Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Fri, 10 Jul 2026 19:50:12 +0530 Subject: [PATCH 24/37] Fix TSIO summary lint and failed-shard detection. Use gh_run_id on the display report URL and detect failed shards without nested ternaries so CI ESLint passes. Co-authored-by: Cursor --- e2e/utils/tsio-report-status.js | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/e2e/utils/tsio-report-status.js b/e2e/utils/tsio-report-status.js index ec0f69b4a14..ea1d26727dd 100644 --- a/e2e/utils/tsio-report-status.js +++ b/e2e/utils/tsio-report-status.js @@ -34,7 +34,8 @@ function buildDisplayReportUrl(baseUrl, compositeIdentity) { const branch = encodeURIComponent(compositeIdentity.branch || 'main'); const shortSha = (compositeIdentity.commit_sha || '').slice(0, 7); const name = encodeURIComponent(compositeIdentity.name); - return `${baseUrl}/reports/${repo}/${branch}/${shortSha}/${name}`; + const runId = encodeURIComponent(compositeIdentity.gh_run_id || ''); + return `${baseUrl}/reports/${repo}/${branch}/${shortSha}/${name}?gh_run_id=${runId}`; } /** @@ -167,7 +168,15 @@ async function reportTsioStatus({ const isComplete = detail.status === 'completed'; const isIncomplete = detail.status === 'incomplete'; const uploadedShards = Array.isArray(detail.reports) ? detail.reports.length : 0; - const hasFailures = (stats.failed || 0) > 0; + const failedShards = []; + if (Array.isArray(detail.reports)) { + for (const report of detail.reports) { + if (report.status === 'failed') { + failedShards.push(report.display_name || report.gh_job_name || report.id); + } + } + } + const hasFailures = (stats.failed || 0) > 0 || failedShards.length > 0; let overallState = 'failure'; if (isComplete && !hasFailures && upstreamJobsSucceeded) { @@ -191,6 +200,10 @@ async function reportTsioStatus({ summaryLines.push(`**Shards uploaded:** ${uploadedShards}/${totalReportsExpected}`); } + if (failedShards.length > 0) { + summaryLines.push(`**Failed shards:** ${failedShards.join(', ')}`); + } + if (!upstreamJobsSucceeded) { summaryLines.push( '', @@ -229,6 +242,8 @@ async function reportTsioStatus({ let reason; if (!upstreamJobsSucceeded && !hasFailures) { reason = 'an upstream CI job failed with no corresponding test failure'; + } else if (failedShards.length > 0 && (stats.failed || 0) === 0) { + reason = `shard(s) failed: ${failedShards.join(', ')}`; } else { reason = `status=${detail.status}, failed=${stats.failed || 0}`; } From e9b5893c24f4b34836921d735606ad2ae858f4f4 Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Fri, 10 Jul 2026 23:42:53 +0530 Subject: [PATCH 25/37] Point PR status link at this run's TSIO group report. The commit-level display URL dedupes cross-platform specs and can show Passed while GitHub reports failures; /reports/g/{id} matches the status description. Co-authored-by: Cursor --- e2e/utils/tsio-report-status.js | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/e2e/utils/tsio-report-status.js b/e2e/utils/tsio-report-status.js index ea1d26727dd..02ad5b23599 100644 --- a/e2e/utils/tsio-report-status.js +++ b/e2e/utils/tsio-report-status.js @@ -24,9 +24,9 @@ function positiveInt(value, fallback) { } /** - * Dashboard URL keyed by display identity (repo / branch / short SHA / name). - * Lists every TSIO run for that commit+name — matches test-system-io-summary - * and test-system-io-dispatch-begin. Use for commit-status target_url. + * Commit-level rollup URL (repo / branch / short SHA / name). Dedupes the same + * spec across platform shards — headline stats can disagree with GitHub when a + * failure is platform-specific. Job summary only; not for commit-status links. */ function buildDisplayReportUrl(baseUrl, compositeIdentity) { const repoTrailing = (compositeIdentity.repository || '').split('/').pop() || compositeIdentity.repository; @@ -34,8 +34,7 @@ function buildDisplayReportUrl(baseUrl, compositeIdentity) { const branch = encodeURIComponent(compositeIdentity.branch || 'main'); const shortSha = (compositeIdentity.commit_sha || '').slice(0, 7); const name = encodeURIComponent(compositeIdentity.name); - const runId = encodeURIComponent(compositeIdentity.gh_run_id || ''); - return `${baseUrl}/reports/${repo}/${branch}/${shortSha}/${name}?gh_run_id=${runId}`; + return `${baseUrl}/reports/${repo}/${branch}/${shortSha}/${name}`; } /** @@ -125,8 +124,6 @@ async function reportTsioStatus({ ({report_id: reportId} = await beginRes.json()); displayReportUrl = buildDisplayReportUrl(baseUrl, compositeIdentity); - - // Direct link to this run's merged shard group (job summary only). groupReportUrl = `${baseUrl}/reports/g/${reportId}`; for (let attempt = 0; attempt < resolvedPollAttempts; attempt++) { @@ -152,7 +149,7 @@ async function reportTsioStatus({ state: 'failure', context: commitStatusContext, description: 'TSIO reporting error — see workflow run for details', - target_url: displayReportUrl || runUrl, + target_url: groupReportUrl || displayReportUrl || runUrl, }); } catch (statusError) { core.warning(`Failed to create failure commit status: ${statusError.message}`); @@ -185,13 +182,13 @@ async function reportTsioStatus({ let targetUrl = runUrl; if (isComplete || isIncomplete) { - targetUrl = displayReportUrl; + targetUrl = groupReportUrl || displayReportUrl; } const summaryLines = [ `### Test System IO — ${compositeIdentity.name}`, '', - `**Status:** ${detail.status} · **Report:** [${compositeIdentity.name}](${displayReportUrl}) · [this run](${groupReportUrl})`, + `**Status:** ${detail.status} · **Report:** [this run](${groupReportUrl}) · [all runs for commit](${displayReportUrl})`, `**Tests:** ${stats.passed ?? '?'} passed, ${stats.failed ?? '?'} failed, ${stats.flaky ?? 0} flaky, ` + `${stats.skipped ?? '?'} skipped (of ${stats.total ?? '?'})`, ]; @@ -250,7 +247,7 @@ async function reportTsioStatus({ throw new Error(`TSIO report ${reportId} did not pass: ${reason}`); } - return {reportUrl: displayReportUrl, status: detail.status, stats}; + return {reportUrl: groupReportUrl || displayReportUrl, status: detail.status, stats}; } module.exports = reportTsioStatus; From 7622b657842dff5bcd734a8f3f1ab57db5f9c630 Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Sat, 11 Jul 2026 00:18:40 +0530 Subject: [PATCH 26/37] test fixes --- e2e/helpers/userAttributes.ts | 45 ++++++++++++++++++- .../user_attributes/user_attributes.test.ts | 2 + 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/e2e/helpers/userAttributes.ts b/e2e/helpers/userAttributes.ts index 1c0f70a4b4c..92a0c6c45b5 100644 --- a/e2e/helpers/userAttributes.ts +++ b/e2e/helpers/userAttributes.ts @@ -268,6 +268,43 @@ export async function recoverFromProfileSettings(win: ServerView): Promise await waitForChannelPostListLoaded(win); } +async function isCustomAttributeEditVisible(win: ServerView, fieldId: string): Promise { + return win.runInRenderer(` + const btn = document.querySelector('#customAttribute_${fieldId}Edit'); + if (!(btn instanceof HTMLElement)) { + return false; + } + const rect = btn.getBoundingClientRect(); + return rect.width > 0 && rect.height > 0; + `); +} + +/** Webapp caches CPA field defs; reload once if API-created fields are missing from Profile Settings. */ +export async function waitForCustomAttributeEditInProfileSettings(win: ServerView, fieldId: string): Promise { + await ensureCustomAttributeEditReady(win, fieldId); +} + +async function ensureCustomAttributeEditReady(win: ServerView, fieldId: string): Promise { + for (let attempt = 0; attempt < 2; attempt++) { + try { + await expect.poll(async () => isCustomAttributeEditVisible(win, fieldId), { + timeout: attempt === 0 ? 5_000 : 15_000, + message: `Custom attribute Edit button #customAttribute_${fieldId}Edit must be visible`, + }).toBe(true); + return; + } catch (error) { + if (attempt === 1) { + throw error; + } + await closeProfileSettings(win).catch(() => undefined); + await reloadServerView(win.app, win.webContentsId); + await waitForMattermostShellReady(win); + await dismissBlockingOverlays(win); + await openProfileSettings(win); + } + } +} + export async function getCustomAttributeLabelsInSettings(win: ServerView): Promise { return win.runInRenderer(` const modal = document.querySelector(${JSON.stringify(PROFILE_SETTINGS_MODAL_SELECTOR)}) @@ -307,11 +344,15 @@ export async function editTextCustomAttribute( newValue: string, save = true, ): Promise { + await ensureCustomAttributeEditReady(win, fieldId); await win.runInRenderer(` const fieldId = ${JSON.stringify(fieldId)}; const editBtn = document.querySelector('#customAttribute_' + fieldId + 'Edit'); - editBtn?.scrollIntoView({block: 'center'}); - editBtn?.click(); + if (!(editBtn instanceof HTMLElement)) { + throw new Error('Custom attribute Edit button not found for ' + fieldId); + } + editBtn.scrollIntoView({block: 'center'}); + editBtn.click(); `); await win.waitForSelector(`#customAttribute_${fieldId}`, {timeout: 10_000}); await win.runInRenderer(` diff --git a/e2e/specs/user_attributes/user_attributes.test.ts b/e2e/specs/user_attributes/user_attributes.test.ts index 9aa7add6c28..b79b18710f4 100644 --- a/e2e/specs/user_attributes/user_attributes.test.ts +++ b/e2e/specs/user_attributes/user_attributes.test.ts @@ -41,6 +41,7 @@ import { recoverFromProfileSettings, updateCustomProfileAttributeValues, type UserPropertyField, + waitForCustomAttributeEditInProfileSettings, } from '../../helpers/userAttributes'; const FIELD_PREFIX = 'E2E_UA_'; @@ -236,6 +237,7 @@ test.describe('user_attributes/user_attributes', () => { test.skip(true, 'Profile settings UI is not available on this server'); return; } + await waitForCustomAttributeEditInProfileSettings(win, created!.id); await win.runInRenderer(` document.querySelector('#customAttribute_${created!.id}Edit')?.click(); `); From 96cd73083b6b24da7f0af28d34b4f6e97ed31022 Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Sat, 11 Jul 2026 00:57:11 +0530 Subject: [PATCH 27/37] test fixes --- .github/workflows/e2e-functional-template.yml | 13 +++++-------- .github/workflows/e2e-functional.yml | 13 +++++-------- e2e/utils/tsio-report-status.js | 4 ++-- 3 files changed, 12 insertions(+), 18 deletions(-) diff --git a/.github/workflows/e2e-functional-template.yml b/.github/workflows/e2e-functional-template.yml index d2e23843218..8fd878698c3 100644 --- a/.github/workflows/e2e-functional-template.yml +++ b/.github/workflows/e2e-functional-template.yml @@ -301,15 +301,12 @@ jobs: - name: e2e/handle-nonzero-playwright-exit if: always() run: | - if [ "${PLAYWRIGHT_EXIT_CODE:-1}" == "0" ]; then + if [ -z "${PLAYWRIGHT_EXIT_CODE:-}" ]; then exit 0 fi - if [ -n "${TSIO_CONFIG}" ]; then - echo "::warning::Playwright exited with code ${PLAYWRIGHT_EXIT_CODE:-unset} — not failing the job on this alone, see TSIO report for actual test results" - else - echo "Playwright exited with code ${PLAYWRIGHT_EXIT_CODE:-unset} and no TSIO config is set for this run — failing the job since there is no other failure signal" >&2 - exit 1 + if [ "${PLAYWRIGHT_EXIT_CODE}" = "0" ]; then + exit 0 fi - env: - TSIO_CONFIG: ${{ inputs.tsio-config }} + echo "Playwright exited with code ${PLAYWRIGHT_EXIT_CODE}" >&2 + exit 1 diff --git a/.github/workflows/e2e-functional.yml b/.github/workflows/e2e-functional.yml index f69cd8c0db1..42c3a124ed6 100644 --- a/.github/workflows/e2e-functional.yml +++ b/.github/workflows/e2e-functional.yml @@ -431,17 +431,14 @@ jobs: - name: e2e/handle-nonzero-playwright-exit if: always() run: | - if [ "${PLAYWRIGHT_EXIT_CODE:-1}" == "0" ]; then + if [ -z "${PLAYWRIGHT_EXIT_CODE:-}" ]; then exit 0 fi - if [ -n "${TSIO_COMPOSITE_IDENTITY}" ]; then - echo "::warning::Playwright exited with code ${PLAYWRIGHT_EXIT_CODE:-unset} — not failing the job on this alone, see TSIO report for actual test results" - else - echo "Playwright exited with code ${PLAYWRIGHT_EXIT_CODE:-unset} and no TSIO config is set for this run — failing the job since there is no other failure signal" >&2 - exit 1 + if [ "${PLAYWRIGHT_EXIT_CODE}" = "0" ]; then + exit 0 fi - env: - TSIO_COMPOSITE_IDENTITY: ${{ needs.prepare-matrix.outputs.tsio-composite-identity }} + echo "Playwright exited with code ${PLAYWRIGHT_EXIT_CODE}" >&2 + exit 1 - name: Upload test results if: always() diff --git a/e2e/utils/tsio-report-status.js b/e2e/utils/tsio-report-status.js index 02ad5b23599..14fb709ef10 100644 --- a/e2e/utils/tsio-report-status.js +++ b/e2e/utils/tsio-report-status.js @@ -201,7 +201,7 @@ async function reportTsioStatus({ summaryLines.push(`**Failed shards:** ${failedShards.join(', ')}`); } - if (!upstreamJobsSucceeded) { + if (!upstreamJobsSucceeded && !hasFailures) { summaryLines.push( '', ':warning: One or more CI jobs failed outside of any tracked test (e.g. a hung worker, a crashed runner) — forcing this status to failure even though the test stats above may show no failures.', @@ -223,7 +223,7 @@ async function reportTsioStatus({ summaryLines.push(''); await core.summary.addRaw(summaryLines.join('\n')).write(); - const descriptionPrefix = upstreamJobsSucceeded ? '' : 'CI job failed (untracked by TSIO), '; + const descriptionPrefix = !upstreamJobsSucceeded && !hasFailures ? 'CI job failed (untracked by TSIO), ' : ''; const description = `${descriptionPrefix}${stats.passed ?? 0}/${stats.total ?? 0} passed, ${stats.failed ?? 0} failed, ${stats.skipped ?? 0} skipped`.slice(0, 140); await github.rest.repos.createCommitStatus({ owner: context.repo.owner, From 52084a6459ad07302003502bbbcadb600865a97b Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Sat, 11 Jul 2026 14:41:13 +0530 Subject: [PATCH 28/37] test fixes --- e2e/helpers/methodSpy.ts | 4 + e2e/helpers/modalPage.ts | 63 +++++++++++++++ e2e/helpers/notificationEffects.ts | 4 +- e2e/helpers/settingsWindow.ts | 44 +++++----- e2e/helpers/trayMenu.ts | 36 ++------- .../flash_taskbar.test.ts | 4 +- .../no_flash_taskbar.test.ts | 4 +- e2e/specs/settings/keyboard_shortcuts.test.ts | 45 ++--------- e2e/utils/tsio-report-status.js | 16 +++- src/app/mainWindow/mainWindow.ts | 3 + src/main/app/appReady.test.js | 58 ++++++++++++++ src/main/app/initialize.test.js | 3 + src/main/app/intercom.test.js | 80 ------------------- src/main/app/intercom.ts | 4 - src/main/e2e/appReady.ts | 37 ++++----- 15 files changed, 196 insertions(+), 209 deletions(-) create mode 100644 e2e/helpers/modalPage.ts create mode 100644 src/main/app/appReady.test.js diff --git a/e2e/helpers/methodSpy.ts b/e2e/helpers/methodSpy.ts index 0e314e0120a..865b5d941fd 100644 --- a/e2e/helpers/methodSpy.ts +++ b/e2e/helpers/methodSpy.ts @@ -66,3 +66,7 @@ export async function restoreFlashFrameSpy(app: ElectronApplication): Promise { + return evaluateInMainProcess(app, () => (global as any).__e2eFlashFrameCalls ?? []); +} diff --git a/e2e/helpers/modalPage.ts b/e2e/helpers/modalPage.ts new file mode 100644 index 00000000000..90464b0aa64 --- /dev/null +++ b/e2e/helpers/modalPage.ts @@ -0,0 +1,63 @@ +// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +import {expect} from '@playwright/test'; +import type {ElectronApplication} from 'playwright'; + +import {ServerView} from './serverView'; +import {evaluateInMainProcessWithArg} from './testRefs'; + +export const SETTINGS_MODAL_KEY = 'settingsModal'; + +type ModalLookupOptions = { + urlIncludes: string; +}; + +type ModalLookupResult = { + webContentsId: number; + url: string; +}; + +export async function lookupModalByUrl( + app: ElectronApplication, + options: ModalLookupOptions, +): Promise { + return evaluateInMainProcessWithArg(app, ({webContents}, payload) => { + for (const wc of webContents.getAllWebContents()) { + if (wc.isDestroyed()) { + continue; + } + try { + const url = wc.getURL(); + if (url.includes(payload.urlIncludes)) { + return {webContentsId: wc.id, url}; + } + } catch { + // Ignore webContents that disappear while iterating. + } + } + return null; + }, options); +} + +export async function waitForModalView( + app: ElectronApplication, + options: ModalLookupOptions & {timeout?: number}, +): Promise { + const timeout = options.timeout ?? 15_000; + let modalView: ServerView | undefined; + + await expect.poll(async () => { + const modal = await lookupModalByUrl(app, {urlIncludes: options.urlIncludes}); + if (!modal) { + return null; + } + modalView = new ServerView(app, modal.webContentsId); + return modalView; + }, { + timeout, + message: `WebContents with URL containing "${options.urlIncludes}" must be available`, + }).not.toBeNull(); + + return modalView!; +} diff --git a/e2e/helpers/notificationEffects.ts b/e2e/helpers/notificationEffects.ts index 1d0665cdc74..7b4629a9fb8 100644 --- a/e2e/helpers/notificationEffects.ts +++ b/e2e/helpers/notificationEffects.ts @@ -3,6 +3,8 @@ import type {ElectronApplication} from 'playwright'; +import {evaluateInMainProcessWithArg} from './testRefs'; + /** * Invoke the E2E mirror of notifications/index.ts flashFrame(). * @@ -11,7 +13,7 @@ import type {ElectronApplication} from 'playwright'; * exercise the same flashFrame() gate the notification `show` handler calls. */ export async function triggerNotificationEffects(app: ElectronApplication, flash = true): Promise { - await app.evaluate((_, shouldFlash: boolean) => { + await evaluateInMainProcessWithArg(app, (_electron, shouldFlash: boolean) => { const trigger = (global as any).__e2eNotificationEffects as ((value: boolean) => void) | undefined; if (!trigger) { throw new Error('__e2eNotificationEffects not exposed (NODE_ENV must be test)'); diff --git a/e2e/helpers/settingsWindow.ts b/e2e/helpers/settingsWindow.ts index 625e2e839ad..86eb27b289e 100644 --- a/e2e/helpers/settingsWindow.ts +++ b/e2e/helpers/settingsWindow.ts @@ -1,43 +1,28 @@ // Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. -import type {ElectronApplication, Page} from 'playwright'; +import type {ElectronApplication} from 'playwright'; +import {lookupModalByUrl, waitForModalView} from './modalPage'; +import type {ServerView} from './serverView'; import {SHOW_SETTINGS_WINDOW} from './ipcChannels'; import {evaluateInMainProcessWithArg} from './testRefs'; -export async function openSettingsWindow(electronApp: ElectronApplication): Promise { +const SETTINGS_URL_FRAGMENT = 'settings'; + +export async function openSettingsWindow(electronApp: ElectronApplication): Promise { for (let attempt = 0; attempt < 5; attempt++) { - const existingWindow = electronApp.windows().find((window) => window.url().includes('settings')); - if (existingWindow) { - try { - await existingWindow.waitForLoadState(); - return existingWindow; - } catch (error) { - if (attempt === 4) { - throw error; - } - await new Promise((resolve) => setTimeout(resolve, 250)); - continue; - } + const existingModal = await lookupModalByUrl(electronApp, {urlIncludes: SETTINGS_URL_FRAGMENT}); + if (existingModal) { + return waitForModalView(electronApp, {urlIncludes: SETTINGS_URL_FRAGMENT}); } - // Route through evaluateInMainProcessWithArg to reuse its transient - // "Execution context was destroyed" retry behavior instead of - // duplicating the try/catch loop here. await evaluateInMainProcessWithArg(electronApp, ({ipcMain}, showWindow) => { ipcMain.emit(showWindow); }, SHOW_SETTINGS_WINDOW); try { - const settingsWindow = electronApp.windows().find((window) => window.url().includes('settings')) ?? - await electronApp.waitForEvent('window', { - predicate: (window) => window.url().includes('settings'), - timeout: 3_000, - }); - - await settingsWindow.waitForLoadState(); - return settingsWindow; + return await waitForModalView(electronApp, {urlIncludes: SETTINGS_URL_FRAGMENT}); } catch (error) { if (attempt === 4) { throw error; @@ -46,5 +31,12 @@ export async function openSettingsWindow(electronApp: ElectronApplication): Prom } } - throw new Error('Settings window did not open'); + throw new Error('Settings modal did not open'); +} + +export async function waitForSettingsModal( + app: ElectronApplication, + options?: {timeout?: number}, +): Promise { + return waitForModalView(app, {urlIncludes: SETTINGS_URL_FRAGMENT, timeout: options?.timeout}); } diff --git a/e2e/helpers/trayMenu.ts b/e2e/helpers/trayMenu.ts index 4409bc930cf..7d28dd0b3bb 100644 --- a/e2e/helpers/trayMenu.ts +++ b/e2e/helpers/trayMenu.ts @@ -1,43 +1,21 @@ // Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. -import {expect} from '@playwright/test'; -import type {ElectronApplication, Page} from 'playwright'; +import type {ElectronApplication} from 'playwright'; import {clickTrayMenuItem} from './tray'; - -async function findSettingsPage(app: ElectronApplication): Promise { - return app.windows().find((window) => { - try { - return window.url().includes('settings'); - } catch { - return false; - } - }) ?? null; -} - -async function waitForSettingsPage(app: ElectronApplication): Promise { - let settingsWindow: Page | null = null; - await expect.poll(async () => { - settingsWindow = await findSettingsPage(app); - return settingsWindow; - }, {timeout: 15_000, message: 'Settings page must open after tray menu click'}).not.toBeNull(); - await settingsWindow!.waitForLoadState(); - return settingsWindow!; -} +import {waitForSettingsModal} from './settingsWindow'; export function traySettingsMenuLabel(): string { return process.platform === 'darwin' ? 'Preferences...' : 'Settings'; } -export async function openSettingsFromTray(app: ElectronApplication): Promise { - const existingSettings = await findSettingsPage(app); - if (existingSettings) { - await existingSettings.waitForLoadState(); - return existingSettings; +export async function openSettingsFromTray(app: ElectronApplication) { + const existingModal = await waitForSettingsModal(app, {timeout: 1_000}).catch(() => null); + if (existingModal) { + return existingModal; } - // Semantic click avoids i18n / mnemonic label mismatches ("Settings" vs "Settings..."). try { await clickTrayMenuItem(app, 'tray:settings'); } catch { @@ -62,7 +40,7 @@ export async function openSettingsFromTray(app: ElectronApplication): Promise { diff --git a/e2e/specs/notification_trigger/flash_taskbar.test.ts b/e2e/specs/notification_trigger/flash_taskbar.test.ts index 1e059500978..452968724f8 100644 --- a/e2e/specs/notification_trigger/flash_taskbar.test.ts +++ b/e2e/specs/notification_trigger/flash_taskbar.test.ts @@ -5,7 +5,7 @@ import {test, expect} from '../../fixtures/index'; import {waitForAppReady} from '../../helpers/appReadiness'; import {demoConfig} from '../../helpers/config'; import {acquireExclusiveLock} from '../../helpers/exclusiveLock'; -import {installFlashFrameSpy, restoreFlashFrameSpy} from '../../helpers/methodSpy'; +import {getFlashFrameCalls, installFlashFrameSpy, restoreFlashFrameSpy} from '../../helpers/methodSpy'; import {triggerNotificationEffects} from '../../helpers/notificationEffects'; test.describe('notification_trigger/flash_taskbar', () => { @@ -33,7 +33,7 @@ test.describe('notification_trigger/flash_taskbar', () => { await triggerNotificationEffects(electronApp, true); await expect.poll( - () => electronApp.evaluate(() => (global as any).__e2eFlashFrameCalls ?? []), + () => getFlashFrameCalls(electronApp), {timeout: 10_000, message: 'flashFrame(true) must be called when flashWindow is enabled'}, ).toContain(true); } finally { diff --git a/e2e/specs/notification_trigger/no_flash_taskbar.test.ts b/e2e/specs/notification_trigger/no_flash_taskbar.test.ts index 9878246797b..ee586633e17 100644 --- a/e2e/specs/notification_trigger/no_flash_taskbar.test.ts +++ b/e2e/specs/notification_trigger/no_flash_taskbar.test.ts @@ -5,7 +5,7 @@ import {test, expect} from '../../fixtures/index'; import {waitForAppReady} from '../../helpers/appReadiness'; import {demoConfig} from '../../helpers/config'; import {acquireExclusiveLock} from '../../helpers/exclusiveLock'; -import {installFlashFrameSpy, restoreFlashFrameSpy} from '../../helpers/methodSpy'; +import {getFlashFrameCalls, installFlashFrameSpy, restoreFlashFrameSpy} from '../../helpers/methodSpy'; import {triggerNotificationEffects} from '../../helpers/notificationEffects'; test.describe('notification_trigger/no_flash_taskbar', () => { @@ -44,7 +44,7 @@ test.describe('notification_trigger/no_flash_taskbar', () => { await triggerNotificationEffects(electronApp, true); await expect.poll( - () => electronApp.evaluate(() => (global as any).__e2eFlashFrameCalls ?? []), + () => getFlashFrameCalls(electronApp), {timeout: 10_000, message: 'flashFrame(true) must not be called when flashWindow is disabled'}, ).not.toContain(true); } finally { diff --git a/e2e/specs/settings/keyboard_shortcuts.test.ts b/e2e/specs/settings/keyboard_shortcuts.test.ts index a9c4a66db37..d757161a967 100644 --- a/e2e/specs/settings/keyboard_shortcuts.test.ts +++ b/e2e/specs/settings/keyboard_shortcuts.test.ts @@ -3,49 +3,14 @@ import {test, expect} from '../../fixtures/index'; import {cmdOrCtrl} from '../../helpers/config'; -import {SHOW_SETTINGS_WINDOW} from '../../helpers/ipcChannels'; - -type ElectronApplication = Awaited>; - -async function openSettingsWindow(electronApp: ElectronApplication) { - for (let attempt = 0; attempt < 5; attempt++) { - const existingWindow = electronApp.windows().find((window) => window.url().includes('settings')); - if (existingWindow) { - await existingWindow.waitForLoadState().catch(() => {}); - return existingWindow; - } - - try { - await electronApp.evaluate(({ipcMain}, showWindow) => { - ipcMain.emit(showWindow); - }, SHOW_SETTINGS_WINDOW); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - if (!message.includes('Execution context was destroyed') || attempt === 4) { - throw error; - } - } - - try { - const settingsWindow = await electronApp.waitForEvent('window', { - predicate: (window) => window.url().includes('settings'), - timeout: 3_000, - }); - await settingsWindow.waitForLoadState().catch(() => {}); - return settingsWindow; - } catch (error) { - if (attempt === 4) { - throw error; - } - await new Promise((resolve) => setTimeout(resolve, 250)); - } - } - - throw new Error('Settings window did not open'); -} +import {openSettingsWindow} from '../../helpers/settingsWindow'; test.describe('settings/keyboard_shortcuts', () => { test.describe('MM-T1288 Manipulating Text', () => { + test.beforeEach(async ({mainWindow}) => { + await mainWindow.bringToFront().catch(() => {}); + }); + test('MM-T1288_1 should be able to select and deselect language in the settings window', {tag: ['@P2', '@all']}, async ({electronApp}) => { const settingsWindow = await openSettingsWindow(electronApp); await settingsWindow.waitForSelector('#settingCategoryButton-language'); diff --git a/e2e/utils/tsio-report-status.js b/e2e/utils/tsio-report-status.js index 14fb709ef10..237be0bdf8a 100644 --- a/e2e/utils/tsio-report-status.js +++ b/e2e/utils/tsio-report-status.js @@ -31,10 +31,22 @@ function positiveInt(value, fallback) { function buildDisplayReportUrl(baseUrl, compositeIdentity) { const repoTrailing = (compositeIdentity.repository || '').split('/').pop() || compositeIdentity.repository; const repo = encodeURIComponent(repoTrailing); - const branch = encodeURIComponent(compositeIdentity.branch || 'main'); + const rawBranch = compositeIdentity.branch || 'main'; + let branchLabel = rawBranch.replace(/^refs\/heads\//, '').replace(/^refs\/tags\//, ''); + if (compositeIdentity.gh_pr_number != null) { + branchLabel = `pr-${compositeIdentity.gh_pr_number}`; + } + const branch = encodeURIComponent(branchLabel); const shortSha = (compositeIdentity.commit_sha || '').slice(0, 7); const name = encodeURIComponent(compositeIdentity.name); - return `${baseUrl}/reports/${repo}/${branch}/${shortSha}/${name}`; + let url = `${baseUrl}/reports/${repo}/${branch}/${shortSha}/${name}`; + if (compositeIdentity.gh_run_id) { + const params = new URLSearchParams(); + params.set('gh_run_id', String(compositeIdentity.gh_run_id)); + params.set('gh_run_attempt', String(compositeIdentity.gh_run_attempt || '1')); + url += `?${params}`; + } + return url; } /** diff --git a/src/app/mainWindow/mainWindow.ts b/src/app/mainWindow/mainWindow.ts index e314de3a7b3..7907e0fb394 100644 --- a/src/app/mainWindow/mainWindow.ts +++ b/src/app/mainWindow/mainWindow.ts @@ -41,6 +41,7 @@ import {DEFAULT_WINDOW_HEIGHT, DEFAULT_WINDOW_WIDTH} from 'common/utils/constant import * as Validator from 'common/Validator'; import ViewManager from 'common/views/viewManager'; import {boundsInfoPath} from 'main/constants'; +import {registerMainWindowE2EReadiness} from 'main/e2e/appReady'; import {localizeMessage} from 'main/i18nManager'; import performanceMonitor from 'main/performanceMonitor'; import ThemeManager from 'main/themeManager'; @@ -90,6 +91,8 @@ export class MainWindow extends EventEmitter { throw new Error('unable to create main window'); } + registerMainWindowE2EReadiness(this.win.browserWindow); + this.win.browserWindow.webContents.once('did-finish-load', () => { if (!this.win || this.win.browserWindow.isDestroyed()) { return; diff --git a/src/main/app/appReady.test.js b/src/main/app/appReady.test.js new file mode 100644 index 00000000000..b97b3f34af3 --- /dev/null +++ b/src/main/app/appReady.test.js @@ -0,0 +1,58 @@ +// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +import Config from 'common/config'; + +import {registerMainWindowE2EReadiness} from 'main/e2e/appReady'; + +jest.mock('common/config', () => ({ + hideOnStart: false, +})); + +describe('main/e2e/appReady', () => { + afterEach(() => { + delete global.__e2eAppReady; + Config.hideOnStart = false; + }); + + const makeWindow = (visible) => { + const listeners = {}; + return { + isDestroyed: () => false, + isVisible: () => visible, + once: jest.fn((event, cb) => { + listeners[event] = cb; + }), + webContents: { + getURL: () => 'mattermost-desktop://renderer/index.html', + once: jest.fn((event, cb) => { + listeners[`wc:${event}`] = cb; + }), + }, + fire: (event) => listeners[event]?.(), + fireWebContents: (event) => listeners[`wc:${event}`]?.(), + }; + }; + + it('should mark ready on show when hideOnStart is false', () => { + const win = makeWindow(false); + registerMainWindowE2EReadiness(win); + + expect(global.__e2eAppReady).toBeUndefined(); + expect(win.once).toHaveBeenCalledWith('show', expect.any(Function)); + expect(win.webContents.once).not.toHaveBeenCalled(); + + win.fire('show'); + expect(global.__e2eAppReady).toBe(true); + }); + + it('should mark ready on index did-finish-load when hideOnStart is true', () => { + Config.hideOnStart = true; + const win = makeWindow(false); + registerMainWindowE2EReadiness(win); + + expect(win.webContents.once).toHaveBeenCalledWith('did-finish-load', expect.any(Function)); + win.fireWebContents('did-finish-load'); + expect(global.__e2eAppReady).toBe(true); + }); +}); diff --git a/src/main/app/initialize.test.js b/src/main/app/initialize.test.js index 047dc06f2bb..7dad008945c 100644 --- a/src/main/app/initialize.test.js +++ b/src/main/app/initialize.test.js @@ -125,6 +125,9 @@ jest.mock('main/app/config', () => ({ handleConfigUpdate: jest.fn(), handleUpdateTheme: jest.fn(), })); +jest.mock('main/e2e/appReady', () => ({ + registerMainWindowE2EReadiness: jest.fn(), +})); jest.mock('main/app/intercom', () => ({ handleMainWindowIsShown: jest.fn(), })); diff --git a/src/main/app/intercom.test.js b/src/main/app/intercom.test.js index e793dd96b6d..78603b14157 100644 --- a/src/main/app/intercom.test.js +++ b/src/main/app/intercom.test.js @@ -79,35 +79,6 @@ describe('main/app/intercom', () => { }); describe('handleMainWindowIsShown', () => { - // Helper: build a BrowserWindow mock whose `once` records listeners so - // tests can fire them on demand. - const makeWindow = (initialVisible) => { - const listeners = {}; - let visible = initialVisible; - return { - listeners, - setVisible: (v) => { - visible = v; - }, - isVisible: jest.fn(() => visible), - once: jest.fn((event, cb) => { - listeners[event] = cb; - }), - removeListener: jest.fn((event, cb) => { - if (listeners[event] === cb) { - delete listeners[event]; - } - }), - fire: (event) => listeners[event] && listeners[event](), - }; - }; - - afterEach(() => { - delete global.__e2eAppReady; - jest.useRealTimers(); - jest.clearAllMocks(); - }); - it('MM-48079 should not show onboarding screen or server screen if GPO server is pre-configured', () => { getLocalPreload.mockReturnValue('/some/preload.js'); MainWindow.get.mockReturnValue({ @@ -119,57 +90,6 @@ describe('main/app/intercom', () => { handleMainWindowIsShown(); expect(ModalManager.addModal).not.toHaveBeenCalled(); }); - - it('should mark __e2eAppReady synchronously without attaching listeners when the main window is already visible', () => { - ServerManager.hasServers.mockReturnValue(true); - const win = makeWindow(true); - MainWindow.get.mockReturnValue(win); - - handleMainWindowIsShown(); - - expect(global.__e2eAppReady).toBe(true); - expect(win.once).not.toHaveBeenCalled(); - expect(MainWindow.once).not.toHaveBeenCalled(); - }); - - it('should set __e2eAppReady via a `show` listener (not `ready-to-show`) when the window is not yet visible', () => { - ServerManager.hasServers.mockReturnValue(true); - const win = makeWindow(false); - MainWindow.get.mockReturnValue(win); - - handleMainWindowIsShown(); - - // We listen to `show` only — never `ready-to-show`, which fires - // *before* the window is visible. - expect(win.once).toHaveBeenCalledWith('show', expect.any(Function)); - expect(win.once).not.toHaveBeenCalledWith('ready-to-show', expect.any(Function)); - expect(global.__e2eAppReady).toBeUndefined(); - - // The window becomes visible and the `show` event fires. - win.fire('show'); - expect(global.__e2eAppReady).toBe(true); - }); - - it('should defer to MAIN_WINDOW_CREATED if no main window exists yet, then mark ready when it appears', () => { - ServerManager.hasServers.mockReturnValue(true); - MainWindow.get.mockReturnValue(undefined); - - handleMainWindowIsShown(); - - // No window yet — should have registered a one-shot listener. - expect(MainWindow.once).toHaveBeenCalledWith('main-window-created', expect.any(Function)); - expect(global.__e2eAppReady).toBeUndefined(); - - // Window comes into existence (and happens to already be visible). - const win = makeWindow(true); - MainWindow.get.mockReturnValue(win); - - // Invoke the captured listener (simulating MainWindow emitting). - const createdCb = MainWindow.once.mock.calls[0][1]; - createdCb(); - - expect(global.__e2eAppReady).toBe(true); - }); }); describe('handleShowSettingsModal', () => { diff --git a/src/main/app/intercom.ts b/src/main/app/intercom.ts index 079fc03cbf3..73ac8359a9a 100644 --- a/src/main/app/intercom.ts +++ b/src/main/app/intercom.ts @@ -13,8 +13,6 @@ import {Logger} from 'common/log'; import ServerManager from 'common/servers/serverManager'; import {ping} from 'common/utils/requests'; import {parseURL} from 'common/utils/url'; -import {signalE2EAppReadyWhenShown} from 'main/e2e/appReady'; -import NotificationManager from 'main/notifications'; import {getLocalPreload} from 'main/utils'; import type {UniqueServer} from 'types/config'; @@ -87,8 +85,6 @@ export function handleMainWindowIsShown() { handleShowOnboardingScreens(showWelcomeScreen(), showNewServerModal(), false); }); } - - signalE2EAppReadyWhenShown(); } export function handleWelcomeScreenModal(prefillURL?: string) { diff --git a/src/main/e2e/appReady.ts b/src/main/e2e/appReady.ts index b0c2608570d..777aa6c7fce 100644 --- a/src/main/e2e/appReady.ts +++ b/src/main/e2e/appReady.ts @@ -3,38 +3,29 @@ import type {BrowserWindow} from 'electron'; -import MainWindow from 'app/mainWindow/mainWindow'; -import {MAIN_WINDOW_CREATED} from 'common/communication'; +import Config from 'common/config'; import {setTestField} from 'common/utils/util'; /** - * Signals `__e2eAppReady` once the main window is visible so Playwright can wait - * on app readiness. No-op outside NODE_ENV=test. + * Register listeners on the main BrowserWindow so `__e2eAppReady` is set at the + * first reliable lifecycle point. Must run from MainWindow.init() before + * index.html finishes loading — otherwise a fast `show` during later startup + * work can fire before any listener is attached. */ -export function signalE2EAppReadyWhenShown(): void { +export function registerMainWindowE2EReadiness(win: BrowserWindow): void { if (process.env.NODE_ENV !== 'test') { return; } const markReady = () => setTestField('__e2eAppReady', true); - const whenVisible = (win: BrowserWindow) => { - if (win.isVisible()) { - markReady(); - } else { - win.once('show', markReady); - } - }; - const win = MainWindow.get(); - if (win) { - whenVisible(win); - return; - } + win.once('show', markReady); - MainWindow.once(MAIN_WINDOW_CREATED, () => { - const created = MainWindow.get(); - if (created) { - whenVisible(created); - } - }); + if (Config.hideOnStart) { + win.webContents.once('did-finish-load', () => { + if (!win.isDestroyed() && win.webContents.getURL().includes('index')) { + markReady(); + } + }); + } } From 9bf9beb1543587bfee7d21e317e70400f6a2ffba Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Sat, 11 Jul 2026 15:59:55 +0530 Subject: [PATCH 29/37] test lint --- e2e/helpers/modalPage.ts | 63 ----------------------------------- e2e/helpers/settingsWindow.ts | 52 ++++++++++++++++++++++------- e2e/helpers/trayMenu.ts | 8 ++--- src/main/app/intercom.ts | 1 + 4 files changed, 45 insertions(+), 79 deletions(-) delete mode 100644 e2e/helpers/modalPage.ts diff --git a/e2e/helpers/modalPage.ts b/e2e/helpers/modalPage.ts deleted file mode 100644 index 90464b0aa64..00000000000 --- a/e2e/helpers/modalPage.ts +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. - -import {expect} from '@playwright/test'; -import type {ElectronApplication} from 'playwright'; - -import {ServerView} from './serverView'; -import {evaluateInMainProcessWithArg} from './testRefs'; - -export const SETTINGS_MODAL_KEY = 'settingsModal'; - -type ModalLookupOptions = { - urlIncludes: string; -}; - -type ModalLookupResult = { - webContentsId: number; - url: string; -}; - -export async function lookupModalByUrl( - app: ElectronApplication, - options: ModalLookupOptions, -): Promise { - return evaluateInMainProcessWithArg(app, ({webContents}, payload) => { - for (const wc of webContents.getAllWebContents()) { - if (wc.isDestroyed()) { - continue; - } - try { - const url = wc.getURL(); - if (url.includes(payload.urlIncludes)) { - return {webContentsId: wc.id, url}; - } - } catch { - // Ignore webContents that disappear while iterating. - } - } - return null; - }, options); -} - -export async function waitForModalView( - app: ElectronApplication, - options: ModalLookupOptions & {timeout?: number}, -): Promise { - const timeout = options.timeout ?? 15_000; - let modalView: ServerView | undefined; - - await expect.poll(async () => { - const modal = await lookupModalByUrl(app, {urlIncludes: options.urlIncludes}); - if (!modal) { - return null; - } - modalView = new ServerView(app, modal.webContentsId); - return modalView; - }, { - timeout, - message: `WebContents with URL containing "${options.urlIncludes}" must be available`, - }).not.toBeNull(); - - return modalView!; -} diff --git a/e2e/helpers/settingsWindow.ts b/e2e/helpers/settingsWindow.ts index 86eb27b289e..3be24cb4923 100644 --- a/e2e/helpers/settingsWindow.ts +++ b/e2e/helpers/settingsWindow.ts @@ -1,20 +1,48 @@ // Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. -import type {ElectronApplication} from 'playwright'; +import type {ElectronApplication, Page} from 'playwright'; -import {lookupModalByUrl, waitForModalView} from './modalPage'; -import type {ServerView} from './serverView'; import {SHOW_SETTINGS_WINDOW} from './ipcChannels'; import {evaluateInMainProcessWithArg} from './testRefs'; -const SETTINGS_URL_FRAGMENT = 'settings'; +function findSettingsPage(app: ElectronApplication): Page | undefined { + return app.windows().find((window) => { + try { + return window.url().includes('settings'); + } catch { + return false; + } + }); +} + +async function waitForSettingsPage(app: ElectronApplication, timeoutMs = 15_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const settingsWindow = findSettingsPage(app); + if (settingsWindow) { + await settingsWindow.waitForLoadState().catch(() => {}); + return settingsWindow; + } + await new Promise((resolve) => setTimeout(resolve, 200)); + } + throw new Error('Settings window did not open'); +} -export async function openSettingsWindow(electronApp: ElectronApplication): Promise { +export async function openSettingsWindow(electronApp: ElectronApplication): Promise { for (let attempt = 0; attempt < 5; attempt++) { - const existingModal = await lookupModalByUrl(electronApp, {urlIncludes: SETTINGS_URL_FRAGMENT}); - if (existingModal) { - return waitForModalView(electronApp, {urlIncludes: SETTINGS_URL_FRAGMENT}); + const existingWindow = findSettingsPage(electronApp); + if (existingWindow) { + try { + await existingWindow.waitForLoadState(); + return existingWindow; + } catch (error) { + if (attempt === 4) { + throw error; + } + await new Promise((resolve) => setTimeout(resolve, 250)); + continue; + } } await evaluateInMainProcessWithArg(electronApp, ({ipcMain}, showWindow) => { @@ -22,7 +50,7 @@ export async function openSettingsWindow(electronApp: ElectronApplication): Prom }, SHOW_SETTINGS_WINDOW); try { - return await waitForModalView(electronApp, {urlIncludes: SETTINGS_URL_FRAGMENT}); + return await waitForSettingsPage(electronApp); } catch (error) { if (attempt === 4) { throw error; @@ -31,12 +59,12 @@ export async function openSettingsWindow(electronApp: ElectronApplication): Prom } } - throw new Error('Settings modal did not open'); + throw new Error('Settings window did not open'); } export async function waitForSettingsModal( app: ElectronApplication, options?: {timeout?: number}, -): Promise { - return waitForModalView(app, {urlIncludes: SETTINGS_URL_FRAGMENT, timeout: options?.timeout}); +): Promise { + return waitForSettingsPage(app, options?.timeout ?? 15_000); } diff --git a/e2e/helpers/trayMenu.ts b/e2e/helpers/trayMenu.ts index 7d28dd0b3bb..371f9c5c753 100644 --- a/e2e/helpers/trayMenu.ts +++ b/e2e/helpers/trayMenu.ts @@ -3,17 +3,17 @@ import type {ElectronApplication} from 'playwright'; -import {clickTrayMenuItem} from './tray'; import {waitForSettingsModal} from './settingsWindow'; +import {clickTrayMenuItem} from './tray'; export function traySettingsMenuLabel(): string { return process.platform === 'darwin' ? 'Preferences...' : 'Settings'; } export async function openSettingsFromTray(app: ElectronApplication) { - const existingModal = await waitForSettingsModal(app, {timeout: 1_000}).catch(() => null); - if (existingModal) { - return existingModal; + const existingSettings = await waitForSettingsModal(app, {timeout: 1_000}).catch(() => null); + if (existingSettings) { + return existingSettings; } try { diff --git a/src/main/app/intercom.ts b/src/main/app/intercom.ts index 73ac8359a9a..b47aeb090ac 100644 --- a/src/main/app/intercom.ts +++ b/src/main/app/intercom.ts @@ -13,6 +13,7 @@ import {Logger} from 'common/log'; import ServerManager from 'common/servers/serverManager'; import {ping} from 'common/utils/requests'; import {parseURL} from 'common/utils/url'; +import NotificationManager from 'main/notifications'; import {getLocalPreload} from 'main/utils'; import type {UniqueServer} from 'types/config'; From 9f5b1ac860aff4420203af2aab0871862b5ecf20 Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Sat, 11 Jul 2026 17:05:50 +0530 Subject: [PATCH 30/37] fix test --- .../menu_bar/devtools_current_server.test.ts | 38 +++++-------------- 1 file changed, 10 insertions(+), 28 deletions(-) diff --git a/e2e/specs/menu_bar/devtools_current_server.test.ts b/e2e/specs/menu_bar/devtools_current_server.test.ts index fbbc219d931..0926ca329f7 100644 --- a/e2e/specs/menu_bar/devtools_current_server.test.ts +++ b/e2e/specs/menu_bar/devtools_current_server.test.ts @@ -60,38 +60,20 @@ test.describe('menu_bar/devtools_current_server', () => { {timeout: 15_000, message: 'DevTools must open for the current server webContents after menu click'}, ).toBe(true); - // MattermostWebContentsView.openDevTools() runs a 500ms macOS reset and documents - // that isDevToolsOpened() may not reflect close — use closeDevTools() and assert - // the server view is usable instead of polling isDevToolsOpened() on darwin. + // MattermostWebContentsView.openDevTools() uses detach mode. toggleDevTools() does not + // reliably close detached DevTools on Linux/Windows, and isDevToolsOpened() can lie on + // macOS — close explicitly and assert the server view stays usable. if (process.platform === 'darwin') { await new Promise((resolve) => setTimeout(resolve, 750)); - await evaluateInMainProcessWithArg(electronApp, ({webContents}, id) => { - const wc = webContents.fromId(id); - if (wc && !wc.isDestroyed() && wc.isDevToolsOpened()) { - wc.closeDevTools(); - } - }, webContentsId); - } else { - await evaluateInMainProcessWithArg(electronApp, ({webContents}, id) => { - try { - const wc = webContents.fromId(id); - if (wc && !wc.isDestroyed() && wc.isDevToolsOpened()) { - wc.toggleDevTools(); - } - } catch { - // DevTools may already be detaching. - } - }, webContentsId).catch(() => {}); - await expect.poll( - () => evaluateInMainProcessWithArg(electronApp, ({webContents}, id) => { - const wc = webContents.fromId(id); - return wc && !wc.isDestroyed() ? !wc.isDevToolsOpened() : true; - }, webContentsId).catch(() => true), - {timeout: 15_000, message: 'DevTools must close after toggle'}, - ).toBe(true); } + await evaluateInMainProcessWithArg(electronApp, ({webContents}, id) => { + const wc = webContents.fromId(id); + if (wc && !wc.isDestroyed() && wc.isDevToolsOpened()) { + wc.closeDevTools(); + } + }, webContentsId); - // DevTools attach/detach can briefly invalidate Playwright's Electron context on macOS. + // DevTools attach/detach can briefly invalidate Playwright's Electron context. await prepareMattermostServerView(electronApp, webContentsId); await firstServer!.waitForSelector('#post_textbox', {timeout: 15_000}); }, From c3ed3b865c4c755a4e12af542d896ab28c37c0d0 Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Sat, 11 Jul 2026 19:45:44 +0530 Subject: [PATCH 31/37] fix(e2e): stop Linux worker teardown hangs after all tests pass Playwright kept waiting up to 90s when app.close() never settled after SIGKILL on the fast-teardown path. Request app.exit() first on Linux, wait for close() to settle after force-kill, pkill stray Electron PIDs at worker/global cleanup, use one CI worker on Linux, and register focus suite PIDs for reaping. Co-authored-by: Cursor --- e2e/fixtures/index.ts | 2 +- e2e/helpers/electronApp.ts | 95 +++++++++++++++++++++++++++++++++----- e2e/playwright.config.ts | 6 ++- e2e/specs/focus.test.ts | 3 +- 4 files changed, 92 insertions(+), 14 deletions(-) diff --git a/e2e/fixtures/index.ts b/e2e/fixtures/index.ts index 9c936c01aa5..d04ab8d0ab5 100644 --- a/e2e/fixtures/index.ts +++ b/e2e/fixtures/index.ts @@ -66,7 +66,7 @@ export const test = base.extend({ await Promise.race([ cleanupRegisteredElectronProcesses(), new Promise((resolve) => { - timeoutHandle = setTimeout(resolve, 20_000); + timeoutHandle = setTimeout(resolve, 45_000); timeoutHandle.unref?.(); }), ]); diff --git a/e2e/helpers/electronApp.ts b/e2e/helpers/electronApp.ts index 0fdb6f1d04a..05398775943 100644 --- a/e2e/helpers/electronApp.ts +++ b/e2e/helpers/electronApp.ts @@ -146,6 +146,7 @@ export async function cleanupRegisteredElectronProcesses(): Promise { const pids = readPidsFromFile(file); fs.rmSync(file, {force: true}); await reapPids(pids); + await forceKillStrayElectronProcesses(); } /** @@ -160,6 +161,7 @@ export async function cleanupAllRegisteredElectronProcesses(): Promise { fs.rmSync(file, {force: true}); } await reapPids(pids); + await forceKillStrayElectronProcesses(); } /** @@ -275,6 +277,59 @@ async function drainPlaywrightClose(closePromise: Promise, remainingMs: nu await Promise.race([closePromise, sleep(remainingMs)]); } +async function requestElectronQuit(app: ElectronApplication): Promise { + try { + await Promise.race([ + app.evaluate(() => { + const {app: electronApp} = require('electron'); + electronApp.exit(0); + }), + sleep(2_000), + ]); + } catch { + // evaluation may fail if the app is already shutting down + } +} + +async function settleElectronClose(closePromise: Promise, pid: number | undefined, budgetMs: number): Promise { + const deadline = Date.now() + budgetMs; + while (Date.now() < deadline) { + await drainPlaywrightClose(closePromise, Math.min(1_000, deadline - Date.now())); + if (pid && !isProcessAlive(pid)) { + // Give Playwright a moment to observe the dead process and reject close(). + await drainPlaywrightClose(closePromise, Math.min(5_000, deadline - Date.now())); + return; + } + if (await isClosePromiseSettled(closePromise)) { + return; + } + await sleep(200); + } +} + +async function isClosePromiseSettled(closePromise: Promise): Promise { + const result = await Promise.race([ + closePromise.then(() => 'settled' as const, () => 'settled' as const), + sleep(50).then(() => 'pending' as const), + ]); + return result === 'settled'; +} + +/** + * Linux CI can leave Mattermost Electron processes running when app.close() never + * settles. Reap by executable path as a worker/global backstop after registry cleanup. + */ +export async function forceKillStrayElectronProcesses(): Promise { + if (process.platform !== 'linux') { + return; + } + try { + execFileSync('pkill', ['-KILL', '-f', 'node_modules/electron/dist/electron'], {stdio: 'ignore'}); + } catch { + // no matching processes + } +} + async function forceShutdownLinux(pid: number): Promise { if (!isProcessAlive(pid)) { return; @@ -306,7 +361,12 @@ function signalShutdownAndReturn(pid: number): void { } } -async function attemptClose(app: ElectronApplication, timeoutMs: number): Promise { +type AttemptCloseResult = { + closed: boolean; + closePromise: Promise; +}; + +async function attemptClose(app: ElectronApplication, timeoutMs: number): Promise { let closed = false; // Only mark closed on successful resolve; a rejected close() must leave @@ -316,8 +376,10 @@ async function attemptClose(app: ElectronApplication, timeoutMs: number): Promis }, () => {}); const start = Date.now(); await Promise.race([closePromise, sleep(timeoutMs)]); - await drainPlaywrightClose(closePromise, timeoutMs - (Date.now() - start)); - return closed; + const remainingMs = Math.max(0, timeoutMs - (Date.now() - start)); + // Playwright keeps a gracefullyClose entry until close() settles (#29431). + await drainPlaywrightClose(closePromise, Math.max(remainingMs, 2_000)); + return {closed, closePromise}; } export async function waitForWindow(app: ElectronApplication, pattern: string, timeout = 30_000) { @@ -360,17 +422,28 @@ export async function closeElectronApp( // same userDataDir may be relaunched, so we force-kill on failure (Linux) // and always wait for the SingletonLock to release. const fastTeardown = Boolean(options.skipLockWaitUnlessCleanClose); + const closeTimeoutMs = process.platform === 'linux' && fastTeardown ? 5_000 : 10_000; - const cleanClosed = await attemptClose(app, 10_000); + if (process.platform === 'linux') { + await requestElectronQuit(app); + } - if (!cleanClosed && pid) { - if (process.platform === 'linux') { - // Always SIGKILL stuck trees on Linux so worker teardown does not sit - // in Playwright's 90s gracefullyClose wait with live Electron PIDs. - await forceShutdownLinux(pid); - } else { - signalShutdownAndReturn(pid); + const {closed: cleanClosed, closePromise} = await attemptClose(app, closeTimeoutMs); + + if (!cleanClosed) { + if (pid) { + if (process.platform === 'linux') { + // Always SIGKILL stuck trees on Linux so worker teardown does not sit + // in Playwright's 90s gracefullyClose wait with live Electron PIDs. + await forceShutdownLinux(pid); + } else { + signalShutdownAndReturn(pid); + } } + + // Killing the process does not settle app.close(); wait until Playwright + // drops the gracefullyClose entry before worker teardown. + await settleElectronClose(closePromise, pid, 20_000); } // Fast path on a failed close: return immediately (master-style). The lock diff --git a/e2e/playwright.config.ts b/e2e/playwright.config.ts index e862a74a75a..a633f00e8db 100644 --- a/e2e/playwright.config.ts +++ b/e2e/playwright.config.ts @@ -28,7 +28,11 @@ const PLATFORM_GREP: Record = { // Electron processes are heavy (~300MB each), so cap at 2 in CI and half the CPU // count locally (max 4). Override with E2E_WORKERS env var. const cpuCount = os.cpus().length; -const defaultWorkers = process.env.CI ? 2 : Math.min(4, Math.max(1, Math.floor(cpuCount / 2))); +// Linux CI hits Playwright worker-teardown hangs with 2 parallel Electron workers; +// one worker can finish with a stuck app.close() and burn the full 90s budget. +const defaultWorkers = process.env.CI ? + (getActivePlatform() === 'linux' ? 1 : 2) : + Math.min(4, Math.max(1, Math.floor(cpuCount / 2))); const parsedWorkers = process.env.E2E_WORKERS ? Number.parseInt(process.env.E2E_WORKERS, 10) : NaN; const workers = Number.isFinite(parsedWorkers) && parsedWorkers > 0 ? parsedWorkers : defaultWorkers; diff --git a/e2e/specs/focus.test.ts b/e2e/specs/focus.test.ts index d45f3ea3c4d..6e25ac64455 100644 --- a/e2e/specs/focus.test.ts +++ b/e2e/specs/focus.test.ts @@ -8,7 +8,7 @@ import * as path from 'path'; import {test, expect} from '../fixtures/index'; import {waitForAppReady} from '../helpers/appReadiness'; import {electronBinaryPath, appDir, demoMattermostConfig, writeConfigFile} from '../helpers/config'; -import {closeElectronAppFast} from '../helpers/electronApp'; +import {closeElectronAppFast, registerElectronMainProcess} from '../helpers/electronApp'; import {SHOW_NEW_SERVER_MODAL, SHOW_SETTINGS_WINDOW} from '../helpers/ipcChannels'; import {loginToMattermost} from '../helpers/login'; import {buildServerMap, type ServerMap} from '../helpers/serverMap'; @@ -129,6 +129,7 @@ test.describe('focus', () => { env: {...process.env, NODE_ENV: 'test'}, timeout: 60_000, }); + registerElectronMainProcess(electronApp.process()?.pid); await waitForAppReady(electronApp); // Poll until both servers are registered in WebContentsManager From cff190ac7f9af38065de1f1ec4b0e46d87b09e2e Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Sat, 11 Jul 2026 21:09:18 +0530 Subject: [PATCH 32/37] test lint --- .github/workflows/e2e-functional-template.yml | 13 +++++++---- e2e/helpers/electronApp.ts | 23 ++++++++++++------- e2e/playwright.config.ts | 11 ++++++--- 3 files changed, 31 insertions(+), 16 deletions(-) diff --git a/.github/workflows/e2e-functional-template.yml b/.github/workflows/e2e-functional-template.yml index 8fd878698c3..e9dc11d34c6 100644 --- a/.github/workflows/e2e-functional-template.yml +++ b/.github/workflows/e2e-functional-template.yml @@ -301,12 +301,15 @@ jobs: - name: e2e/handle-nonzero-playwright-exit if: always() run: | - if [ -z "${PLAYWRIGHT_EXIT_CODE:-}" ]; then + if [ "${PLAYWRIGHT_EXIT_CODE:-1}" = "0" ]; then exit 0 fi - if [ "${PLAYWRIGHT_EXIT_CODE}" = "0" ]; then - exit 0 + if [ -n "${TSIO_CONFIG}" ]; then + echo "::warning::Playwright exited with code ${PLAYWRIGHT_EXIT_CODE:-unset} — not failing the job on this alone, see TSIO report for actual test results" + else + echo "Playwright exited with code ${PLAYWRIGHT_EXIT_CODE:-unset} and no TSIO config is set for this run — failing the job since there is no other failure signal" >&2 + exit 1 fi - echo "Playwright exited with code ${PLAYWRIGHT_EXIT_CODE}" >&2 - exit 1 + env: + TSIO_CONFIG: ${{ inputs.tsio-config }} diff --git a/e2e/helpers/electronApp.ts b/e2e/helpers/electronApp.ts index 05398775943..604809d75e1 100644 --- a/e2e/helpers/electronApp.ts +++ b/e2e/helpers/electronApp.ts @@ -294,15 +294,17 @@ async function requestElectronQuit(app: ElectronApplication): Promise { async function settleElectronClose(closePromise: Promise, pid: number | undefined, budgetMs: number): Promise { const deadline = Date.now() + budgetMs; while (Date.now() < deadline) { - await drainPlaywrightClose(closePromise, Math.min(1_000, deadline - Date.now())); - if (pid && !isProcessAlive(pid)) { - // Give Playwright a moment to observe the dead process and reject close(). - await drainPlaywrightClose(closePromise, Math.min(5_000, deadline - Date.now())); - return; - } if (await isClosePromiseSettled(closePromise)) { return; } + + // Keep draining until Playwright observes a dead process and rejects close(). + // Returning early when the PID is gone but close() is still pending leaves a + // gracefullyClose entry that burns the full workerTeardownTimeout (#29431). + await drainPlaywrightClose(closePromise, Math.min(2_000, deadline - Date.now())); + if (pid && !isProcessAlive(pid)) { + await forceKillStrayElectronProcesses(); + } await sleep(200); } } @@ -377,6 +379,7 @@ async function attemptClose(app: ElectronApplication, timeoutMs: number): Promis const start = Date.now(); await Promise.race([closePromise, sleep(timeoutMs)]); const remainingMs = Math.max(0, timeoutMs - (Date.now() - start)); + // Playwright keeps a gracefullyClose entry until close() settles (#29431). await drainPlaywrightClose(closePromise, Math.max(remainingMs, 2_000)); return {closed, closePromise}; @@ -443,12 +446,16 @@ export async function closeElectronApp( // Killing the process does not settle app.close(); wait until Playwright // drops the gracefullyClose entry before worker teardown. - await settleElectronClose(closePromise, pid, 20_000); + const settleBudgetMs = process.platform === 'linux' ? 45_000 : 20_000; + await settleElectronClose(closePromise, pid, settleBudgetMs); } - // Fast path on a failed close: return immediately (master-style). The lock + // Fast path on a failed close: return once close() has settled. The lock // lives in an abandoned dir and worker/global cleanup reaps any live PID. if (fastTeardown && !cleanClosed) { + if (!(await isClosePromiseSettled(closePromise))) { + await settleElectronClose(closePromise, pid, process.platform === 'linux' ? 30_000 : 15_000); + } if (pid && !isProcessAlive(pid)) { unregisterElectronMainProcess(pid); } diff --git a/e2e/playwright.config.ts b/e2e/playwright.config.ts index a633f00e8db..9f814721cb9 100644 --- a/e2e/playwright.config.ts +++ b/e2e/playwright.config.ts @@ -28,11 +28,16 @@ const PLATFORM_GREP: Record = { // Electron processes are heavy (~300MB each), so cap at 2 in CI and half the CPU // count locally (max 4). Override with E2E_WORKERS env var. const cpuCount = os.cpus().length; + // Linux CI hits Playwright worker-teardown hangs with 2 parallel Electron workers; // one worker can finish with a stuck app.close() and burn the full 90s budget. -const defaultWorkers = process.env.CI ? - (getActivePlatform() === 'linux' ? 1 : 2) : - Math.min(4, Math.max(1, Math.floor(cpuCount / 2))); +function getDefaultWorkers(): number { + if (process.env.CI) { + return getActivePlatform() === 'linux' ? 1 : 2; + } + return Math.min(4, Math.max(1, Math.floor(cpuCount / 2))); +} +const defaultWorkers = getDefaultWorkers(); const parsedWorkers = process.env.E2E_WORKERS ? Number.parseInt(process.env.E2E_WORKERS, 10) : NaN; const workers = Number.isFinite(parsedWorkers) && parsedWorkers > 0 ? parsedWorkers : defaultWorkers; From cbe461edd21c919ff50445673b2e4b3988af8ab3 Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Sun, 12 Jul 2026 00:07:34 +0530 Subject: [PATCH 33/37] fix(e2e): use branch-based TSIO display report URL Build commit-status and rollup links as /reports/desktop/{branch}/{shortSha}/desktop-pr instead of pr-{number} paths with gh_run_id query params. Co-authored-by: Cursor --- e2e/utils/tsio-report-status.js | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/e2e/utils/tsio-report-status.js b/e2e/utils/tsio-report-status.js index 237be0bdf8a..eaeba16bb9b 100644 --- a/e2e/utils/tsio-report-status.js +++ b/e2e/utils/tsio-report-status.js @@ -24,29 +24,18 @@ function positiveInt(value, fallback) { } /** - * Commit-level rollup URL (repo / branch / short SHA / name). Dedupes the same - * spec across platform shards — headline stats can disagree with GitHub when a - * failure is platform-specific. Job summary only; not for commit-status links. + * Commit-level rollup URL: /reports/{repo}/{branch}/{shortSha}/{name} + * e.g. https://test-io.test.mattermost.com/reports/desktop/tsio-spike/cff190a/desktop-pr */ function buildDisplayReportUrl(baseUrl, compositeIdentity) { const repoTrailing = (compositeIdentity.repository || '').split('/').pop() || compositeIdentity.repository; const repo = encodeURIComponent(repoTrailing); - const rawBranch = compositeIdentity.branch || 'main'; - let branchLabel = rawBranch.replace(/^refs\/heads\//, '').replace(/^refs\/tags\//, ''); - if (compositeIdentity.gh_pr_number != null) { - branchLabel = `pr-${compositeIdentity.gh_pr_number}`; - } - const branch = encodeURIComponent(branchLabel); + const branch = encodeURIComponent( + (compositeIdentity.branch || 'main').replace(/^refs\/heads\//, '').replace(/^refs\/tags\//, ''), + ); const shortSha = (compositeIdentity.commit_sha || '').slice(0, 7); const name = encodeURIComponent(compositeIdentity.name); - let url = `${baseUrl}/reports/${repo}/${branch}/${shortSha}/${name}`; - if (compositeIdentity.gh_run_id) { - const params = new URLSearchParams(); - params.set('gh_run_id', String(compositeIdentity.gh_run_id)); - params.set('gh_run_attempt', String(compositeIdentity.gh_run_attempt || '1')); - url += `?${params}`; - } - return url; + return `${baseUrl}/reports/${repo}/${branch}/${shortSha}/${name}`; } /** @@ -194,7 +183,7 @@ async function reportTsioStatus({ let targetUrl = runUrl; if (isComplete || isIncomplete) { - targetUrl = groupReportUrl || displayReportUrl; + targetUrl = displayReportUrl || groupReportUrl; } const summaryLines = [ @@ -259,7 +248,7 @@ async function reportTsioStatus({ throw new Error(`TSIO report ${reportId} did not pass: ${reason}`); } - return {reportUrl: groupReportUrl || displayReportUrl, status: detail.status, stats}; + return {reportUrl: displayReportUrl || groupReportUrl, status: detail.status, stats}; } module.exports = reportTsioStatus; From a268b7a5f2f59d71966f28731605c8e46f6d9145 Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Sun, 12 Jul 2026 03:15:12 +0530 Subject: [PATCH 34/37] add new tests --- e2e/specs/menu_bar/help_menu.test.ts | 38 +++++++++++++++ .../system_tray_icon/hide_to_tray.test.ts | 46 +++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 e2e/specs/system_tray_icon/hide_to_tray.test.ts diff --git a/e2e/specs/menu_bar/help_menu.test.ts b/e2e/specs/menu_bar/help_menu.test.ts index cd150fd4e5b..4422f618221 100644 --- a/e2e/specs/menu_bar/help_menu.test.ts +++ b/e2e/specs/menu_bar/help_menu.test.ts @@ -5,6 +5,7 @@ import {test, expect} from '../../fixtures/index'; import {waitForAppReady} from '../../helpers/appReadiness'; import {clickApplicationMenuItem} from '../../helpers/menu'; import {evaluateInMainProcess} from '../../helpers/testRefs'; +import {getShellOpenExternalCalls, restoreShellOpenExternal, stubShellOpenExternal} from '../../helpers/shell'; test.describe('menu_bar/help_menu', () => { test( @@ -118,4 +119,41 @@ test.describe('menu_bar/help_menu', () => { } }, ); + + test( + 'MM-T828 Learn More in the Menu Bar opens help documentation externally', + {tag: ['@P2', '@darwin']}, + async ({electronApp}) => { + await waitForAppReady(electronApp); + + const helpMenuItem = await evaluateInMainProcess(electronApp, ({app: electronAppInstance}) => { + const refs = (global as any).__e2eTestRefs; + const menu = electronAppInstance.applicationMenu?.getMenuItemById('help'); + const userGuideItem = menu?.submenu?.items?.find((item) => { + return typeof item.label === 'string' && + item.label.includes('User guide') && + typeof item.click === 'function'; + }); + return { + label: typeof userGuideItem?.label === 'string' ? userGuideItem.label : '', + helpLink: refs?.Config?.helpLink ?? '', + }; + }); + + expect(helpMenuItem.label, 'Help menu must expose a User guide item').toContain('User guide'); + expect(helpMenuItem.helpLink, 'Config.helpLink must be set').toContain('docs.mattermost.com'); + + await stubShellOpenExternal(electronApp); + try { + await clickApplicationMenuItem(electronApp, 'help', {label: helpMenuItem.label}); + + await expect.poll( + () => getShellOpenExternalCalls(electronApp), + {timeout: 10_000, message: 'Help > User guide must open documentation via shell.openExternal'}, + ).toContain(helpMenuItem.helpLink); + } finally { + await restoreShellOpenExternal(electronApp); + } + }, + ); }); diff --git a/e2e/specs/system_tray_icon/hide_to_tray.test.ts b/e2e/specs/system_tray_icon/hide_to_tray.test.ts new file mode 100644 index 00000000000..75a1824c889 --- /dev/null +++ b/e2e/specs/system_tray_icon/hide_to_tray.test.ts @@ -0,0 +1,46 @@ +// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +import {test, expect} from '../../fixtures/index'; +import {demoConfig, type AppConfig} from '../../helpers/config'; +import {clickTrayMenuItem, emitTrayIconClick, hideMainWindow, isMainWindowVisible} from '../../helpers/tray'; + +const trayConfig: AppConfig = { + ...demoConfig, + showTrayIcon: true, + minimizeToTray: true, +}; + +test.describe('system_tray_icon/hide_to_tray', () => { + test.use({appConfig: trayConfig}); + + test( + 'MM-T6194 main window can be hidden to tray and restored', + {tag: ['@P1', '@all']}, + async ({electronApp}) => { + await expect.poll( + () => isMainWindowVisible(electronApp), + {timeout: 10_000, message: 'Main window should be visible after launch'}, + ).toBe(true); + + await hideMainWindow(electronApp); + + await expect.poll( + () => isMainWindowVisible(electronApp), + {timeout: 10_000, message: 'Main window should be hidden after hide()'}, + ).toBe(false); + + await emitTrayIconClick(electronApp); + + // macOS tray click opens the context menu; choosing a server raises the window. + if (process.platform === 'darwin') { + await clickTrayMenuItem(electronApp, trayConfig.servers[0].name); + } + + await expect.poll( + () => isMainWindowVisible(electronApp), + {timeout: 10_000, message: 'Main window should be visible after tray restore'}, + ).toBe(true); + }, + ); +}); From d6fdb7452634053fd52fba3f90262d24a160c5fb Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Tue, 14 Jul 2026 00:09:41 +0530 Subject: [PATCH 35/37] move unit test for src/main/e2e/appReady.ts --- package.json | 2 +- src/main/{app => e2e}/appReady.test.js | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/main/{app => e2e}/appReady.test.js (100%) diff --git a/package.json b/package.json index ae1da3c57d4..566e8bc85f9 100644 --- a/package.json +++ b/package.json @@ -81,7 +81,7 @@ ], "testPathIgnorePatterns": [ "/node_modules/", - "/e2e/", + "/e2e/", "/webpack.config.test.js" ], "globals": { diff --git a/src/main/app/appReady.test.js b/src/main/e2e/appReady.test.js similarity index 100% rename from src/main/app/appReady.test.js rename to src/main/e2e/appReady.test.js From 35b20a157ccd639a1f721d1790d31d44f44cceba Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Tue, 14 Jul 2026 12:46:38 +0530 Subject: [PATCH 36/37] update linux runner version --- .github/workflows/cmt-provisioner.yml | 2 +- .../compatibility-matrix-testing.yml | 31 ++++++++++++++----- .github/workflows/e2e-functional.yml | 4 +-- .github/workflows/e2e-label-cleanup.yml | 2 +- .github/workflows/e2e-pr-trigger.yml | 2 +- 5 files changed, 29 insertions(+), 12 deletions(-) diff --git a/.github/workflows/cmt-provisioner.yml b/.github/workflows/cmt-provisioner.yml index 8b3e86d3283..5e974791f26 100644 --- a/.github/workflows/cmt-provisioner.yml +++ b/.github/workflows/cmt-provisioner.yml @@ -25,7 +25,7 @@ permissions: {} jobs: trigger-matterwick: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Signal Matterwick to provision CMT servers run: | diff --git a/.github/workflows/compatibility-matrix-testing.yml b/.github/workflows/compatibility-matrix-testing.yml index b548a9b5479..a11589d4a24 100644 --- a/.github/workflows/compatibility-matrix-testing.yml +++ b/.github/workflows/compatibility-matrix-testing.yml @@ -26,7 +26,7 @@ jobs: # every (OS × server-version) matrix leg. ## This is picked up after the finish for cleanup upload-cmt-server-detals: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: cmt/generate-instance-details-file run: echo '${{ inputs.CMT_MATRIX }}' > instance-details.json @@ -39,9 +39,10 @@ jobs: retention-days: 1 calculate-commit-hash: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 outputs: DESKTOP_SHA: ${{ steps.repo.outputs.DESKTOP_SHA }} + CMT_MATRIX: ${{ steps.normalize-matrix.outputs.matrix }} TSIO_COMPOSITE_IDENTITY: ${{ steps.tsio-identity.outputs.composite-identity-json }} TSIO_TOTAL_REPORTS_EXPECTED: ${{ steps.tsio-identity.outputs.total-reports-expected }} steps: @@ -54,6 +55,21 @@ jobs: id: repo run: echo "DESKTOP_SHA=$(git rev-parse HEAD)" >> ${GITHUB_OUTPUT} + # Matterwick still dispatches ubuntu-22.04 for Linux; pin to ubuntu-latest so + # CMT matches PR E2E (24.04) and install-os-dependencies can resolve + # libasound2t64. Same idea as e2e-functional.yml remapping macos-latest → macos-26. + - name: cmt/normalize-matrix-runners + id: normalize-matrix + env: + CMT_MATRIX: ${{ inputs.CMT_MATRIX }} + run: | + NORMALIZED=$(echo "${CMT_MATRIX}" | jq -c ' + .environment |= map( + if .runner == "ubuntu-22.04" then .runner = "ubuntu-latest" else . end + ) + ') + echo "matrix=${NORMALIZED}" >> "$GITHUB_OUTPUT" + # One composite identity + total-reports-expected for the WHOLE CMT run # (every OS x server-version leg), computed once here and threaded into # every `e2e` matrix leg below so they land in a single TSIO report group. @@ -63,7 +79,7 @@ jobs: GITHUB_REPOSITORY: ${{ github.repository }} MM_SHA: ${{ steps.repo.outputs.DESKTOP_SHA }} MM_BRANCH: ${{ inputs.DESKTOP_VERSION }} - CMT_MATRIX: ${{ inputs.CMT_MATRIX }} + CMT_MATRIX: ${{ steps.normalize-matrix.outputs.matrix }} run: | TOTAL=$(echo "${CMT_MATRIX}" | jq -c '(.environment | length) * (.server | length)') echo "total-reports-expected=${TOTAL}" >> "$GITHUB_OUTPUT" @@ -78,7 +94,7 @@ jobs: echo "composite-identity-json=${COMPOSITE_IDENTITY}" >> "$GITHUB_OUTPUT" update-initial-status: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: - calculate-commit-hash permissions: @@ -98,7 +114,8 @@ jobs: description: "Compatibility Matrix Testing for ${{ inputs.DESKTOP_VERSION }} version" status: pending - # Input follows the below schema + # Input follows the below schema (Matterwick may still send ubuntu-22.04; + # calculate-commit-hash remaps Linux to ubuntu-latest before the e2e matrix). # { # "environment": [ # { @@ -133,7 +150,7 @@ jobs: - update-initial-status strategy: fail-fast: false - matrix: ${{ fromJson(inputs.CMT_MATRIX) }} + matrix: ${{ fromJson(needs.calculate-commit-hash.outputs.CMT_MATRIX) }} permissions: contents: read actions: read @@ -151,7 +168,7 @@ jobs: tsio-config: ${{ format('{{"composite_identity":{0},"total_reports_expected":"{1}"}}', needs.calculate-commit-hash.outputs.TSIO_COMPOSITE_IDENTITY, needs.calculate-commit-hash.outputs.TSIO_TOTAL_REPORTS_EXPECTED) }} update-final-status: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 if: always() permissions: contents: read diff --git a/.github/workflows/e2e-functional.yml b/.github/workflows/e2e-functional.yml index 42c3a124ed6..41f71d1ad31 100644 --- a/.github/workflows/e2e-functional.yml +++ b/.github/workflows/e2e-functional.yml @@ -99,7 +99,7 @@ jobs: update-initial-status: name: Set pending TSIO status - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: - prepare-matrix permissions: @@ -197,7 +197,7 @@ jobs: remove-e2e-label: name: Remove E2E label from PR - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 permissions: issues: write pull-requests: write diff --git a/.github/workflows/e2e-label-cleanup.yml b/.github/workflows/e2e-label-cleanup.yml index c0d71cc143a..12235b4c142 100644 --- a/.github/workflows/e2e-label-cleanup.yml +++ b/.github/workflows/e2e-label-cleanup.yml @@ -19,7 +19,7 @@ permissions: jobs: remove-e2e-label: name: Remove E2E/Run label from PR - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 if: ${{ github.event.workflow_run.event == 'workflow_dispatch' }} steps: - name: Checkout code diff --git a/.github/workflows/e2e-pr-trigger.yml b/.github/workflows/e2e-pr-trigger.yml index a5abbe1d369..ba0134c921a 100644 --- a/.github/workflows/e2e-pr-trigger.yml +++ b/.github/workflows/e2e-pr-trigger.yml @@ -26,7 +26,7 @@ concurrency: jobs: e2e-label-orchestration: name: E2E label - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 permissions: issues: write pull-requests: write From 8c1fe48445ca9aaa466b19af047b0a10ba42d7bd Mon Sep 17 00:00:00 2001 From: yasserfaraazkhan Date: Tue, 14 Jul 2026 13:32:12 +0530 Subject: [PATCH 37/37] fix tests --- e2e/helpers/channelReadiness.ts | 6 ++++-- e2e/helpers/login.ts | 9 +++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/e2e/helpers/channelReadiness.ts b/e2e/helpers/channelReadiness.ts index 847e5131b9b..7cc32e9a1ba 100644 --- a/e2e/helpers/channelReadiness.ts +++ b/e2e/helpers/channelReadiness.ts @@ -12,7 +12,7 @@ import { IS_CHANNEL_VIEW_LOADED_JS, IS_COMPOSER_INTERACTIVE_JS, } from './rendererUtils'; -import {activateServerView, loadServerViewUrl} from './serverContext'; +import {activateServerView, loadServerViewUrl, reloadServerView} from './serverContext'; import {resolveChannelByName} from './server_api/channel'; import type {ServerEntry} from './serverMap'; import type {ServerView} from './serverView'; @@ -139,7 +139,9 @@ export async function recoverServerViewIfNeeded( return; } - await win.runInRenderer('window.location.reload(); return true;', true); + // Prefer the Electron view reload path — window.location.reload() can leave + // the WebContentsView blank on macOS/Windows CI. + await reloadServerView(win.app, win.webContentsId); await waitForMattermostShell(win, {channelItem}); } diff --git a/e2e/helpers/login.ts b/e2e/helpers/login.ts index 5ce050e66c7..bfa8426b65b 100644 --- a/e2e/helpers/login.ts +++ b/e2e/helpers/login.ts @@ -5,6 +5,7 @@ import {expect} from '@playwright/test'; import {isTransientEvaluateError} from './testRefs'; import {CHANNEL_HEADER_SELECTORS, POST_TEXTBOX_SELECTOR} from './rendererUtils'; +import {reloadServerView} from './serverContext'; import type {ServerView} from './serverView'; async function isMattermostServerUrl(win: ServerView): Promise { @@ -64,7 +65,11 @@ export async function loginToMattermost(win: ServerView): Promise { const loginSelector = '#input_loginId'; const passwordSelector = '#input_password-input, input[type="password"]'; const submitSelector = '#saveSetting, button[type="submit"]'; + const pollStart = Date.now(); + const reloadAt = pollStart + Math.min(timeout / 2, 15_000); + let reloaded = false; + // Cold cloud hosts often paint a blank hex shell; one mid-wait reload unsticks it. await expect.poll(async () => { if (await hasAppShell(win)) { return 'logged-in'; @@ -72,6 +77,10 @@ export async function loginToMattermost(win: ServerView): Promise { if (await hasLoginForm(win)) { return 'login-form'; } + if (!reloaded && Date.now() >= reloadAt) { + reloaded = true; + await reloadServerView(win.app, win.webContentsId).catch(() => undefined); + } return 'loading'; }, { timeout,