ci(e2e): CMT scheduling + macOS E2E stabilization + CI/security hardening

.github/workflows/cmt-provisioner.yml

@@ -1,29 +1,44 @@
 name: CMT Provisioner
 
-# This is a stub workflow. Its only purpose is to fire a workflow_run webhook
-# to Matterwick, which provisions cloud servers for each server version and
-# then dispatches compatibility-matrix-testing.yml with the CMT_MATRIX input.
+# Lightweight trigger workflow for Compatibility Matrix Testing (CMT).
 #
-# Usage: Go to Actions → CMT Provisioner → Run workflow
-#   server_versions: comma-separated Mattermost server versions
-#   Example: "v11.1.0, v11.2.0, v12.0.0"
+# Matterwick listens for this workflow's workflow_run:requested event, then provisions one
+# Mattermost cloud instance per version in its hardcoded CMT version set (Matterwick's
+# CMTServerVersions config / CMTVersions, e.g. the active ESR plus the current feature
+# release) and dispatches compatibility-matrix-testing.yml with the CMT_MATRIX input.
+# When compatibility-matrix-testing.yml completes, Matterwick destroys the provisioned
+# instances (cleanup is keyed off the completed workflow_run, matched by commit SHA).
+#
+# The server-version set lives in Matterwick config (managed via gitops), so this workflow
+# takes no inputs. Run it manually from the Actions tab, or let it fire automatically on
+# release branches (below).
+#
+# NOTE (intentional duplicate coverage): a release-v* push ALSO triggers Matterwick's normal
+# release E2E flow (whole suite against the single latest server version). CMT additionally
+# runs the whole suite against its multi-version matrix, whose set includes the latest server.
+# So on a release push the latest server version gets the suite run twice (once by the normal
+# release flow, once by the CMT latest-version leg). This is expected and accepted.
 
 on:
-  workflow_dispatch:
-    inputs:
-      server_versions:
-        description: "Comma-separated Mattermost server versions (e.g. v11.1.0, v11.2.0)"
-        required: true
-        type: string
+  # Run CMT when a release branch is cut and on every subsequent push to it. The first push
+  # that creates release-v* is the "cut"; later pushes to it re-run CMT.
+  push:
+    branches:
+      - "release-v*"
+  # Manual runs from the Actions tab (against any branch).
+  workflow_dispatch: {}
+
+# This workflow does not use GITHUB_TOKEN. Its only purpose is to emit a workflow_run event
+# that Matterwick reacts to; it does not check out the repo or call the GitHub API. Drop all
+# GITHUB_TOKEN scopes so the runner-issued token cannot modify the repo if a step is added later.
+permissions: {}
 
 jobs:
-  notify:
+  signal:
     runs-on: ubuntu-22.04
     steps:
-      - name: Log CMT request
-        env:
-          SERVER_VERSIONS: ${{ inputs.server_versions }}
+      - name: Signal Matterwick to provision CMT servers
         run: |
-          echo "CMT Provisioner triggered"
-          echo "Server versions: ${SERVER_VERSIONS}"
-          echo "Matterwick will provision cloud instances and dispatch compatibility-matrix-testing.yml"
+          echo "CMT trigger started."
+          echo "Matterwick provisions one server per configured version and then"
+          echo "dispatches compatibility-matrix-testing.yml automatically."

.github/workflows/compatibility-matrix-testing.yml

@@ -10,11 +10,13 @@ on:
       DESKTOP_VERSION:
         description: "The desktop version to test"
         required: true
-      cmt_run_id:
-        description: "CMT provisioner run ID used for instance cleanup"
-        required: false
-        default: "0"
-        type: string
+
+# Supersede an in-flight CMT run for the same ref (e.g. rapid release-branch pushes) so we don't
+# pay for overlapping multi-version matrices. Matterwick reaps the cancelled run's provisioned
+# servers on its workflow_run "completed" (cancelled) event, matched by commit SHA.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
   ## This is picked up after the finish for cleanup
@@ -141,27 +143,6 @@ jobs:
           description: "Compatibility Matrix Testing for ${{ inputs.DESKTOP_VERSION }} version"
           status: success
 
-  cleanup-cmt-instances:
-    runs-on: ubuntu-22.04
-    if: always()
-    needs:
-      - e2e
-    steps:
-      - name: cmt/cleanup-instances
-        if: ${{ inputs.cmt_run_id != '0' && inputs.cmt_run_id != '' }}
-        env:
-          MATTERWICK_URL: ${{ vars.MATTERWICK_URL }}
-          CLEANUP_TOKEN: ${{ secrets.MATTERWICK_CLEANUP_SECRET }}
-          CMT_RUN_ID: ${{ inputs.cmt_run_id }}
-        run: |
-          if ! [[ "${CMT_RUN_ID}" =~ ^[0-9]+$ ]]; then
-            echo "ERROR: cmt_run_id '${CMT_RUN_ID}' is not a valid numeric ID"
-            exit 1
-          fi
-          curl -S -X POST "${MATTERWICK_URL}/cleanup_e2e" \
-            -H "Content-Type: application/json" \
-            -H "X-Cleanup-Token: ${CLEANUP_TOKEN}" \
-            -d "{\"repo\": \"desktop\", \"run_id\": ${CMT_RUN_ID}}" \
-            --fail-with-body \
-            --retry 3 --retry-delay 5 --retry-connrefused \
-            --connect-timeout 10 --max-time 30
+  # Instance cleanup is handled by Matterwick: when this workflow completes, GitHub sends a
+  # workflow_run "completed" event and Matterwick destroys the servers it provisioned for this
+  # run (matched by commit SHA). No in-workflow cleanup call is needed.

.github/workflows/e2e-functional-template.yml

@@ -67,6 +67,33 @@ on:
       STATUS_MACOS:
         description: "The status of the macOS test"
         value: ${{ jobs.e2e.outputs.STATUS_MACOS }}
+      PASSED_LINUX:
+        description: "Number of passed tests on Linux"
+        value: ${{ jobs.e2e.outputs.PASSED_LINUX }}
+      PASSED_MACOS:
+        description: "Number of passed tests on macOS"
+        value: ${{ jobs.e2e.outputs.PASSED_MACOS }}
+      PASSED_WINDOWS:
+        description: "Number of passed tests on Windows"
+        value: ${{ jobs.e2e.outputs.PASSED_WINDOWS }}
+      TOTAL_LINUX:
+        description: "Total tests on Linux (passed + failed + skipped)"
+        value: ${{ jobs.e2e.outputs.TOTAL_LINUX }}
+      TOTAL_MACOS:
+        description: "Total tests on macOS (passed + failed + skipped)"
+        value: ${{ jobs.e2e.outputs.TOTAL_MACOS }}
+      TOTAL_WINDOWS:
+        description: "Total tests on Windows (passed + failed + skipped)"
+        value: ${{ jobs.e2e.outputs.TOTAL_WINDOWS }}
+      SKIPPED_LINUX:
+        description: "Number of skipped tests on Linux"
+        value: ${{ jobs.e2e.outputs.SKIPPED_LINUX }}
+      SKIPPED_MACOS:
+        description: "Number of skipped tests on macOS"
+        value: ${{ jobs.e2e.outputs.SKIPPED_MACOS }}
+      SKIPPED_WINDOWS:
+        description: "Number of skipped tests on Windows"
+        value: ${{ jobs.e2e.outputs.SKIPPED_WINDOWS }}
       STATUS_WINDOWS:
         description: "The status of the windows test"
         value: ${{ jobs.e2e.outputs.STATUS_WINDOWS }}
@@ -130,6 +157,7 @@ jobs:
   e2e:
     name: e2e-on-${{ inputs.runs-on }}
     runs-on: ${{ inputs.runs-on }}
+    # Runs untrusted PR code (npm ci / Playwright) with a read-only token.
     permissions:
       contents: read
     defaults:
@@ -145,6 +173,15 @@ jobs:
       STATUS_LINUX: ${{ steps.analyze-flaky-tests.outputs.STATUS_LINUX }}
       STATUS_WINDOWS: ${{ steps.analyze-flaky-tests.outputs.STATUS_WINDOWS }}
       STATUS_MACOS: ${{ steps.analyze-flaky-tests.outputs.STATUS_MACOS }}
+      PASSED_LINUX: ${{ steps.analyze-flaky-tests.outputs.PASSED_LINUX }}
+      PASSED_MACOS: ${{ steps.analyze-flaky-tests.outputs.PASSED_MACOS }}
+      PASSED_WINDOWS: ${{ steps.analyze-flaky-tests.outputs.PASSED_WINDOWS }}
+      TOTAL_LINUX: ${{ steps.analyze-flaky-tests.outputs.TOTAL_LINUX }}
+      TOTAL_MACOS: ${{ steps.analyze-flaky-tests.outputs.TOTAL_MACOS }}
+      TOTAL_WINDOWS: ${{ steps.analyze-flaky-tests.outputs.TOTAL_WINDOWS }}
+      SKIPPED_LINUX: ${{ steps.analyze-flaky-tests.outputs.SKIPPED_LINUX }}
+      SKIPPED_MACOS: ${{ steps.analyze-flaky-tests.outputs.SKIPPED_MACOS }}
+      SKIPPED_WINDOWS: ${{ steps.analyze-flaky-tests.outputs.SKIPPED_WINDOWS }}
     steps:
       - name: e2e/set-required-variables
         id: variables
@@ -229,6 +266,18 @@ jobs:
           npm ci
           cd e2e && npm ci
 
+      - name: e2e/suppress-macos-dialogs
+        if: runner.os == 'macOS'
+        run: |
+          # Suppress the "Reopen windows" dialog that blocks Electron startup
+          # when a previous test instance was killed by SIGTERM/SIGKILL.
+          # The global-setup.ts also does this, but running it here as well ensures
+          # the setting is applied even if the Playwright globalSetup step is skipped.
+          defaults write com.github.Electron NSQuitAlwaysKeepsWindows -bool false || true
+          defaults write com.github.Electron ApplePersistenceIgnoreState -bool YES || true
+          defaults write com.apple.LaunchServices LSQuarantine -bool false || true
+          defaults write com.apple.CrashReporter DialogType none || true
+
       - name: e2e/run-playwright-tests-linux
         if: runner.os == 'Linux'
         run: |
@@ -274,31 +323,72 @@ jobs:
           DESKTOP_VERSION: ${{ inputs.DESKTOP_VERSION }}
           CI_ENVIRONMENT_NAME: ${{ env.CI_ENVIRONMENT_NAME }}
 
+      - name: e2e/generate-html-report
+        id: generate-html-report
+        if: always()
+        run: |
+          cd e2e
+          if [ -d blob-report ] && find blob-report -type f | grep -q .; then
+            npx playwright merge-reports --reporter=html blob-report
+            echo "html_report_ready=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "No blob report produced for this OS — skipping HTML generation."
+            echo "html_report_ready=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: e2e/upload-html-report-to-s3
+        id: upload-html-report-to-s3
+        if: always() && steps.generate-html-report.outputs.html_report_ready == 'true'
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.MM_DESKTOP_E2E_AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.MM_DESKTOP_E2E_AWS_SECRET_ACCESS_KEY }}
+          AWS_REGION: us-east-1
+          AWS_S3_BUCKET: mattermost-cypress-report
+          RUN_ID: ${{ github.run_id }}
+          RUN_ATTEMPT: ${{ github.run_attempt }}
+        run: |
+          # Per-OS report path so each platform's status check links to a clean,
+          # single-OS view — not an aggregated multi-OS report where counts and
+          # "skipped" totals are inflated by cross-OS tags.
+          S3_PREFIX="desktop-e2e/${RUN_ID}-${RUN_ATTEMPT}/${RUNNER_OS}"
+          aws s3 sync e2e/playwright-report/ "s3://${AWS_S3_BUCKET}/${S3_PREFIX}/" \
+            --acl public-read \
+            --cache-control "no-cache"
+          REPORT_URL="https://${AWS_S3_BUCKET}.s3.amazonaws.com/${S3_PREFIX}/index.html"
+          echo "report_url=${REPORT_URL}" >> "$GITHUB_OUTPUT"
+          echo "Playwright report (${RUNNER_OS}) uploaded: ${REPORT_URL}" >> "$GITHUB_STEP_SUMMARY"
+
       - name: e2e/analyze-flaky-tests
         id: analyze-flaky-tests
+        if: always()
         uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        env:
+          PER_OS_REPORT_URL: ${{ steps.upload-html-report-to-s3.outputs.report_url }}
         with:
           script: |
             process.chdir('./e2e');
             const { analyzeFlakyTests } = require('./utils/analyze-flaky-test.js');
-            const { failureCount, os } = analyzeFlakyTests();
+            const { failureCount, passCount, skipCount, totalCount, os } = analyzeFlakyTests();
             const runUrl = `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`;
+            const reportUrl = process.env.PER_OS_REPORT_URL || runUrl;
             const testStatus = failureCount > 0 || Number(process.env.PLAYWRIGHT_EXIT_CODE || 0) !== 0 ? 'failure' : 'success';
+            const setOSOutputs = (suffix) => {
+              core.setOutput(`NEW_FAILURES_${suffix}`, String(failureCount));
+              core.setOutput(`REPORT_LINK_${suffix}`, reportUrl);
+              core.setOutput(`STATUS_${suffix}`, testStatus);
+              core.setOutput(`PASSED_${suffix}`, String(passCount));
+              core.setOutput(`SKIPPED_${suffix}`, String(skipCount));
+              core.setOutput(`TOTAL_${suffix}`, String(totalCount));
+            };
             switch (os) {
               case 'linux':
-                core.setOutput('NEW_FAILURES_LINUX', String(failureCount));
-                core.setOutput('REPORT_LINK_LINUX', runUrl);
-                core.setOutput('STATUS_LINUX', testStatus);
+                setOSOutputs('LINUX');
                 break;
               case 'darwin':
-                core.setOutput('NEW_FAILURES_MACOS', String(failureCount));
-                core.setOutput('REPORT_LINK_MACOS', runUrl);
-                core.setOutput('STATUS_MACOS', testStatus);
+                setOSOutputs('MACOS');
                 break;
               case 'win32':
-                core.setOutput('NEW_FAILURES_WINDOWS', String(failureCount));
-                core.setOutput('REPORT_LINK_WINDOWS', runUrl);
-                core.setOutput('STATUS_WINDOWS', testStatus);
+                setOSOutputs('WINDOWS');
                 break;
               default:
                 throw new Error(`Unsupported OS: ${os}`);