diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7626d699416..16070403c68 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -214,6 +214,7 @@ jobs: - name: ci/publish-results uses: EnricoMi/publish-unit-test-result-action@d0a4676d0e0b938bc201470d88276b7c74c712b3 # v2.24.0 with: + check_name: Unit Tests (Jest) comment_mode: failures compare_to_earlier_commit: false junit_files: "**/*.xml" diff --git a/.github/workflows/cmt-provisioner.yml b/.github/workflows/cmt-provisioner.yml index 8b3e86d3283..5e974791f26 100644 --- a/.github/workflows/cmt-provisioner.yml +++ b/.github/workflows/cmt-provisioner.yml @@ -25,7 +25,7 @@ permissions: {} jobs: trigger-matterwick: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Signal Matterwick to provision CMT servers run: | diff --git a/.github/workflows/compatibility-matrix-testing.yml b/.github/workflows/compatibility-matrix-testing.yml index 5aa0b1551dc..a11589d4a24 100644 --- a/.github/workflows/compatibility-matrix-testing.yml +++ b/.github/workflows/compatibility-matrix-testing.yml @@ -20,9 +20,13 @@ concurrency: cancel-in-progress: true jobs: + # TSIO reporting model (CMT): + # 1 workflow run → 1 RC cut (DESKTOP_VERSION) → 1 resolved SHA + # → 1 composite_identity (name=cmt-desktop) → 1 TSIO report covering + # every (OS × server-version) matrix leg. ## This is picked up after the finish for cleanup upload-cmt-server-detals: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: cmt/generate-instance-details-file run: echo '${{ inputs.CMT_MATRIX }}' > instance-details.json @@ -35,9 +39,12 @@ jobs: retention-days: 1 calculate-commit-hash: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 outputs: DESKTOP_SHA: ${{ steps.repo.outputs.DESKTOP_SHA }} + CMT_MATRIX: ${{ steps.normalize-matrix.outputs.matrix }} + TSIO_COMPOSITE_IDENTITY: ${{ steps.tsio-identity.outputs.composite-identity-json }} + TSIO_TOTAL_REPORTS_EXPECTED: ${{ steps.tsio-identity.outputs.total-reports-expected }} steps: - name: cmt/checkout-desktop uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -48,12 +55,56 @@ jobs: id: repo run: echo "DESKTOP_SHA=$(git rev-parse HEAD)" >> ${GITHUB_OUTPUT} + # Matterwick still dispatches ubuntu-22.04 for Linux; pin to ubuntu-latest so + # CMT matches PR E2E (24.04) and install-os-dependencies can resolve + # libasound2t64. Same idea as e2e-functional.yml remapping macos-latest → macos-26. + - name: cmt/normalize-matrix-runners + id: normalize-matrix + env: + CMT_MATRIX: ${{ inputs.CMT_MATRIX }} + run: | + NORMALIZED=$(echo "${CMT_MATRIX}" | jq -c ' + .environment |= map( + if .runner == "ubuntu-22.04" then .runner = "ubuntu-latest" else . end + ) + ') + echo "matrix=${NORMALIZED}" >> "$GITHUB_OUTPUT" + + # One composite identity + total-reports-expected for the WHOLE CMT run + # (every OS x server-version leg), computed once here and threaded into + # every `e2e` matrix leg below so they land in a single TSIO report group. + - name: cmt/tsio-identity + id: tsio-identity + env: + GITHUB_REPOSITORY: ${{ github.repository }} + MM_SHA: ${{ steps.repo.outputs.DESKTOP_SHA }} + MM_BRANCH: ${{ inputs.DESKTOP_VERSION }} + CMT_MATRIX: ${{ steps.normalize-matrix.outputs.matrix }} + run: | + TOTAL=$(echo "${CMT_MATRIX}" | jq -c '(.environment | length) * (.server | length)') + echo "total-reports-expected=${TOTAL}" >> "$GITHUB_OUTPUT" + COMPOSITE_IDENTITY=$(jq -nc \ + --arg repo "${GITHUB_REPOSITORY}" \ + --arg sha "${MM_SHA}" \ + --arg run_id "${GITHUB_RUN_ID}" \ + --arg attempt "${GITHUB_RUN_ATTEMPT}" \ + --arg name "cmt-desktop" \ + --arg branch "${MM_BRANCH}" \ + '{repository:$repo, commit_sha:$sha, gh_run_id:$run_id, name:$name, gh_run_attempt:$attempt, branch:$branch}') + echo "composite-identity-json=${COMPOSITE_IDENTITY}" >> "$GITHUB_OUTPUT" + update-initial-status: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 needs: - calculate-commit-hash + permissions: + contents: read + statuses: write steps: + # Best-effort: a transient status-API hiccup here must never block the + # actual test matrix from running — the `e2e` job below needs this job. - uses: mattermost/actions/delivery/update-commit-status@218fd96a63451259dc100e1292a5e44af92fe15d + continue-on-error: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: @@ -63,7 +114,8 @@ jobs: description: "Compatibility Matrix Testing for ${{ inputs.DESKTOP_VERSION }} version" status: pending - # Input follows the below schema + # Input follows the below schema (Matterwick may still send ubuntu-22.04; + # calculate-commit-hash remaps Linux to ubuntu-latest before the e2e matrix). # { # "environment": [ # { @@ -94,10 +146,15 @@ jobs: name: ${{ matrix.environment.os }}-${{ matrix.server.version }} uses: ./.github/workflows/e2e-functional-template.yml needs: + - calculate-commit-hash - update-initial-status strategy: fail-fast: false - matrix: ${{ fromJson(inputs.CMT_MATRIX) }} + matrix: ${{ fromJson(needs.calculate-commit-hash.outputs.CMT_MATRIX) }} + permissions: + contents: read + actions: read + id-token: write secrets: inherit with: runs-on: ${{ matrix.environment.runner }} @@ -106,43 +163,55 @@ jobs: DESKTOP_VERSION: ${{ inputs.DESKTOP_VERSION }} MM_SERVER_VERSION: ${{ matrix.server.version }} TYPE: "CMT" + # Packed as one JSON input (see e2e-functional-template.yml's tsio-config) — + # workflow_call caps reusable workflows at 10 inputs total. + tsio-config: ${{ format('{{"composite_identity":{0},"total_reports_expected":"{1}"}}', needs.calculate-commit-hash.outputs.TSIO_COMPOSITE_IDENTITY, needs.calculate-commit-hash.outputs.TSIO_TOTAL_REPORTS_EXPECTED) }} - # We need to duplicate here in order to set the proper commit status - # https://mattermost.atlassian.net/browse/CLD-5815 - update-failure-final-status: - runs-on: ubuntu-22.04 - if: failure() || cancelled() + update-final-status: + runs-on: ubuntu-24.04 + if: always() + permissions: + contents: read + id-token: write + statuses: write needs: - calculate-commit-hash - e2e steps: - - uses: mattermost/actions/delivery/update-commit-status@218fd96a63451259dc100e1292a5e44af92fe15d - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # tsio-report-status.js is repo code, not fetched by github-script itself. + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: - repository_full_name: mattermost/desktop - commit_sha: ${{ needs.calculate-commit-hash.outputs.DESKTOP_SHA }} - context: e2e/compatibility-matrix-testing - description: "Compatibility Matrix Testing for ${{ inputs.DESKTOP_VERSION }} version" - status: failure + ref: ${{ inputs.DESKTOP_VERSION }} + persist-credentials: false + sparse-checkout: | + e2e/utils/tsio-report-status.js + sparse-checkout-cone-mode: false - # https://mattermost.atlassian.net/browse/CLD-5815 - update-success-final-status: - runs-on: ubuntu-22.04 - if: success() - needs: - - calculate-commit-hash - - e2e - steps: - - uses: mattermost/actions/delivery/update-commit-status@218fd96a63451259dc100e1292a5e44af92fe15d + - name: Render TSIO summary + flip commit status + id: summary + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TSIO_COMPOSITE_IDENTITY: ${{ needs.calculate-commit-hash.outputs.TSIO_COMPOSITE_IDENTITY }} + TSIO_TOTAL_REPORTS_EXPECTED: ${{ needs.calculate-commit-hash.outputs.TSIO_TOTAL_REPORTS_EXPECTED }} + TSIO_POLL_ATTEMPTS: 24 + TSIO_POLL_DELAY_MS: 5000 + COMMIT_STATUS_CONTEXT: e2e/compatibility-matrix-testing + # TSIO only sees test-case-level results — a job-level failure with no + # matching failed test (hung worker teardown, crashed runner, npm ci + # failure) would otherwise still read as "100% passed". + UPSTREAM_JOBS_SUCCEEDED: ${{ needs.e2e.result == 'success' }} with: - repository_full_name: mattermost/desktop - commit_sha: ${{ needs.calculate-commit-hash.outputs.DESKTOP_SHA }} - context: e2e/compatibility-matrix-testing - description: "Compatibility Matrix Testing for ${{ inputs.DESKTOP_VERSION }} version" - status: success + script: | + const {reportUrl, status, stats} = await require('./e2e/utils/tsio-report-status.js')({ + core, context, github, + compositeIdentity: JSON.parse(process.env.TSIO_COMPOSITE_IDENTITY), + totalReportsExpected: parseInt(process.env.TSIO_TOTAL_REPORTS_EXPECTED, 10), + commitStatusContext: process.env.COMMIT_STATUS_CONTEXT, + upstreamJobsSucceeded: process.env.UPSTREAM_JOBS_SUCCEEDED === 'true', + failOnTestFailures: true, + }); + core.info(`TSIO report ${reportUrl}: ${status} (${JSON.stringify(stats)})`); + # Instance cleanup is handled by Matterwick: when this workflow completes, GitHub sends a # workflow_run "completed" event and Matterwick destroys the servers it provisioned for this diff --git a/.github/workflows/e2e-functional-template.yml b/.github/workflows/e2e-functional-template.yml index 459afcbd798..e9dc11d34c6 100644 --- a/.github/workflows/e2e-functional-template.yml +++ b/.github/workflows/e2e-functional-template.yml @@ -37,71 +37,14 @@ on: MM_SERVER_VERSION: type: string required: true - outputs: - NEW_FAILURES_LINUX: - description: "The output to comment" - value: ${{ jobs.e2e.outputs.NEW_FAILURES_LINUX }} - NEW_FAILURES_MACOS: - description: "The output to comment" - value: ${{ jobs.e2e.outputs.NEW_FAILURES_MACOS }} - NEW_FAILURES_WINDOWS: - description: "The output to comment" - value: ${{ jobs.e2e.outputs.NEW_FAILURES_WINDOWS }} - REPORT_LINK_LINUX: - description: "Link to Linux report" - value: ${{ jobs.e2e.outputs.REPORT_LINK_LINUX }} - REPORT_LINK_MACOS: - description: "Link to MacOS report" - value: ${{ jobs.e2e.outputs.REPORT_LINK_MACOS }} - REPORT_LINK_WINDOWS: - description: "Link to Windows report" - value: ${{ jobs.e2e.outputs.REPORT_LINK_WINDOWS }} - STATUS_LINUX: - description: "The status of the linux test" - value: ${{ jobs.e2e.outputs.STATUS_LINUX }} - STATUS_MACOS: - description: "The status of the macOS test" - value: ${{ jobs.e2e.outputs.STATUS_MACOS }} - PASSED_LINUX: - description: "Number of passed tests on Linux" - value: ${{ jobs.e2e.outputs.PASSED_LINUX }} - PASSED_MACOS: - description: "Number of passed tests on macOS" - value: ${{ jobs.e2e.outputs.PASSED_MACOS }} - PASSED_WINDOWS: - description: "Number of passed tests on Windows" - value: ${{ jobs.e2e.outputs.PASSED_WINDOWS }} - TOTAL_LINUX: - description: "Total tests on Linux (passed + failed + skipped)" - value: ${{ jobs.e2e.outputs.TOTAL_LINUX }} - TOTAL_MACOS: - description: "Total tests on macOS (passed + failed + skipped)" - value: ${{ jobs.e2e.outputs.TOTAL_MACOS }} - TOTAL_WINDOWS: - description: "Total tests on Windows (passed + failed + skipped)" - value: ${{ jobs.e2e.outputs.TOTAL_WINDOWS }} - SKIPPED_LINUX: - description: "Number of skipped tests on Linux" - value: ${{ jobs.e2e.outputs.SKIPPED_LINUX }} - SKIPPED_MACOS: - description: "Number of skipped tests on macOS" - value: ${{ jobs.e2e.outputs.SKIPPED_MACOS }} - SKIPPED_WINDOWS: - description: "Number of skipped tests on Windows" - value: ${{ jobs.e2e.outputs.SKIPPED_WINDOWS }} - COLLECTION_FAILED_LINUX: - description: "Whether Playwright collected zero tests on Linux" - value: ${{ jobs.e2e.outputs.COLLECTION_FAILED_LINUX }} - COLLECTION_FAILED_MACOS: - description: "Whether Playwright collected zero tests on macOS" - value: ${{ jobs.e2e.outputs.COLLECTION_FAILED_MACOS }} - COLLECTION_FAILED_WINDOWS: - description: "Whether Playwright collected zero tests on Windows" - value: ${{ jobs.e2e.outputs.COLLECTION_FAILED_WINDOWS }} - STATUS_WINDOWS: - description: "The status of the windows test" - value: ${{ jobs.e2e.outputs.STATUS_WINDOWS }} - + # Packed as one JSON input, not two, because workflow_call caps reusable + # workflows at 10 inputs total — this file was already at 8. + # Shape: {"composite_identity": {...}, "total_reports_expected": "N"} + tsio-config: + description: "TSIO reporting config: {composite_identity, total_reports_expected}. Empty disables TSIO reporting for this leg." + required: false + type: string + default: "" workflow_dispatch: inputs: MM_TEST_SERVER_URL: @@ -139,6 +82,11 @@ on: MM_SERVER_VERSION: type: string required: true + tsio-config: + description: "TSIO reporting config: {composite_identity, total_reports_expected}. Empty disables TSIO reporting for this leg." + required: false + type: string + default: "" env: BRANCH: ${{ github.head_ref || github.ref_name }} @@ -154,36 +102,18 @@ env: jobs: e2e: - name: e2e-on-${{ inputs.runs-on }} + # Includes MM_SERVER_VERSION, not just runs-on: CMT tests multiple server + # versions against the same runner (e.g. macos-13 x 9.6.1 and macos-13 x + # 9.5.2) + name: e2e-on-${{ inputs.runs-on }}-${{ inputs.MM_SERVER_VERSION }} runs-on: ${{ inputs.runs-on }} - # Runs untrusted PR code (npm ci / Playwright) with a read-only token. permissions: contents: read + id-token: write + actions: read defaults: run: shell: bash - outputs: - NEW_FAILURES_LINUX: ${{ steps.analyze-flaky-tests.outputs.NEW_FAILURES_LINUX }} - NEW_FAILURES_MACOS: ${{ steps.analyze-flaky-tests.outputs.NEW_FAILURES_MACOS }} - NEW_FAILURES_WINDOWS: ${{ steps.analyze-flaky-tests.outputs.NEW_FAILURES_WINDOWS }} - REPORT_LINK_LINUX: ${{ steps.analyze-flaky-tests.outputs.REPORT_LINK_LINUX }} - REPORT_LINK_MACOS: ${{ steps.analyze-flaky-tests.outputs.REPORT_LINK_MACOS }} - REPORT_LINK_WINDOWS: ${{ steps.analyze-flaky-tests.outputs.REPORT_LINK_WINDOWS }} - STATUS_LINUX: ${{ steps.analyze-flaky-tests.outputs.STATUS_LINUX }} - STATUS_WINDOWS: ${{ steps.analyze-flaky-tests.outputs.STATUS_WINDOWS }} - STATUS_MACOS: ${{ steps.analyze-flaky-tests.outputs.STATUS_MACOS }} - PASSED_LINUX: ${{ steps.analyze-flaky-tests.outputs.PASSED_LINUX }} - PASSED_MACOS: ${{ steps.analyze-flaky-tests.outputs.PASSED_MACOS }} - PASSED_WINDOWS: ${{ steps.analyze-flaky-tests.outputs.PASSED_WINDOWS }} - TOTAL_LINUX: ${{ steps.analyze-flaky-tests.outputs.TOTAL_LINUX }} - TOTAL_MACOS: ${{ steps.analyze-flaky-tests.outputs.TOTAL_MACOS }} - TOTAL_WINDOWS: ${{ steps.analyze-flaky-tests.outputs.TOTAL_WINDOWS }} - SKIPPED_LINUX: ${{ steps.analyze-flaky-tests.outputs.SKIPPED_LINUX }} - SKIPPED_MACOS: ${{ steps.analyze-flaky-tests.outputs.SKIPPED_MACOS }} - SKIPPED_WINDOWS: ${{ steps.analyze-flaky-tests.outputs.SKIPPED_WINDOWS }} - COLLECTION_FAILED_LINUX: ${{ steps.analyze-flaky-tests.outputs.COLLECTION_FAILED_LINUX }} - COLLECTION_FAILED_MACOS: ${{ steps.analyze-flaky-tests.outputs.COLLECTION_FAILED_MACOS }} - COLLECTION_FAILED_WINDOWS: ${{ steps.analyze-flaky-tests.outputs.COLLECTION_FAILED_WINDOWS }} steps: - name: e2e/set-required-variables id: variables @@ -348,74 +278,38 @@ jobs: DESKTOP_VERSION: ${{ inputs.DESKTOP_VERSION }} CI_ENVIRONMENT_NAME: ${{ env.CI_ENVIRONMENT_NAME }} - - name: e2e/generate-html-report - id: generate-html-report + - name: e2e/write-tsio-failure-stub + if: ${{ always() && inputs.tsio-config != '' && hashFiles('e2e/test-results/results.json') == '' && (failure() || cancelled() || (env.PLAYWRIGHT_EXIT_CODE != '0' && env.PLAYWRIGHT_EXIT_CODE != '')) }} + run: node e2e/utils/write-tsio-failure-stub.mjs + env: + TSIO_GH_JOB_NAME: e2e-on-${{ inputs.runs-on }}-${{ inputs.MM_SERVER_VERSION }} + PLAYWRIGHT_EXIT_CODE: ${{ env.PLAYWRIGHT_EXIT_CODE }} + + - name: e2e/upload-report-to-tsio + if: ${{ always() && inputs.tsio-config != '' && hashFiles('e2e/test-results/results.json') != '' }} + uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-report-upload@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 0.10.0 / 2026-05-16 + with: + composite-identity: ${{ toJSON(fromJSON(inputs.tsio-config).composite_identity) }} + total-reports-expected: ${{ fromJSON(inputs.tsio-config).total_reports_expected }} + framework: playwright + github-token: ${{ secrets.GITHUB_TOKEN }} + # MUST match this job's `name:` field above. + gh-job-name: e2e-on-${{ inputs.runs-on }}-${{ inputs.MM_SERVER_VERSION }} + json-path: e2e/test-results/results.json + screenshots-dir: e2e/test-results + + - name: e2e/handle-nonzero-playwright-exit if: always() run: | - cd e2e - if [ -d blob-report ] && find blob-report -type f | grep -q .; then - npx playwright merge-reports --reporter=html blob-report - echo "html_report_ready=true" >> "$GITHUB_OUTPUT" + if [ "${PLAYWRIGHT_EXIT_CODE:-1}" = "0" ]; then + exit 0 + fi + if [ -n "${TSIO_CONFIG}" ]; then + echo "::warning::Playwright exited with code ${PLAYWRIGHT_EXIT_CODE:-unset} — not failing the job on this alone, see TSIO report for actual test results" else - echo "No blob report produced for this OS — skipping HTML generation." - echo "html_report_ready=false" >> "$GITHUB_OUTPUT" + echo "Playwright exited with code ${PLAYWRIGHT_EXIT_CODE:-unset} and no TSIO config is set for this run — failing the job since there is no other failure signal" >&2 + exit 1 fi - - - name: e2e/upload-html-report-to-s3 - id: upload-html-report-to-s3 - if: always() && steps.generate-html-report.outputs.html_report_ready == 'true' env: - AWS_ACCESS_KEY_ID: ${{ secrets.MM_DESKTOP_E2E_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.MM_DESKTOP_E2E_AWS_SECRET_ACCESS_KEY }} - AWS_REGION: us-east-1 - AWS_S3_BUCKET: mattermost-cypress-report - RUN_ID: ${{ github.run_id }} - RUN_ATTEMPT: ${{ github.run_attempt }} - run: | - # Per-OS report path so each platform's status check links to a clean, - # single-OS view — not an aggregated multi-OS report where counts and - # "skipped" totals are inflated by cross-OS tags. - S3_PREFIX="desktop-e2e/${RUN_ID}-${RUN_ATTEMPT}/${RUNNER_OS}" - aws s3 sync e2e/playwright-report/ "s3://${AWS_S3_BUCKET}/${S3_PREFIX}/" \ - --acl public-read \ - --cache-control "no-cache" - REPORT_URL="https://${AWS_S3_BUCKET}.s3.amazonaws.com/${S3_PREFIX}/index.html" - echo "report_url=${REPORT_URL}" >> "$GITHUB_OUTPUT" - echo "Playwright report (${RUNNER_OS}) uploaded: ${REPORT_URL}" >> "$GITHUB_STEP_SUMMARY" + TSIO_CONFIG: ${{ inputs.tsio-config }} - - name: e2e/analyze-flaky-tests - id: analyze-flaky-tests - if: always() - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - PER_OS_REPORT_URL: ${{ steps.upload-html-report-to-s3.outputs.report_url }} - JOB_STATUS: ${{ job.status }} - with: - script: | - process.chdir('./e2e'); - const { analyzeFlakyTests } = require('./utils/analyze-flaky-test.js'); - const { failureCount, passCount, skipCount, totalCount, os, testStatus, collectionFailed } = analyzeFlakyTests(); - const runUrl = `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`; - const reportUrl = process.env.PER_OS_REPORT_URL || runUrl; - const setOSOutputs = (suffix) => { - core.setOutput(`NEW_FAILURES_${suffix}`, String(failureCount)); - core.setOutput(`REPORT_LINK_${suffix}`, reportUrl); - core.setOutput(`STATUS_${suffix}`, testStatus); - core.setOutput(`PASSED_${suffix}`, String(passCount)); - core.setOutput(`SKIPPED_${suffix}`, String(skipCount)); - core.setOutput(`TOTAL_${suffix}`, String(totalCount)); - core.setOutput(`COLLECTION_FAILED_${suffix}`, String(collectionFailed)); - }; - switch (os) { - case 'linux': - setOSOutputs('LINUX'); - break; - case 'darwin': - setOSOutputs('MACOS'); - break; - case 'win32': - setOSOutputs('WINDOWS'); - break; - default: - throw new Error(`Unsupported OS: ${os}`); - } diff --git a/.github/workflows/e2e-functional.yml b/.github/workflows/e2e-functional.yml index 3dfb2df31b4..41f71d1ad31 100644 --- a/.github/workflows/e2e-functional.yml +++ b/.github/workflows/e2e-functional.yml @@ -39,11 +39,18 @@ permissions: contents: read jobs: + # TSIO reporting model (PR / master / release): + # 1 workflow run → 1 composite_identity (gh_run_id + resolved commit SHA + name) + # → 1 TSIO report group aggregating every platform leg + both policy legs. + # Matterwick sets run_type=MASTER for master pushes; PR runs default to desktop-pr. prepare-matrix: if: ${{ github.event_name == 'workflow_dispatch' && inputs.instance_details != '' }} runs-on: ubuntu-latest outputs: platforms: ${{ steps.generate.outputs.platforms }} + desktop-sha: ${{ steps.resolve-sha.outputs.sha }} + tsio-composite-identity: ${{ steps.tsio-identity.outputs.composite-identity-json }} + tsio-total-reports-expected: ${{ steps.tsio-identity.outputs.total-reports-expected }} steps: - id: generate env: @@ -54,30 +61,66 @@ jobs: platforms=$(echo "${INSTANCE_DETAILS}" | jq -c 'map(if .runner == "macos-latest" then .runner = "macos-26" else . end)') echo "platforms=${platforms}" >> "$GITHUB_OUTPUT" + # Resolve the SHA under test from version_name (branch/tag/SHA), not github.sha + # from the dispatch ref — same approach as compatibility-matrix-testing.yml. + - name: e2e/checkout-desktop-version + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ inputs.version_name }} + + - id: resolve-sha + run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + + - id: tsio-identity + env: + GITHUB_REPOSITORY: ${{ github.repository }} + MM_SHA: ${{ steps.resolve-sha.outputs.sha }} + MM_BRANCH: ${{ inputs.version_name }} + PR_NUMBER: ${{ inputs.pr_number }} + RUN_TYPE: ${{ inputs.run_type || 'PR' }} + PLATFORMS: ${{ steps.generate.outputs.platforms }} + run: | + # platform legs (linux/macos/windows) + policy-tests-macos + policy-tests-windows + TOTAL=$(( $(echo "${PLATFORMS}" | jq -c 'length') + 2 )) + echo "total-reports-expected=${TOTAL}" >> "$GITHUB_OUTPUT" + NAME="desktop-$(echo "${RUN_TYPE}" | tr '[:upper:]' '[:lower:]')" + if [ -n "${PR_NUMBER}" ]; then + COMPOSITE_IDENTITY=$(jq -nc \ + --arg repo "${GITHUB_REPOSITORY}" --arg sha "${MM_SHA}" --arg run_id "${GITHUB_RUN_ID}" \ + --arg name "${NAME}" --arg attempt "${GITHUB_RUN_ATTEMPT}" --arg branch "${MM_BRANCH}" --arg pr "${PR_NUMBER}" \ + '{repository:$repo, commit_sha:$sha, gh_run_id:$run_id, name:$name, gh_run_attempt:$attempt, branch:$branch, gh_pr_number:$pr}') + else + COMPOSITE_IDENTITY=$(jq -nc \ + --arg repo "${GITHUB_REPOSITORY}" --arg sha "${MM_SHA}" --arg run_id "${GITHUB_RUN_ID}" \ + --arg name "${NAME}" --arg attempt "${GITHUB_RUN_ATTEMPT}" --arg branch "${MM_BRANCH}" \ + '{repository:$repo, commit_sha:$sha, gh_run_id:$run_id, name:$name, gh_run_attempt:$attempt, branch:$branch}') + fi + echo "composite-identity-json=${COMPOSITE_IDENTITY}" >> "$GITHUB_OUTPUT" + update-initial-status: - name: Update initial status - needs: prepare-matrix + name: Set pending TSIO status runs-on: ubuntu-24.04 + needs: + - prepare-matrix permissions: contents: read statuses: write steps: - - name: Checkout code - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - - name: Update initial status for all platforms - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + # Best-effort: a transient status-API hiccup here must never block the + # actual test matrix from running. + - uses: mattermost/actions/delivery/update-commit-status@218fd96a63451259dc100e1292a5e44af92fe15d + continue-on-error: true env: - PLATFORMS: ${{ needs.prepare-matrix.outputs.platforms }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - github-token: ${{ github.token }} - script: | - const { updateInitialStatus } = require('./e2e/utils/github-actions.js'); - const platforms = JSON.parse(process.env.PLATFORMS); - await updateInitialStatus({ github, context, platforms }); + repository_full_name: ${{ github.repository }} + commit_sha: ${{ needs.prepare-matrix.outputs.desktop-sha }} + context: e2e-test/desktop-playwright + description: "Running Electron Playwright E2E tests..." + status: pending e2e-tests: - needs: + needs: - prepare-matrix - update-initial-status name: ${{ matrix.platform }} @@ -85,8 +128,10 @@ jobs: matrix: include: ${{ fromJson(needs.prepare-matrix.outputs.platforms) }} fail-fast: false - # The reusable template runs only untrusted PR code (npm ci / Playwright) with a read-only - # token; it inherits the workflow-level contents:read default. + permissions: + contents: read + actions: read + id-token: write uses: ./.github/workflows/e2e-functional-template.yml with: runs-on: ${{ matrix.runner }} @@ -95,60 +140,71 @@ jobs: MM_SERVER_VERSION: ${{ inputs.MM_SERVER_VERSION }} MM_TEST_USER_NAME: ${{ inputs.MM_TEST_USER_NAME }} MM_TEST_PASSWORD: ${{ inputs.MM_TEST_PASSWORD }} - # When matterwick fires this workflow it sets run_type explicitly (MASTER for master - # pushes; PR-label runs are dispatched without run_type, defaulting to PR via the `|| 'PR'` - # below). The previous `startsWith(version_name, 'release-') && 'RELEASE'` fallback was - # for matterwick's release-branch push, which is no longer wired — removed. TYPE: ${{ inputs.run_type || 'PR' }} + tsio-config: ${{ format('{{"composite_identity":{0},"total_reports_expected":"{1}"}}', needs.prepare-matrix.outputs.tsio-composite-identity, needs.prepare-matrix.outputs.tsio-total-reports-expected) }} secrets: inherit - update-final-status: - name: Update final status + # The single E2E status check for PR/master (e2e-test/desktop-playwright), + # replacing the former 5 (e2e/linux, e2e/macos, e2e/windows, + # policy-test/macos, policy-test/windows). + # + # Uses e2e/utils/tsio-report-status.js, not test-system-io-summary — summary + # only reads /api/v1/orchestration/status, which report-upload never + # populates (see that file's header comment). + tsio-summary: + name: TSIO summary runs-on: ubuntu-24.04 needs: - prepare-matrix - e2e-tests + - e2e-policy-tests if: always() permissions: contents: read + id-token: write statuses: write steps: - - name: Checkout code - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + # tsio-report-status.js is repo code, not fetched by github-script itself. + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ inputs.version_name }} + persist-credentials: false + sparse-checkout: | + e2e/utils/tsio-report-status.js + sparse-checkout-cone-mode: false - - name: Update final status for all platforms + - name: Render TSIO summary + flip commit status + id: summary uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: - PR_NUMBER: ${{ inputs.pr_number }} - PLATFORMS: ${{ needs.prepare-matrix.outputs.platforms }} - OUTPUTS: ${{ toJSON(needs.e2e-tests.outputs) }} - E2E_TESTS_RESULT: ${{ needs.e2e-tests.result }} + TSIO_COMPOSITE_IDENTITY: ${{ needs.prepare-matrix.outputs.tsio-composite-identity }} + TSIO_TOTAL_REPORTS_EXPECTED: ${{ needs.prepare-matrix.outputs.tsio-total-reports-expected }} + TSIO_POLL_ATTEMPTS: 12 + TSIO_POLL_DELAY_MS: 5000 + COMMIT_STATUS_CONTEXT: e2e-test/desktop-playwright + UPSTREAM_JOBS_SUCCEEDED: ${{ needs.e2e-tests.result == 'success' && needs.e2e-policy-tests.result == 'success' }} with: - github-token: ${{ github.token }} script: | - const { updateFinalStatus } = require('./e2e/utils/github-actions.js'); - const platforms = JSON.parse(process.env.PLATFORMS); - const outputs = JSON.parse(process.env.OUTPUTS); - const prNumber = parseInt(process.env.PR_NUMBER, 10) || null; - await updateFinalStatus({ - github, - context, - platforms, - outputs, - e2eTestsResult: process.env.E2E_TESTS_RESULT, - prNumber, + const {reportUrl, status, stats} = await require('./e2e/utils/tsio-report-status.js')({ + core, context, github, + compositeIdentity: JSON.parse(process.env.TSIO_COMPOSITE_IDENTITY), + totalReportsExpected: parseInt(process.env.TSIO_TOTAL_REPORTS_EXPECTED, 10), + upstreamJobsSucceeded: process.env.UPSTREAM_JOBS_SUCCEEDED === 'true', + commitStatusContext: process.env.COMMIT_STATUS_CONTEXT, + failOnTestFailures: true, }); + core.info(`TSIO report ${reportUrl}: ${status} (${JSON.stringify(stats)})`); remove-e2e-label: name: Remove E2E label from PR - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 permissions: issues: write pull-requests: write needs: - e2e-tests - e2e-policy-tests - - update-final-status + - tsio-summary if: always() steps: - name: e2e/remove-label-from-pr @@ -205,11 +261,15 @@ jobs: e2e-policy-tests: name: policy-tests-${{ matrix.platform }} - # Runs untrusted PR code (npm ci / Playwright); grant only the commit-status write its - # check-for-failures step needs, never pull-requests: write. + needs: + - prepare-matrix + - update-initial-status + # Runs untrusted PR code (npm ci / Playwright); grant only what the TSIO + # upload needs, never pull-requests: write. permissions: contents: read - statuses: write + id-token: write + actions: read strategy: matrix: include: @@ -227,14 +287,10 @@ jobs: run: shell: bash env: - AWS_S3_BUCKET: "mattermost-cypress-report" BRANCH: ${{ github.ref_name }} BUILD_TAG: ${{ github.sha }} MM_TEST_USER_NAME: ${{ inputs.MM_TEST_USER_NAME || secrets.MM_DESKTOP_E2E_USER_NAME }} MM_TEST_PASSWORD: ${{ inputs.MM_TEST_PASSWORD || secrets.MM_DESKTOP_E2E_USER_CREDENTIALS }} - AWS_ACCESS_KEY_ID: ${{ secrets.MM_DESKTOP_E2E_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.MM_DESKTOP_E2E_AWS_SECRET_ACCESS_KEY }} - AWS_REGION: "us-east-1" PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1 NODE_ENV: "test" DEBUG_E2E: "true" @@ -265,8 +321,6 @@ jobs: with: node-version: '22.x' package-manager-cache: false - # node_modules is cached in e2e/cache-node-modules; disable setup-node's - # npm cache so actions/cache v5 does not hit legacy entries. - name: e2e/use-gnu-tar-macos if: runner.os == 'macOS' @@ -302,16 +356,9 @@ jobs: - name: e2e/setup-python uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 env: - # Bypass pip's on-disk HTTP cache entirely so a stale/corrupt entry on the - # runner image cannot trigger deserialization warnings (actions/setup-python#1317). PIP_NO_CACHE_DIR: "1" with: - # Use a version pre-installed on macOS-26 (3.13 is in its toolcache), so - # setup-python doesn't install a mismatched interpreter whose pip can't - # deserialize the image's pre-baked cache. PIP_NO_CACHE_DIR above is the - # remaining guard for non-macOS runners. python-version: "3.13" - # Omit `cache` — setup-python only accepts pip/pipenv/poetry, not `false`. - name: e2e/install-os-dependencies uses: ./.github/actions/install-os-dependencies @@ -360,43 +407,38 @@ jobs: SERVER_VERSION: ${{ inputs.MM_SERVER_VERSION }} DESKTOP_VERSION: ${{ inputs.version_name }} - - name: e2e/check-for-failures - id: check-failures - if: always() - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - script: | - process.chdir('./e2e'); - const { analyzeFlakyTests } = require('./utils/analyze-flaky-test.js'); - const { formatStatusDescription } = require('./utils/github-actions.js'); - const { newFailedTests, failureCount, passCount, skipCount, totalCount, collectionFailed } = analyzeFlakyTests(); - - const runUrl = `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`; - const platform = process.platform === 'darwin' ? 'macos' : 'windows'; - const hasFailed = newFailedTests && newFailedTests.length > 0; - const description = formatStatusDescription({ - passed: passCount, - failed: failureCount, - collectionFailed, - }); + # Joins the same TSIO report group as e2e-tests — gh-job-name must match + # this job's rendered name (policy-tests-) above. + - name: e2e/write-tsio-failure-stub + if: ${{ always() && needs.prepare-matrix.outputs.tsio-composite-identity != '' && hashFiles('e2e/test-results/results.json') == '' && (failure() || cancelled() || (env.PLAYWRIGHT_EXIT_CODE != '0' && env.PLAYWRIGHT_EXIT_CODE != '')) }} + run: node e2e/utils/write-tsio-failure-stub.mjs + env: + TSIO_GH_JOB_NAME: policy-tests-${{ matrix.platform }} + PLAYWRIGHT_EXIT_CODE: ${{ env.PLAYWRIGHT_EXIT_CODE }} - try { - await github.rest.repos.createCommitStatus({ - owner: context.repo.owner, - repo: context.repo.repo, - sha: context.sha, - state: hasFailed ? 'failure' : 'success', - context: `policy-test/${platform}`, - description, - target_url: runUrl, - }); - } catch (e) { - console.log(`Could not update commit status: ${e.message}`); - } + - name: e2e/upload-report-to-tsio + if: ${{ always() && needs.prepare-matrix.outputs.tsio-composite-identity != '' && hashFiles('e2e/test-results/results.json') != '' }} + uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-report-upload@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 0.10.0 / 2026-05-16 + with: + composite-identity: ${{ needs.prepare-matrix.outputs.tsio-composite-identity }} + total-reports-expected: ${{ needs.prepare-matrix.outputs.tsio-total-reports-expected }} + framework: playwright + github-token: ${{ secrets.GITHUB_TOKEN }} + gh-job-name: policy-tests-${{ matrix.platform }} + json-path: e2e/test-results/results.json + screenshots-dir: e2e/test-results - if (hasFailed) { - core.setFailed(`${newFailedTests.length} policy test(s) failed:\n${newFailedTests.join('\n')}`); - } + - name: e2e/handle-nonzero-playwright-exit + if: always() + run: | + if [ -z "${PLAYWRIGHT_EXIT_CODE:-}" ]; then + exit 0 + fi + if [ "${PLAYWRIGHT_EXIT_CODE}" = "0" ]; then + exit 0 + fi + echo "Playwright exited with code ${PLAYWRIGHT_EXIT_CODE}" >&2 + exit 1 - name: Upload test results if: always() diff --git a/.github/workflows/e2e-label-cleanup.yml b/.github/workflows/e2e-label-cleanup.yml index c0d71cc143a..12235b4c142 100644 --- a/.github/workflows/e2e-label-cleanup.yml +++ b/.github/workflows/e2e-label-cleanup.yml @@ -19,7 +19,7 @@ permissions: jobs: remove-e2e-label: name: Remove E2E/Run label from PR - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 if: ${{ github.event.workflow_run.event == 'workflow_dispatch' }} steps: - name: Checkout code diff --git a/.github/workflows/e2e-pr-trigger.yml b/.github/workflows/e2e-pr-trigger.yml index c484573332c..ba0134c921a 100644 --- a/.github/workflows/e2e-pr-trigger.yml +++ b/.github/workflows/e2e-pr-trigger.yml @@ -1,21 +1,9 @@ name: E2E PR Trigger -# Automatically adds the E2E/Run label to non-draft PRs targeting master. -# Adding the label signals Matterwick to provision cloud servers and dispatch -# the Electron Playwright Tests (e2e-functional.yml) workflow. -# After tests complete, e2e-functional.yml / e2e-label-cleanup.yml remove the label. -# -# E2E/Override (same contract as mattermost-mobile): when present on a PR, -# opened/synchronize events do not add E2E/Run, and applying the override label -# strips E2E/Run and cancels in-flight E2E runs. -# -# On synchronize: in-progress Electron Playwright Tests runs for this PR are -# cancelled before re-adding the label so stale runs for the old commit do not -# block the new one. Runs on other PR branches are left running. -# -# The concurrency group ensures rapid pushes to the same PR don't queue multiple -# label operations: only the most recent push proceeds. - +# Adds E2E/Run on non-draft PRs to master (Matterwick dispatches e2e-functional.yml). +# E2E/Override skips/cancels E2E; manual E2E/Run removal cancels in-flight runs. +# One job keeps the PR checks list to a single row. Concurrency is keyed by +# event type so unrelated label bots cannot cancel an in-progress trigger (PR #3891). on: pull_request: types: @@ -29,35 +17,95 @@ on: - master concurrency: - group: e2e-pr-trigger-${{ github.event.pull_request.number }} + group: >- + e2e-pr-trigger-${{ github.event.pull_request.number }}-${{ + contains(fromJSON('["opened", "reopened", "ready_for_review", "synchronize"]'), github.event.action) + && 'trigger' || format('other-{0}-{1}', github.event.action, github.event.label.name) }} cancel-in-progress: true jobs: - add-e2e-label: - name: Add E2E/Run label - runs-on: ubuntu-22.04 + e2e-label-orchestration: + name: E2E label + runs-on: ubuntu-24.04 permissions: issues: write pull-requests: write actions: write statuses: write if: >- - !github.event.pull_request.draft - && contains(fromJSON('["opened", "reopened", "ready_for_review", "synchronize"]'), github.event.action) + ( + !github.event.pull_request.draft + && contains(fromJSON('["opened", "reopened", "ready_for_review", "synchronize"]'), github.event.action) + ) + || ( + github.event.action == 'labeled' + && github.event.label.name == 'E2E/Override' + ) + || ( + github.event.action == 'unlabeled' + && github.event.label.name == 'E2E/Run' + && github.event.sender.login != 'github-actions[bot]' + ) steps: + # Always the base ref's own reviewed copy — this job holds write-scoped + # tokens, so it must never execute code from the untrusted PR head. - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ github.event.pull_request.base.ref }} - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - if: github.event.pull_request.head.repo.full_name == github.repository + - name: Cancel E2E on manual label removal + if: >- + github.event.action == 'unlabeled' + && github.event.label.name == 'E2E/Run' + && github.event.sender.login != 'github-actions[bot]' + uses: ./.github/actions/cancel-e2e-runs with: - ref: ${{ github.event.pull_request.head.sha }} - sparse-checkout: | - e2e/utils/github-actions.js - sparse-checkout-cone-mode: false + pr_number: ${{ github.event.pull_request.number }} + reason: E2E cancelled (E2E/Run label removed) - - name: Cancel running E2E tests and re-trigger + - name: Honor E2E/Override + if: >- + github.event.action == 'labeled' + && github.event.label.name == 'E2E/Override' + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + github-token: ${{ github.token }} + script: | + const {cancelActiveE2ERuns, markE2EStatusesCancelled} = require('./e2e/utils/github-actions.js'); + const { owner, repo } = context.repo; + const issue_number = context.issue.number; + const pr = context.payload.pull_request; + + try { + await github.rest.issues.removeLabel({ + owner, + repo, + issue_number, + name: 'E2E/Run', + }); + } catch (error) { + if (error.status !== 404) { + throw error; + } + } + + await cancelActiveE2ERuns({ + github, + context, + prNumber: issue_number, + headBranch: pr.head.ref, + }); + await markE2EStatusesCancelled({ + github, + context, + sha: pr.head.sha, + reason: 'E2E cancelled (E2E/Override label applied)', + }); + + - name: Refresh E2E/Run label + if: >- + !github.event.pull_request.draft + && contains(fromJSON('["opened", "reopened", "ready_for_review", "synchronize"]'), github.event.action) uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: github-token: ${{ github.token }} @@ -133,93 +181,3 @@ jobs: issue_number, labels: ['E2E/Run'], }); - - honor-e2e-override: - name: Honor E2E/Override - runs-on: ubuntu-22.04 - permissions: - issues: write - pull-requests: write - actions: write - statuses: write - if: >- - github.event.action == 'labeled' - && github.event.label.name == 'E2E/Override' - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - ref: ${{ github.event.pull_request.base.ref }} - - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - if: github.event.pull_request.head.repo.full_name == github.repository - with: - ref: ${{ github.event.pull_request.head.sha }} - sparse-checkout: | - e2e/utils/github-actions.js - sparse-checkout-cone-mode: false - - - name: Strip E2E/Run and cancel in-flight E2E - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - github-token: ${{ github.token }} - script: | - const {cancelActiveE2ERuns, markE2EStatusesCancelled} = require('./e2e/utils/github-actions.js'); - const { owner, repo } = context.repo; - const issue_number = context.issue.number; - const pr = context.payload.pull_request; - - try { - await github.rest.issues.removeLabel({ - owner, - repo, - issue_number, - name: 'E2E/Run', - }); - } catch (error) { - if (error.status !== 404) { - throw error; - } - } - - await cancelActiveE2ERuns({ - github, - context, - prNumber: issue_number, - headBranch: pr.head.ref, - }); - await markE2EStatusesCancelled({ - github, - context, - sha: pr.head.sha, - reason: 'E2E cancelled (E2E/Override label applied)', - }); - - cancel-on-manual-unlabel: - name: Cancel E2E on manual label removal - runs-on: ubuntu-22.04 - permissions: - actions: write - statuses: write - pull-requests: read - if: >- - github.event.action == 'unlabeled' - && github.event.label.name == 'E2E/Run' - && github.event.sender.login != 'github-actions[bot]' - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - ref: ${{ github.event.pull_request.base.ref }} - - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - if: github.event.pull_request.head.repo.full_name == github.repository - with: - ref: ${{ github.event.pull_request.head.sha }} - sparse-checkout: | - e2e/utils/github-actions.js - sparse-checkout-cone-mode: false - - - name: Cancel E2E runs and mark statuses skipped - uses: ./.github/actions/cancel-e2e-runs - with: - pr_number: ${{ github.event.pull_request.number }} - reason: E2E cancelled (E2E/Run label removed) diff --git a/e2e/fixtures/index.ts b/e2e/fixtures/index.ts index 9c936c01aa5..d04ab8d0ab5 100644 --- a/e2e/fixtures/index.ts +++ b/e2e/fixtures/index.ts @@ -66,7 +66,7 @@ export const test = base.extend({ await Promise.race([ cleanupRegisteredElectronProcesses(), new Promise((resolve) => { - timeoutHandle = setTimeout(resolve, 20_000); + timeoutHandle = setTimeout(resolve, 45_000); timeoutHandle.unref?.(); }), ]); diff --git a/e2e/helpers/channelReadiness.ts b/e2e/helpers/channelReadiness.ts index 847e5131b9b..7cc32e9a1ba 100644 --- a/e2e/helpers/channelReadiness.ts +++ b/e2e/helpers/channelReadiness.ts @@ -12,7 +12,7 @@ import { IS_CHANNEL_VIEW_LOADED_JS, IS_COMPOSER_INTERACTIVE_JS, } from './rendererUtils'; -import {activateServerView, loadServerViewUrl} from './serverContext'; +import {activateServerView, loadServerViewUrl, reloadServerView} from './serverContext'; import {resolveChannelByName} from './server_api/channel'; import type {ServerEntry} from './serverMap'; import type {ServerView} from './serverView'; @@ -139,7 +139,9 @@ export async function recoverServerViewIfNeeded( return; } - await win.runInRenderer('window.location.reload(); return true;', true); + // Prefer the Electron view reload path — window.location.reload() can leave + // the WebContentsView blank on macOS/Windows CI. + await reloadServerView(win.app, win.webContentsId); await waitForMattermostShell(win, {channelItem}); } diff --git a/e2e/helpers/electronApp.ts b/e2e/helpers/electronApp.ts index 0fdb6f1d04a..604809d75e1 100644 --- a/e2e/helpers/electronApp.ts +++ b/e2e/helpers/electronApp.ts @@ -146,6 +146,7 @@ export async function cleanupRegisteredElectronProcesses(): Promise { const pids = readPidsFromFile(file); fs.rmSync(file, {force: true}); await reapPids(pids); + await forceKillStrayElectronProcesses(); } /** @@ -160,6 +161,7 @@ export async function cleanupAllRegisteredElectronProcesses(): Promise { fs.rmSync(file, {force: true}); } await reapPids(pids); + await forceKillStrayElectronProcesses(); } /** @@ -275,6 +277,61 @@ async function drainPlaywrightClose(closePromise: Promise, remainingMs: nu await Promise.race([closePromise, sleep(remainingMs)]); } +async function requestElectronQuit(app: ElectronApplication): Promise { + try { + await Promise.race([ + app.evaluate(() => { + const {app: electronApp} = require('electron'); + electronApp.exit(0); + }), + sleep(2_000), + ]); + } catch { + // evaluation may fail if the app is already shutting down + } +} + +async function settleElectronClose(closePromise: Promise, pid: number | undefined, budgetMs: number): Promise { + const deadline = Date.now() + budgetMs; + while (Date.now() < deadline) { + if (await isClosePromiseSettled(closePromise)) { + return; + } + + // Keep draining until Playwright observes a dead process and rejects close(). + // Returning early when the PID is gone but close() is still pending leaves a + // gracefullyClose entry that burns the full workerTeardownTimeout (#29431). + await drainPlaywrightClose(closePromise, Math.min(2_000, deadline - Date.now())); + if (pid && !isProcessAlive(pid)) { + await forceKillStrayElectronProcesses(); + } + await sleep(200); + } +} + +async function isClosePromiseSettled(closePromise: Promise): Promise { + const result = await Promise.race([ + closePromise.then(() => 'settled' as const, () => 'settled' as const), + sleep(50).then(() => 'pending' as const), + ]); + return result === 'settled'; +} + +/** + * Linux CI can leave Mattermost Electron processes running when app.close() never + * settles. Reap by executable path as a worker/global backstop after registry cleanup. + */ +export async function forceKillStrayElectronProcesses(): Promise { + if (process.platform !== 'linux') { + return; + } + try { + execFileSync('pkill', ['-KILL', '-f', 'node_modules/electron/dist/electron'], {stdio: 'ignore'}); + } catch { + // no matching processes + } +} + async function forceShutdownLinux(pid: number): Promise { if (!isProcessAlive(pid)) { return; @@ -306,7 +363,12 @@ function signalShutdownAndReturn(pid: number): void { } } -async function attemptClose(app: ElectronApplication, timeoutMs: number): Promise { +type AttemptCloseResult = { + closed: boolean; + closePromise: Promise; +}; + +async function attemptClose(app: ElectronApplication, timeoutMs: number): Promise { let closed = false; // Only mark closed on successful resolve; a rejected close() must leave @@ -316,8 +378,11 @@ async function attemptClose(app: ElectronApplication, timeoutMs: number): Promis }, () => {}); const start = Date.now(); await Promise.race([closePromise, sleep(timeoutMs)]); - await drainPlaywrightClose(closePromise, timeoutMs - (Date.now() - start)); - return closed; + const remainingMs = Math.max(0, timeoutMs - (Date.now() - start)); + + // Playwright keeps a gracefullyClose entry until close() settles (#29431). + await drainPlaywrightClose(closePromise, Math.max(remainingMs, 2_000)); + return {closed, closePromise}; } export async function waitForWindow(app: ElectronApplication, pattern: string, timeout = 30_000) { @@ -360,22 +425,37 @@ export async function closeElectronApp( // same userDataDir may be relaunched, so we force-kill on failure (Linux) // and always wait for the SingletonLock to release. const fastTeardown = Boolean(options.skipLockWaitUnlessCleanClose); + const closeTimeoutMs = process.platform === 'linux' && fastTeardown ? 5_000 : 10_000; - const cleanClosed = await attemptClose(app, 10_000); + if (process.platform === 'linux') { + await requestElectronQuit(app); + } - if (!cleanClosed && pid) { - if (process.platform === 'linux') { - // Always SIGKILL stuck trees on Linux so worker teardown does not sit - // in Playwright's 90s gracefullyClose wait with live Electron PIDs. - await forceShutdownLinux(pid); - } else { - signalShutdownAndReturn(pid); + const {closed: cleanClosed, closePromise} = await attemptClose(app, closeTimeoutMs); + + if (!cleanClosed) { + if (pid) { + if (process.platform === 'linux') { + // Always SIGKILL stuck trees on Linux so worker teardown does not sit + // in Playwright's 90s gracefullyClose wait with live Electron PIDs. + await forceShutdownLinux(pid); + } else { + signalShutdownAndReturn(pid); + } } + + // Killing the process does not settle app.close(); wait until Playwright + // drops the gracefullyClose entry before worker teardown. + const settleBudgetMs = process.platform === 'linux' ? 45_000 : 20_000; + await settleElectronClose(closePromise, pid, settleBudgetMs); } - // Fast path on a failed close: return immediately (master-style). The lock + // Fast path on a failed close: return once close() has settled. The lock // lives in an abandoned dir and worker/global cleanup reaps any live PID. if (fastTeardown && !cleanClosed) { + if (!(await isClosePromiseSettled(closePromise))) { + await settleElectronClose(closePromise, pid, process.platform === 'linux' ? 30_000 : 15_000); + } if (pid && !isProcessAlive(pid)) { unregisterElectronMainProcess(pid); } diff --git a/e2e/helpers/ipcChannels.ts b/e2e/helpers/ipcChannels.ts index bab9c9199e4..5d299ec68f4 100644 --- a/e2e/helpers/ipcChannels.ts +++ b/e2e/helpers/ipcChannels.ts @@ -8,3 +8,4 @@ export const CLOSE_DOWNLOADS_DROPDOWN_MENU = 'close-downloads-dropdown-menu'; export const CALLS_LEAVE_CALL = 'calls-leave-call'; export const EMIT_CONFIGURATION = 'emit-configuration'; export const SHOW_NEW_SERVER_MODAL = 'show_new_server_modal'; +export const NOTIFICATION_CLICKED = 'notification-clicked'; diff --git a/e2e/helpers/login.ts b/e2e/helpers/login.ts index 5ce050e66c7..bfa8426b65b 100644 --- a/e2e/helpers/login.ts +++ b/e2e/helpers/login.ts @@ -5,6 +5,7 @@ import {expect} from '@playwright/test'; import {isTransientEvaluateError} from './testRefs'; import {CHANNEL_HEADER_SELECTORS, POST_TEXTBOX_SELECTOR} from './rendererUtils'; +import {reloadServerView} from './serverContext'; import type {ServerView} from './serverView'; async function isMattermostServerUrl(win: ServerView): Promise { @@ -64,7 +65,11 @@ export async function loginToMattermost(win: ServerView): Promise { const loginSelector = '#input_loginId'; const passwordSelector = '#input_password-input, input[type="password"]'; const submitSelector = '#saveSetting, button[type="submit"]'; + const pollStart = Date.now(); + const reloadAt = pollStart + Math.min(timeout / 2, 15_000); + let reloaded = false; + // Cold cloud hosts often paint a blank hex shell; one mid-wait reload unsticks it. await expect.poll(async () => { if (await hasAppShell(win)) { return 'logged-in'; @@ -72,6 +77,10 @@ export async function loginToMattermost(win: ServerView): Promise { if (await hasLoginForm(win)) { return 'login-form'; } + if (!reloaded && Date.now() >= reloadAt) { + reloaded = true; + await reloadServerView(win.app, win.webContentsId).catch(() => undefined); + } return 'loading'; }, { timeout, diff --git a/e2e/helpers/methodSpy.ts b/e2e/helpers/methodSpy.ts index 0e314e0120a..865b5d941fd 100644 --- a/e2e/helpers/methodSpy.ts +++ b/e2e/helpers/methodSpy.ts @@ -66,3 +66,7 @@ export async function restoreFlashFrameSpy(app: ElectronApplication): Promise { + return evaluateInMainProcess(app, () => (global as any).__e2eFlashFrameCalls ?? []); +} diff --git a/e2e/helpers/notificationEffects.ts b/e2e/helpers/notificationEffects.ts index 1d0665cdc74..7b4629a9fb8 100644 --- a/e2e/helpers/notificationEffects.ts +++ b/e2e/helpers/notificationEffects.ts @@ -3,6 +3,8 @@ import type {ElectronApplication} from 'playwright'; +import {evaluateInMainProcessWithArg} from './testRefs'; + /** * Invoke the E2E mirror of notifications/index.ts flashFrame(). * @@ -11,7 +13,7 @@ import type {ElectronApplication} from 'playwright'; * exercise the same flashFrame() gate the notification `show` handler calls. */ export async function triggerNotificationEffects(app: ElectronApplication, flash = true): Promise { - await app.evaluate((_, shouldFlash: boolean) => { + await evaluateInMainProcessWithArg(app, (_electron, shouldFlash: boolean) => { const trigger = (global as any).__e2eNotificationEffects as ((value: boolean) => void) | undefined; if (!trigger) { throw new Error('__e2eNotificationEffects not exposed (NODE_ENV must be test)'); diff --git a/e2e/helpers/serverContext.ts b/e2e/helpers/serverContext.ts index abde6819864..cf5509a9de5 100644 --- a/e2e/helpers/serverContext.ts +++ b/e2e/helpers/serverContext.ts @@ -9,6 +9,8 @@ import {closeOverlayWindowsIfOpen} from './overlayWindows'; import type {ServerEntry, ServerMap} from './serverMap'; import {evaluateInMainProcessWithArg} from './testRefs'; +const NOT_YET_REGISTERED_PREFIX = 'No server view registered for webContentsId'; + /** * Make a specific server WebContentsView the active, focused target for automation * and application menu handlers (History, View, Find, etc.). @@ -23,40 +25,60 @@ export async function activateServerView( ): Promise { await closeOverlayWindowsIfOpen(app); - await evaluateInMainProcessWithArg(app, ({webContents, BrowserWindow}, id) => { - const refs = (global as any).__e2eTestRefs; - if (!refs) { - throw new Error('__e2eTestRefs is not available'); - } - - const mmView = refs.WebContentsManager.getViewByWebContentsId(id); - if (!mmView) { - throw new Error(`No server view registered for webContentsId ${id}`); - } - - const tabView = refs.ViewManager.getView(mmView.id); - if (tabView) { - refs.ServerManager.updateCurrentServer(tabView.serverId); - refs.TabManager.switchToTab(tabView.id); - } - - refs.TabManager.focusCurrentTab(); - - const wc = webContents.fromId(id); - if (!wc || wc.isDestroyed()) { - throw new Error(`webContents ${id} is not available`); + // WebContentsManager keeps two separate indexes: by internal view id (what + // buildServerMap reads to resolve this webContentsId in the first place) + // and by webContentsId (what this function needs). The second index can + // still be a beat behind the first right after a tab is created, so a + // caller that just got this id from buildServerMap can hit a real but + // transient "not registered yet" here. Poll specifically for that error; + // any other failure (e.g. the webContents was actually destroyed) still + // fails immediately. + const deadline = Date.now() + 10_000; + for (;;) { + try { + await evaluateInMainProcessWithArg(app, ({webContents, BrowserWindow}, id) => { + const refs = (global as any).__e2eTestRefs; + if (!refs) { + throw new Error('__e2eTestRefs is not available'); + } + + const mmView = refs.WebContentsManager.getViewByWebContentsId(id); + if (!mmView) { + throw new Error(`No server view registered for webContentsId ${id}`); + } + + const tabView = refs.ViewManager.getView(mmView.id); + if (tabView) { + refs.ServerManager.updateCurrentServer(tabView.serverId); + refs.TabManager.switchToTab(tabView.id); + } + + refs.TabManager.focusCurrentTab(); + + const wc = webContents.fromId(id); + if (!wc || wc.isDestroyed()) { + throw new Error(`webContents ${id} is not available`); + } + wc.focus(); + + // Menu handlers read this when the app menu blurs the webContents (macOS/Windows). + refs.WebContentsManager.focusedWebContentsView = mmView.id; + + const mainWindow = refs.MainWindow.get() ?? BrowserWindow.getAllWindows().find((win) => { + return !win.isDestroyed() && win.webContents.getURL().includes('index'); + }); + mainWindow?.show(); + mainWindow?.focus(); + }, webContentsId); + break; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (!message.includes(NOT_YET_REGISTERED_PREFIX) || Date.now() >= deadline) { + throw error; + } + await new Promise((resolve) => setTimeout(resolve, 100)); } - wc.focus(); - - // Menu handlers read this when the app menu blurs the webContents (macOS/Windows). - refs.WebContentsManager.focusedWebContentsView = mmView.id; - - const mainWindow = refs.MainWindow.get() ?? BrowserWindow.getAllWindows().find((win) => { - return !win.isDestroyed() && win.webContents.getURL().includes('index'); - }); - mainWindow?.show(); - mainWindow?.focus(); - }, webContentsId); + } const mainWindow = findMainWindow(app); if (mainWindow) { diff --git a/e2e/helpers/settingsWindow.ts b/e2e/helpers/settingsWindow.ts index 625e2e839ad..3be24cb4923 100644 --- a/e2e/helpers/settingsWindow.ts +++ b/e2e/helpers/settingsWindow.ts @@ -6,9 +6,32 @@ import type {ElectronApplication, Page} from 'playwright'; import {SHOW_SETTINGS_WINDOW} from './ipcChannels'; import {evaluateInMainProcessWithArg} from './testRefs'; +function findSettingsPage(app: ElectronApplication): Page | undefined { + return app.windows().find((window) => { + try { + return window.url().includes('settings'); + } catch { + return false; + } + }); +} + +async function waitForSettingsPage(app: ElectronApplication, timeoutMs = 15_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const settingsWindow = findSettingsPage(app); + if (settingsWindow) { + await settingsWindow.waitForLoadState().catch(() => {}); + return settingsWindow; + } + await new Promise((resolve) => setTimeout(resolve, 200)); + } + throw new Error('Settings window did not open'); +} + export async function openSettingsWindow(electronApp: ElectronApplication): Promise { for (let attempt = 0; attempt < 5; attempt++) { - const existingWindow = electronApp.windows().find((window) => window.url().includes('settings')); + const existingWindow = findSettingsPage(electronApp); if (existingWindow) { try { await existingWindow.waitForLoadState(); @@ -22,22 +45,12 @@ export async function openSettingsWindow(electronApp: ElectronApplication): Prom } } - // Route through evaluateInMainProcessWithArg to reuse its transient - // "Execution context was destroyed" retry behavior instead of - // duplicating the try/catch loop here. await evaluateInMainProcessWithArg(electronApp, ({ipcMain}, showWindow) => { ipcMain.emit(showWindow); }, SHOW_SETTINGS_WINDOW); try { - const settingsWindow = electronApp.windows().find((window) => window.url().includes('settings')) ?? - await electronApp.waitForEvent('window', { - predicate: (window) => window.url().includes('settings'), - timeout: 3_000, - }); - - await settingsWindow.waitForLoadState(); - return settingsWindow; + return await waitForSettingsPage(electronApp); } catch (error) { if (attempt === 4) { throw error; @@ -48,3 +61,10 @@ export async function openSettingsWindow(electronApp: ElectronApplication): Prom throw new Error('Settings window did not open'); } + +export async function waitForSettingsModal( + app: ElectronApplication, + options?: {timeout?: number}, +): Promise { + return waitForSettingsPage(app, options?.timeout ?? 15_000); +} diff --git a/e2e/helpers/trayMenu.ts b/e2e/helpers/trayMenu.ts index 4409bc930cf..371f9c5c753 100644 --- a/e2e/helpers/trayMenu.ts +++ b/e2e/helpers/trayMenu.ts @@ -1,43 +1,21 @@ // Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. -import {expect} from '@playwright/test'; -import type {ElectronApplication, Page} from 'playwright'; +import type {ElectronApplication} from 'playwright'; +import {waitForSettingsModal} from './settingsWindow'; import {clickTrayMenuItem} from './tray'; -async function findSettingsPage(app: ElectronApplication): Promise { - return app.windows().find((window) => { - try { - return window.url().includes('settings'); - } catch { - return false; - } - }) ?? null; -} - -async function waitForSettingsPage(app: ElectronApplication): Promise { - let settingsWindow: Page | null = null; - await expect.poll(async () => { - settingsWindow = await findSettingsPage(app); - return settingsWindow; - }, {timeout: 15_000, message: 'Settings page must open after tray menu click'}).not.toBeNull(); - await settingsWindow!.waitForLoadState(); - return settingsWindow!; -} - export function traySettingsMenuLabel(): string { return process.platform === 'darwin' ? 'Preferences...' : 'Settings'; } -export async function openSettingsFromTray(app: ElectronApplication): Promise { - const existingSettings = await findSettingsPage(app); +export async function openSettingsFromTray(app: ElectronApplication) { + const existingSettings = await waitForSettingsModal(app, {timeout: 1_000}).catch(() => null); if (existingSettings) { - await existingSettings.waitForLoadState(); return existingSettings; } - // Semantic click avoids i18n / mnemonic label mismatches ("Settings" vs "Settings..."). try { await clickTrayMenuItem(app, 'tray:settings'); } catch { @@ -62,7 +40,7 @@ export async function openSettingsFromTray(app: ElectronApplication): Promise { diff --git a/e2e/helpers/userAttributes.ts b/e2e/helpers/userAttributes.ts index 7c967493d19..92a0c6c45b5 100644 --- a/e2e/helpers/userAttributes.ts +++ b/e2e/helpers/userAttributes.ts @@ -47,6 +47,34 @@ export type CustomProfileAttributeDef = { const CPA_FIELDS_PATH = '/api/v4/custom_profile_attributes/fields'; +/** + * Renderer-side JS expression resolving the active custom-attribute edit + * section. Inputs and the Save/Cancel row are sibling `.setting-list-item` + * nodes under `.setting-list`, so scoping to the input's list-item misses + * `#saveSetting`. + */ +function customAttributeEditScopeJs(elExpr: string): string { + return `( + ${elExpr}?.closest('.setting-list') || + ${elExpr}?.closest('section.section-max') || + ${elExpr}?.closest('section') + )`; +} + +/** + * Renderer-side JS expression resolving a custom attribute row from an + * element within it (e.g. the Edit button in display mode). + */ +function customAttributeRowJs(elExpr: string): string { + return `( + ${elExpr}?.closest('.setting-list') || + ${elExpr}?.closest('.SettingsBlock') || + ${elExpr}?.closest('section') || + ${elExpr}?.closest('li') || + ${elExpr}?.closest('div') + )`; +} + export const TEST_PHONE = '555-123-4567'; export const TEST_UPDATED_PHONE = '555-987-6543'; export const TEST_URL = 'https://example.com'; @@ -240,6 +268,43 @@ export async function recoverFromProfileSettings(win: ServerView): Promise await waitForChannelPostListLoaded(win); } +async function isCustomAttributeEditVisible(win: ServerView, fieldId: string): Promise { + return win.runInRenderer(` + const btn = document.querySelector('#customAttribute_${fieldId}Edit'); + if (!(btn instanceof HTMLElement)) { + return false; + } + const rect = btn.getBoundingClientRect(); + return rect.width > 0 && rect.height > 0; + `); +} + +/** Webapp caches CPA field defs; reload once if API-created fields are missing from Profile Settings. */ +export async function waitForCustomAttributeEditInProfileSettings(win: ServerView, fieldId: string): Promise { + await ensureCustomAttributeEditReady(win, fieldId); +} + +async function ensureCustomAttributeEditReady(win: ServerView, fieldId: string): Promise { + for (let attempt = 0; attempt < 2; attempt++) { + try { + await expect.poll(async () => isCustomAttributeEditVisible(win, fieldId), { + timeout: attempt === 0 ? 5_000 : 15_000, + message: `Custom attribute Edit button #customAttribute_${fieldId}Edit must be visible`, + }).toBe(true); + return; + } catch (error) { + if (attempt === 1) { + throw error; + } + await closeProfileSettings(win).catch(() => undefined); + await reloadServerView(win.app, win.webContentsId); + await waitForMattermostShellReady(win); + await dismissBlockingOverlays(win); + await openProfileSettings(win); + } + } +} + export async function getCustomAttributeLabelsInSettings(win: ServerView): Promise { return win.runInRenderer(` const modal = document.querySelector(${JSON.stringify(PROFILE_SETTINGS_MODAL_SELECTOR)}) @@ -260,7 +325,7 @@ export async function getCustomAttributeLabelsInSettings(win: ServerView): Promi labels.push(nameEl.textContent.trim()); continue; } - const row = button.closest('.setting-list-item, .SettingsBlock, section, li, div'); + const row = ${customAttributeRowJs('button')}; const rowText = (row?.textContent || '') .replace(/Edit.*$/s, '') .replace(/Click 'Edit' to add your custom attribute/gi, '') @@ -279,11 +344,15 @@ export async function editTextCustomAttribute( newValue: string, save = true, ): Promise { + await ensureCustomAttributeEditReady(win, fieldId); await win.runInRenderer(` const fieldId = ${JSON.stringify(fieldId)}; const editBtn = document.querySelector('#customAttribute_' + fieldId + 'Edit'); - editBtn?.scrollIntoView({block: 'center'}); - editBtn?.click(); + if (!(editBtn instanceof HTMLElement)) { + throw new Error('Custom attribute Edit button not found for ' + fieldId); + } + editBtn.scrollIntoView({block: 'center'}); + editBtn.click(); `); await win.waitForSelector(`#customAttribute_${fieldId}`, {timeout: 10_000}); await win.runInRenderer(` @@ -302,14 +371,18 @@ export async function editTextCustomAttribute( await win.fill(`#customAttribute_${fieldId}`, newValue); } if (save) { + await win.waitForSelector('#saveSetting', {timeout: 10_000}); + await expect.poll(async () => win.runInRenderer(` + const saveBtn = document.querySelector('#saveSetting'); + return saveBtn instanceof HTMLButtonElement && !saveBtn.disabled; + `), {timeout: 10_000, message: 'Save button must be enabled before saving custom attribute'}).toBe(true); await win.runInRenderer(` const fieldId = ${JSON.stringify(fieldId)}; const input = document.querySelector('#customAttribute_' + fieldId); - const row = input?.closest('.setting-list-item, .SettingsBlock, section, li, div') || document; - const saveBtn = Array.from(row.querySelectorAll('button')) - .find((button) => (button.textContent || '').trim() === 'Save'); + const scope = ${customAttributeEditScopeJs('input')} || document; + const saveBtn = scope.querySelector('#saveSetting') || document.querySelector('#saveSetting'); if (!(saveBtn instanceof HTMLButtonElement)) { - throw new Error('Save button not found for custom attribute row'); + throw new Error('Save button not found for custom attribute edit section'); } saveBtn.click(); `); @@ -342,7 +415,7 @@ export async function getCustomAttributeInputValue(win: ServerView, fieldId: str if (!editBtn) { return input instanceof HTMLInputElement ? input.value : ''; } - const row = editBtn.closest('.setting-list-item, .SettingsBlock, section, li, div'); + const row = ${customAttributeRowJs('editBtn')}; if (!row) { return ''; } diff --git a/e2e/playwright.config.ts b/e2e/playwright.config.ts index 5e7b449a5ee..9f814721cb9 100644 --- a/e2e/playwright.config.ts +++ b/e2e/playwright.config.ts @@ -28,7 +28,16 @@ const PLATFORM_GREP: Record = { // Electron processes are heavy (~300MB each), so cap at 2 in CI and half the CPU // count locally (max 4). Override with E2E_WORKERS env var. const cpuCount = os.cpus().length; -const defaultWorkers = process.env.CI ? 2 : Math.min(4, Math.max(1, Math.floor(cpuCount / 2))); + +// Linux CI hits Playwright worker-teardown hangs with 2 parallel Electron workers; +// one worker can finish with a stuck app.close() and burn the full 90s budget. +function getDefaultWorkers(): number { + if (process.env.CI) { + return getActivePlatform() === 'linux' ? 1 : 2; + } + return Math.min(4, Math.max(1, Math.floor(cpuCount / 2))); +} +const defaultWorkers = getDefaultWorkers(); const parsedWorkers = process.env.E2E_WORKERS ? Number.parseInt(process.env.E2E_WORKERS, 10) : NaN; const workers = Number.isFinite(parsedWorkers) && parsedWorkers > 0 ? parsedWorkers : defaultWorkers; @@ -80,7 +89,10 @@ function buildPlatformProjects(): Project[] { const reporters = process.env.CI ? [ ['blob', {outputDir: 'blob-report'}], ['line'], - ['junit', {outputFile: 'test-results/e2e-junit.xml'}], + + // Native Playwright JSON — required by test-system-io-report-upload's + // `framework: playwright` parser (reads suites[].specs[].tests[].results[]). + ['json', {outputFile: 'test-results/results.json'}], ] as const : [ ['html', {open: 'never', outputFolder: 'playwright-report'}], ['list'], diff --git a/e2e/specs/calls/calls_functionality.test.ts b/e2e/specs/calls/calls_functionality.test.ts index 4548c3e691b..17fa6d1d128 100644 --- a/e2e/specs/calls/calls_functionality.test.ts +++ b/e2e/specs/calls/calls_functionality.test.ts @@ -6,6 +6,7 @@ import type {ElectronApplication} from 'playwright'; import {test, expect} from '../../fixtures/index'; import {findCallsWidgetWindow, waitForCallsWidgetWindow} from '../../helpers/callsWidget'; +import {waitForMattermostShellReady} from '../../helpers/channelReadiness'; import {demoMattermostConfig} from '../../helpers/config'; import {loginToMattermost} from '../../helpers/login'; import type {ServerView} from '../../helpers/serverView'; @@ -65,6 +66,14 @@ test.describe('calls/calls_functionality', () => { serverWin = serverEntry!.win; await loginToMattermost(serverWin); + + // loginToMattermost only waits for the generic app shell (post + // textbox / channel header / search bar) — the sidebar's channel + // list can still be hydrating a beat later, so clicking + // #sidebarItem_town-square immediately here raced it intermittently + // ("Element not found for click"). Same wait media_preview.test.ts + // already uses before the identical click. + await waitForMattermostShellReady(serverWin, {channelItem: '#sidebarItem_town-square'}); await serverWin.click('#sidebarItem_town-square'); await serverWin.waitForSelector('#channelHeaderTitle', {timeout: 10_000}); }); diff --git a/e2e/specs/focus.test.ts b/e2e/specs/focus.test.ts index d45f3ea3c4d..6e25ac64455 100644 --- a/e2e/specs/focus.test.ts +++ b/e2e/specs/focus.test.ts @@ -8,7 +8,7 @@ import * as path from 'path'; import {test, expect} from '../fixtures/index'; import {waitForAppReady} from '../helpers/appReadiness'; import {electronBinaryPath, appDir, demoMattermostConfig, writeConfigFile} from '../helpers/config'; -import {closeElectronAppFast} from '../helpers/electronApp'; +import {closeElectronAppFast, registerElectronMainProcess} from '../helpers/electronApp'; import {SHOW_NEW_SERVER_MODAL, SHOW_SETTINGS_WINDOW} from '../helpers/ipcChannels'; import {loginToMattermost} from '../helpers/login'; import {buildServerMap, type ServerMap} from '../helpers/serverMap'; @@ -129,6 +129,7 @@ test.describe('focus', () => { env: {...process.env, NODE_ENV: 'test'}, timeout: 60_000, }); + registerElectronMainProcess(electronApp.process()?.pid); await waitForAppReady(electronApp); // Poll until both servers are registered in WebContentsManager diff --git a/e2e/specs/menu_bar/devtools_current_server.test.ts b/e2e/specs/menu_bar/devtools_current_server.test.ts index fbbc219d931..0926ca329f7 100644 --- a/e2e/specs/menu_bar/devtools_current_server.test.ts +++ b/e2e/specs/menu_bar/devtools_current_server.test.ts @@ -60,38 +60,20 @@ test.describe('menu_bar/devtools_current_server', () => { {timeout: 15_000, message: 'DevTools must open for the current server webContents after menu click'}, ).toBe(true); - // MattermostWebContentsView.openDevTools() runs a 500ms macOS reset and documents - // that isDevToolsOpened() may not reflect close — use closeDevTools() and assert - // the server view is usable instead of polling isDevToolsOpened() on darwin. + // MattermostWebContentsView.openDevTools() uses detach mode. toggleDevTools() does not + // reliably close detached DevTools on Linux/Windows, and isDevToolsOpened() can lie on + // macOS — close explicitly and assert the server view stays usable. if (process.platform === 'darwin') { await new Promise((resolve) => setTimeout(resolve, 750)); - await evaluateInMainProcessWithArg(electronApp, ({webContents}, id) => { - const wc = webContents.fromId(id); - if (wc && !wc.isDestroyed() && wc.isDevToolsOpened()) { - wc.closeDevTools(); - } - }, webContentsId); - } else { - await evaluateInMainProcessWithArg(electronApp, ({webContents}, id) => { - try { - const wc = webContents.fromId(id); - if (wc && !wc.isDestroyed() && wc.isDevToolsOpened()) { - wc.toggleDevTools(); - } - } catch { - // DevTools may already be detaching. - } - }, webContentsId).catch(() => {}); - await expect.poll( - () => evaluateInMainProcessWithArg(electronApp, ({webContents}, id) => { - const wc = webContents.fromId(id); - return wc && !wc.isDestroyed() ? !wc.isDevToolsOpened() : true; - }, webContentsId).catch(() => true), - {timeout: 15_000, message: 'DevTools must close after toggle'}, - ).toBe(true); } + await evaluateInMainProcessWithArg(electronApp, ({webContents}, id) => { + const wc = webContents.fromId(id); + if (wc && !wc.isDestroyed() && wc.isDevToolsOpened()) { + wc.closeDevTools(); + } + }, webContentsId); - // DevTools attach/detach can briefly invalidate Playwright's Electron context on macOS. + // DevTools attach/detach can briefly invalidate Playwright's Electron context. await prepareMattermostServerView(electronApp, webContentsId); await firstServer!.waitForSelector('#post_textbox', {timeout: 15_000}); }, diff --git a/e2e/specs/menu_bar/help_menu.test.ts b/e2e/specs/menu_bar/help_menu.test.ts index 31202263e6c..4422f618221 100644 --- a/e2e/specs/menu_bar/help_menu.test.ts +++ b/e2e/specs/menu_bar/help_menu.test.ts @@ -5,6 +5,7 @@ import {test, expect} from '../../fixtures/index'; import {waitForAppReady} from '../../helpers/appReadiness'; import {clickApplicationMenuItem} from '../../helpers/menu'; import {evaluateInMainProcess} from '../../helpers/testRefs'; +import {getShellOpenExternalCalls, restoreShellOpenExternal, stubShellOpenExternal} from '../../helpers/shell'; test.describe('menu_bar/help_menu', () => { test( @@ -87,7 +88,9 @@ test.describe('menu_bar/help_menu', () => { 'MM-T6151 Show logs menu item opens the log file location', {tag: ['@P1', '@all']}, async ({electronApp}) => { - await electronApp.evaluate(({shell}) => { + await waitForAppReady(electronApp); + + await evaluateInMainProcess(electronApp, ({shell}) => { (global as any).__e2eShownInFolder = [] as string[]; (global as any).__e2eOriginalShowItemInFolder = shell.showItemInFolder.bind(shell); shell.showItemInFolder = (fullPath: string) => { @@ -100,10 +103,12 @@ test.describe('menu_bar/help_menu', () => { await clickApplicationMenuItem(electronApp, 'help', {id: 'Show logs'}); await expect.poll(async () => { - return electronApp.evaluate(() => ((global as any).__e2eShownInFolder as string[] | undefined)?.length ?? 0); + return evaluateInMainProcess(electronApp, () => { + return ((global as any).__e2eShownInFolder as string[] | undefined)?.length ?? 0; + }); }, {timeout: 10_000}).toBeGreaterThan(0); } finally { - await electronApp.evaluate(({shell}) => { + await evaluateInMainProcess(electronApp, ({shell}) => { const original = (global as any).__e2eOriginalShowItemInFolder; if (original) { shell.showItemInFolder = original; @@ -114,4 +119,41 @@ test.describe('menu_bar/help_menu', () => { } }, ); + + test( + 'MM-T828 Learn More in the Menu Bar opens help documentation externally', + {tag: ['@P2', '@darwin']}, + async ({electronApp}) => { + await waitForAppReady(electronApp); + + const helpMenuItem = await evaluateInMainProcess(electronApp, ({app: electronAppInstance}) => { + const refs = (global as any).__e2eTestRefs; + const menu = electronAppInstance.applicationMenu?.getMenuItemById('help'); + const userGuideItem = menu?.submenu?.items?.find((item) => { + return typeof item.label === 'string' && + item.label.includes('User guide') && + typeof item.click === 'function'; + }); + return { + label: typeof userGuideItem?.label === 'string' ? userGuideItem.label : '', + helpLink: refs?.Config?.helpLink ?? '', + }; + }); + + expect(helpMenuItem.label, 'Help menu must expose a User guide item').toContain('User guide'); + expect(helpMenuItem.helpLink, 'Config.helpLink must be set').toContain('docs.mattermost.com'); + + await stubShellOpenExternal(electronApp); + try { + await clickApplicationMenuItem(electronApp, 'help', {label: helpMenuItem.label}); + + await expect.poll( + () => getShellOpenExternalCalls(electronApp), + {timeout: 10_000, message: 'Help > User guide must open documentation via shell.openExternal'}, + ).toContain(helpMenuItem.helpLink); + } finally { + await restoreShellOpenExternal(electronApp); + } + }, + ); }); diff --git a/e2e/specs/multi_window/multi_window.test.ts b/e2e/specs/multi_window/multi_window.test.ts index 4592aadcb2a..ae3a8cbf41d 100644 --- a/e2e/specs/multi_window/multi_window.test.ts +++ b/e2e/specs/multi_window/multi_window.test.ts @@ -12,6 +12,7 @@ import {demoMattermostConfig} from '../../helpers/config'; import {closeDownloadsDropdownIfOpen} from '../../helpers/downloadsDropdown'; import {launchDirectTestApp} from '../../helpers/directLaunch'; import {closeElectronAppFast, waitForWindow} from '../../helpers/electronApp'; +import {NOTIFICATION_CLICKED} from '../../helpers/ipcChannels'; import {loginToMattermost} from '../../helpers/login'; import {waitForMainWindowFocused} from '../../helpers/mainWindowFocus'; import {POST_TEXTBOX_SELECTOR, waitForChannelPostListLoaded, waitForMattermostShellReady} from '../../helpers/mattermostShell'; @@ -32,12 +33,12 @@ import {prepareMattermostServerView} from '../../helpers/prepareServerView'; import {resolvedChannelPath, resolveChannelByName} from '../../helpers/server_api/channel'; import {seedThreadInChannel} from '../../helpers/server_api/post'; import {buildServerMap} from '../../helpers/serverMap'; +import {evaluateInMainProcessWithArg} from '../../helpers/testRefs'; import { clickOpenInNewWindowFromRhsThreadMenu, clickOpenInNewWindowFromThreadsListMenu, clickOpenInNewWindowMenuItem, } from '../../helpers/webappMenu'; -import {NOTIFICATION_CLICKED} from '../../../src/common/communication'; const config = { ...demoMattermostConfig, @@ -469,11 +470,17 @@ test.describe('multi_window/multi_window', () => { const browserWindow = await electronApp.browserWindow(popoutWindow); const initialBounds = await browserWindow.evaluate((w) => (w as Electron.BrowserWindow).getBounds()); + const workArea = await evaluateInMainProcessWithArg( + electronApp, + (electron, bounds) => electron.screen.getDisplayMatching(bounds).workArea, + initialBounds, + ); + const margin = 20; const resizedBounds = { x: initialBounds.x, y: initialBounds.y, - width: initialBounds.width + 200, - height: initialBounds.height + 200, + width: Math.min(initialBounds.width + 200, (workArea.x + workArea.width) - initialBounds.x - margin), + height: Math.min(initialBounds.height + 200, (workArea.y + workArea.height) - initialBounds.y - margin), }; await browserWindow.evaluate((w, bounds) => { diff --git a/e2e/specs/notification_trigger/flash_taskbar.test.ts b/e2e/specs/notification_trigger/flash_taskbar.test.ts index 1e059500978..452968724f8 100644 --- a/e2e/specs/notification_trigger/flash_taskbar.test.ts +++ b/e2e/specs/notification_trigger/flash_taskbar.test.ts @@ -5,7 +5,7 @@ import {test, expect} from '../../fixtures/index'; import {waitForAppReady} from '../../helpers/appReadiness'; import {demoConfig} from '../../helpers/config'; import {acquireExclusiveLock} from '../../helpers/exclusiveLock'; -import {installFlashFrameSpy, restoreFlashFrameSpy} from '../../helpers/methodSpy'; +import {getFlashFrameCalls, installFlashFrameSpy, restoreFlashFrameSpy} from '../../helpers/methodSpy'; import {triggerNotificationEffects} from '../../helpers/notificationEffects'; test.describe('notification_trigger/flash_taskbar', () => { @@ -33,7 +33,7 @@ test.describe('notification_trigger/flash_taskbar', () => { await triggerNotificationEffects(electronApp, true); await expect.poll( - () => electronApp.evaluate(() => (global as any).__e2eFlashFrameCalls ?? []), + () => getFlashFrameCalls(electronApp), {timeout: 10_000, message: 'flashFrame(true) must be called when flashWindow is enabled'}, ).toContain(true); } finally { diff --git a/e2e/specs/notification_trigger/no_flash_taskbar.test.ts b/e2e/specs/notification_trigger/no_flash_taskbar.test.ts index 9878246797b..ee586633e17 100644 --- a/e2e/specs/notification_trigger/no_flash_taskbar.test.ts +++ b/e2e/specs/notification_trigger/no_flash_taskbar.test.ts @@ -5,7 +5,7 @@ import {test, expect} from '../../fixtures/index'; import {waitForAppReady} from '../../helpers/appReadiness'; import {demoConfig} from '../../helpers/config'; import {acquireExclusiveLock} from '../../helpers/exclusiveLock'; -import {installFlashFrameSpy, restoreFlashFrameSpy} from '../../helpers/methodSpy'; +import {getFlashFrameCalls, installFlashFrameSpy, restoreFlashFrameSpy} from '../../helpers/methodSpy'; import {triggerNotificationEffects} from '../../helpers/notificationEffects'; test.describe('notification_trigger/no_flash_taskbar', () => { @@ -44,7 +44,7 @@ test.describe('notification_trigger/no_flash_taskbar', () => { await triggerNotificationEffects(electronApp, true); await expect.poll( - () => electronApp.evaluate(() => (global as any).__e2eFlashFrameCalls ?? []), + () => getFlashFrameCalls(electronApp), {timeout: 10_000, message: 'flashFrame(true) must not be called when flashWindow is disabled'}, ).not.toContain(true); } finally { diff --git a/e2e/specs/permissions/permissions_ipc.test.ts b/e2e/specs/permissions/permissions_ipc.test.ts index fed5302883f..f73a3be4fd2 100644 --- a/e2e/specs/permissions/permissions_ipc.test.ts +++ b/e2e/specs/permissions/permissions_ipc.test.ts @@ -6,6 +6,14 @@ import {SHOW_SETTINGS_WINDOW} from '../../helpers/ipcChannels'; type ElectronApplication = Awaited>; +/** + * Callers must also depend on the `mainWindow` fixture before calling this, + * even though it isn't used directly here. `electronApp` alone launches the + * app without waiting for it to become ready (see fixtures/index.ts) — if + * MainWindow.get() isn't populated yet, handleShowSettingsModal() silently + * no-ops on a missing main window, and the waitForEvent('window') below + * times out 15s later waiting for a window that was never created. + */ async function openSettingsWindow(electronApp: ElectronApplication) { const existingWindow = electronApp.windows().find((window) => window.url().includes('settings')); if (existingWindow) { @@ -39,54 +47,69 @@ async function openSettingsWindow(electronApp: ElectronApplication) { } test.describe('permissions/ipc', () => { - test('MM-T6163 should return a valid media access status via GET_MEDIA_ACCESS_STATUS IPC', {tag: ['@P2', '@darwin', '@win32']}, async ({electronApp}) => { - const settingsWindow = await openSettingsWindow(electronApp); - - const status = await settingsWindow.evaluate( - () => (window as any).desktop.getMediaAccessStatus('microphone'), - ); - expect(['granted', 'denied', 'not-determined', 'restricted', 'unknown']).toContain(status); - }); - - test('MM-T6164 should open ms-settings:privacy-webcam for camera preferences (Windows only)', {tag: ['@P2', '@win32']}, async ({electronApp}) => { - const settingsWindow = await openSettingsWindow(electronApp); - - await electronApp.evaluate(({shell}) => { - (global as any).__testCapturedExternalURL = null; - shell.openExternal = (url: string) => { - (global as any).__testCapturedExternalURL = url; - return Promise.resolve(); - }; - }); - - await settingsWindow.evaluate( - () => (window as any).desktop.openWindowsCameraPreferences(), - ); - - const capturedURL = await electronApp.evaluate( - () => (global as any).__testCapturedExternalURL, - ); - expect(capturedURL).toBe('ms-settings:privacy-webcam'); - }); - - test('MM-T6165 should open ms-settings:privacy-microphone for microphone preferences (Windows only)', {tag: ['@P2', '@win32']}, async ({electronApp}) => { - const settingsWindow = await openSettingsWindow(electronApp); - - await electronApp.evaluate(({shell}) => { - (global as any).__testCapturedExternalURL = null; - shell.openExternal = (url: string) => { - (global as any).__testCapturedExternalURL = url; - return Promise.resolve(); - }; - }); - - await settingsWindow.evaluate( - () => (window as any).desktop.openWindowsMicrophonePreferences(), - ); - - const capturedURL = await electronApp.evaluate( - () => (global as any).__testCapturedExternalURL, - ); - expect(capturedURL).toBe('ms-settings:privacy-microphone'); - }); + test( + 'MM-T6163 should return a valid media access status via GET_MEDIA_ACCESS_STATUS IPC', + {tag: ['@P2', '@darwin', '@win32']}, + // eslint-disable-next-line @typescript-eslint/no-unused-vars -- see openSettingsWindow's doc comment + async ({electronApp, mainWindow: _mainWindow}) => { + const settingsWindow = await openSettingsWindow(electronApp); + + const status = await settingsWindow.evaluate( + () => (window as any).desktop.getMediaAccessStatus('microphone'), + ); + expect(['granted', 'denied', 'not-determined', 'restricted', 'unknown']).toContain(status); + }, + ); + + test( + 'MM-T6164 should open ms-settings:privacy-webcam for camera preferences (Windows only)', + {tag: ['@P2', '@win32']}, + // eslint-disable-next-line @typescript-eslint/no-unused-vars -- see openSettingsWindow's doc comment + async ({electronApp, mainWindow: _mainWindow}) => { + const settingsWindow = await openSettingsWindow(electronApp); + + await electronApp.evaluate(({shell}) => { + (global as any).__testCapturedExternalURL = null; + shell.openExternal = (url: string) => { + (global as any).__testCapturedExternalURL = url; + return Promise.resolve(); + }; + }); + + await settingsWindow.evaluate( + () => (window as any).desktop.openWindowsCameraPreferences(), + ); + + const capturedURL = await electronApp.evaluate( + () => (global as any).__testCapturedExternalURL, + ); + expect(capturedURL).toBe('ms-settings:privacy-webcam'); + }, + ); + + test( + 'MM-T6165 should open ms-settings:privacy-microphone for microphone preferences (Windows only)', + {tag: ['@P2', '@win32']}, + // eslint-disable-next-line @typescript-eslint/no-unused-vars -- see openSettingsWindow's doc comment + async ({electronApp, mainWindow: _mainWindow}) => { + const settingsWindow = await openSettingsWindow(electronApp); + + await electronApp.evaluate(({shell}) => { + (global as any).__testCapturedExternalURL = null; + shell.openExternal = (url: string) => { + (global as any).__testCapturedExternalURL = url; + return Promise.resolve(); + }; + }); + + await settingsWindow.evaluate( + () => (window as any).desktop.openWindowsMicrophonePreferences(), + ); + + const capturedURL = await electronApp.evaluate( + () => (global as any).__testCapturedExternalURL, + ); + expect(capturedURL).toBe('ms-settings:privacy-microphone'); + }, + ); }); diff --git a/e2e/specs/server_management/popout_windows.test.ts b/e2e/specs/server_management/popout_windows.test.ts index b9f3dfbbcae..a8eb38f9888 100644 --- a/e2e/specs/server_management/popout_windows.test.ts +++ b/e2e/specs/server_management/popout_windows.test.ts @@ -16,6 +16,7 @@ import { openPopoutWindow, } from '../../helpers/popoutWindow'; import {buildServerMap} from '../../helpers/serverMap'; +import {evaluateInMainProcessWithArg} from '../../helpers/testRefs'; const config = { ...demoMattermostConfig, @@ -74,11 +75,17 @@ test.describe('server_management/popout_windows', () => { const browserWindow = await electronApp.browserWindow(popoutWindow); const initialBounds = await browserWindow.evaluate((w) => (w as Electron.BrowserWindow).getBounds()); + const workArea = await evaluateInMainProcessWithArg( + electronApp, + (electron, bounds) => electron.screen.getDisplayMatching(bounds).workArea, + initialBounds, + ); + const margin = 20; const newBounds = { x: initialBounds.x, y: initialBounds.y, - width: initialBounds.width + 200, - height: initialBounds.height + 200, + width: Math.min(initialBounds.width + 200, (workArea.x + workArea.width) - initialBounds.x - margin), + height: Math.min(initialBounds.height + 200, (workArea.y + workArea.height) - initialBounds.y - margin), }; await browserWindow.evaluate((w, bounds) => { diff --git a/e2e/specs/settings/keyboard_shortcuts.test.ts b/e2e/specs/settings/keyboard_shortcuts.test.ts index a9c4a66db37..d757161a967 100644 --- a/e2e/specs/settings/keyboard_shortcuts.test.ts +++ b/e2e/specs/settings/keyboard_shortcuts.test.ts @@ -3,49 +3,14 @@ import {test, expect} from '../../fixtures/index'; import {cmdOrCtrl} from '../../helpers/config'; -import {SHOW_SETTINGS_WINDOW} from '../../helpers/ipcChannels'; - -type ElectronApplication = Awaited>; - -async function openSettingsWindow(electronApp: ElectronApplication) { - for (let attempt = 0; attempt < 5; attempt++) { - const existingWindow = electronApp.windows().find((window) => window.url().includes('settings')); - if (existingWindow) { - await existingWindow.waitForLoadState().catch(() => {}); - return existingWindow; - } - - try { - await electronApp.evaluate(({ipcMain}, showWindow) => { - ipcMain.emit(showWindow); - }, SHOW_SETTINGS_WINDOW); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - if (!message.includes('Execution context was destroyed') || attempt === 4) { - throw error; - } - } - - try { - const settingsWindow = await electronApp.waitForEvent('window', { - predicate: (window) => window.url().includes('settings'), - timeout: 3_000, - }); - await settingsWindow.waitForLoadState().catch(() => {}); - return settingsWindow; - } catch (error) { - if (attempt === 4) { - throw error; - } - await new Promise((resolve) => setTimeout(resolve, 250)); - } - } - - throw new Error('Settings window did not open'); -} +import {openSettingsWindow} from '../../helpers/settingsWindow'; test.describe('settings/keyboard_shortcuts', () => { test.describe('MM-T1288 Manipulating Text', () => { + test.beforeEach(async ({mainWindow}) => { + await mainWindow.bringToFront().catch(() => {}); + }); + test('MM-T1288_1 should be able to select and deselect language in the settings window', {tag: ['@P2', '@all']}, async ({electronApp}) => { const settingsWindow = await openSettingsWindow(electronApp); await settingsWindow.waitForSelector('#settingCategoryButton-language'); diff --git a/e2e/specs/system_tray_icon/hide_to_tray.test.ts b/e2e/specs/system_tray_icon/hide_to_tray.test.ts new file mode 100644 index 00000000000..75a1824c889 --- /dev/null +++ b/e2e/specs/system_tray_icon/hide_to_tray.test.ts @@ -0,0 +1,46 @@ +// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +import {test, expect} from '../../fixtures/index'; +import {demoConfig, type AppConfig} from '../../helpers/config'; +import {clickTrayMenuItem, emitTrayIconClick, hideMainWindow, isMainWindowVisible} from '../../helpers/tray'; + +const trayConfig: AppConfig = { + ...demoConfig, + showTrayIcon: true, + minimizeToTray: true, +}; + +test.describe('system_tray_icon/hide_to_tray', () => { + test.use({appConfig: trayConfig}); + + test( + 'MM-T6194 main window can be hidden to tray and restored', + {tag: ['@P1', '@all']}, + async ({electronApp}) => { + await expect.poll( + () => isMainWindowVisible(electronApp), + {timeout: 10_000, message: 'Main window should be visible after launch'}, + ).toBe(true); + + await hideMainWindow(electronApp); + + await expect.poll( + () => isMainWindowVisible(electronApp), + {timeout: 10_000, message: 'Main window should be hidden after hide()'}, + ).toBe(false); + + await emitTrayIconClick(electronApp); + + // macOS tray click opens the context menu; choosing a server raises the window. + if (process.platform === 'darwin') { + await clickTrayMenuItem(electronApp, trayConfig.servers[0].name); + } + + await expect.poll( + () => isMainWindowVisible(electronApp), + {timeout: 10_000, message: 'Main window should be visible after tray restore'}, + ).toBe(true); + }, + ); +}); diff --git a/e2e/specs/system_tray_icon/tray_menu.test.ts b/e2e/specs/system_tray_icon/tray_menu.test.ts index 0ae9e8aa1b9..ec02d50bf6b 100644 --- a/e2e/specs/system_tray_icon/tray_menu.test.ts +++ b/e2e/specs/system_tray_icon/tray_menu.test.ts @@ -19,7 +19,7 @@ test.describe('system_tray_icon/tray_menu', () => { test( 'TRAY-01 tray icon click restores hidden window when minimizeToTray is enabled', - {tag: ['@P0', '@all']}, + {tag: ['@P0', '@linux', '@win32']}, async ({electronApp}) => { await expect.poll( () => isMainWindowVisible(electronApp), diff --git a/e2e/specs/user_attributes/user_attributes.test.ts b/e2e/specs/user_attributes/user_attributes.test.ts index 9aa7add6c28..b79b18710f4 100644 --- a/e2e/specs/user_attributes/user_attributes.test.ts +++ b/e2e/specs/user_attributes/user_attributes.test.ts @@ -41,6 +41,7 @@ import { recoverFromProfileSettings, updateCustomProfileAttributeValues, type UserPropertyField, + waitForCustomAttributeEditInProfileSettings, } from '../../helpers/userAttributes'; const FIELD_PREFIX = 'E2E_UA_'; @@ -236,6 +237,7 @@ test.describe('user_attributes/user_attributes', () => { test.skip(true, 'Profile settings UI is not available on this server'); return; } + await waitForCustomAttributeEditInProfileSettings(win, created!.id); await win.runInRenderer(` document.querySelector('#customAttribute_${created!.id}Edit')?.click(); `); diff --git a/e2e/utils/analyze-flaky-test.js b/e2e/utils/analyze-flaky-test.js deleted file mode 100644 index 683aec879f7..00000000000 --- a/e2e/utils/analyze-flaky-test.js +++ /dev/null @@ -1,299 +0,0 @@ -// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. - -// CommonJS is required here: GitHub Actions workflows load this file via -// require() in actions/github-script, which does not support ES modules. - -const fs = require('fs'); -const path = require('path'); -const {createRequire} = require('module'); - -const JUNIT_REPORT_PATH = path.join(__dirname, '..', 'test-results', 'e2e-junit.xml'); - -function getXMLParserClass() { - const packageCandidates = [ - path.join(__dirname, '..', 'package.json'), - path.join(__dirname, '..', '..', 'package.json'), - ]; - - for (const packageJson of packageCandidates) { - try { - const {XMLParser} = createRequire(packageJson)('fast-xml-parser'); - return XMLParser; - } catch (error) { - const isModuleNotFound = - error && - (error.code === 'MODULE_NOT_FOUND' || error.code === 'ERR_MODULE_NOT_FOUND'); - if (!isModuleNotFound) { - throw error; - } - - // try the other package root (e2e/ vs repo root) - } - } - - throw new Error('fast-xml-parser is not installed. Run npm ci in the repo root and e2e/.'); -} - -function toNumber(value) { - const parsed = parseInt(value, 10); - return Number.isNaN(parsed) ? 0 : parsed; -} - -function asArray(value) { - if (!value) { - return []; - } - - return Array.isArray(value) ? value : [value]; -} - -function escapeRegex(value) { - return String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); -} - -function getSuiteFailureCount(suite) { - if (!suite || typeof suite !== 'object') { - return 0; - } - - // Short-circuit on the common "no failures" case: if the suite advertises - // 0 failures/errors AND has no testcase entries, there's nothing to walk. - const aggregatedFailures = toNumber(suite.failures) + toNumber(suite.errors); - const cases = asArray(suite.testcase); - if (aggregatedFailures === 0 && cases.length === 0) { - return 0; - } - - // If the run ultimately succeeded (exit code 0), the only failures present - // are retries that later passed — don't count any of them. - const exitCode = toNumber(process.env.PLAYWRIGHT_EXIT_CODE || '0'); - if (exitCode === 0) { - return 0; - } - - // Walk testcases and filter out failures that were later retried and passed. - // Playwright's JUnit aggregate `failures` attribute over-counts these, so - // never trust it as a final number. - const definitiveFailures = cases.filter((testcase) => { - // Empty self-closing elements () parse to "" — check for - // presence rather than truthiness. - if (testcase.failure === undefined && testcase.error === undefined) { - return false; - } - - // If this test name ends with a retry suffix like " (retry #1)", - // and the base test (without suffix) also appears as a passing case, - // this failure was retried and resolved — don't count it. - const name = testcase.name || ''; - const retryMatch = name.match(/^(.*) \(retry #\d+\)$/); - if (retryMatch) { - const baseName = retryMatch[1]; - const hasPassingRetry = cases.some( - (c) => c.name === baseName && c.failure === undefined && c.error === undefined, - ); - if (hasPassingRetry) { - return false; - } - } else { - const baseName = name; - const retryPattern = new RegExp(`^${escapeRegex(baseName)} \\(retry #\\d+\\)$`); - const hasPassingRetry = cases.some( - (c) => Boolean(c.name && retryPattern.test(c.name)) && - c.failure === undefined && c.error === undefined, - ); - if (hasPassingRetry) { - return false; - } - } - return true; - }); - - // If aggregate said failures>0 but we couldn't see any testcases (rare — - // summary-only reporter), fall back to the aggregate so we don't silently - // report 0 when something did fail. - if (definitiveFailures.length === 0 && cases.length === 0 && aggregatedFailures > 0) { - return aggregatedFailures; - } - - return definitiveFailures.length; -} - -function getFailureCountFromReport(report) { - if (!report || typeof report !== 'object') { - return 0; - } - - if (report.testsuites) { - const testsuites = report.testsuites; - - // Always walk the per-suite/testcase tree. The top-level aggregate - // `failures` / `errors` attributes include retried-and-passed tests, so - // returning them directly would over-count flaky tests as failures. - // getSuiteFailureCount() filters retries against passing reruns and - // honors PLAYWRIGHT_EXIT_CODE. - return asArray(testsuites.testsuite).reduce((total, suite) => total + getSuiteFailureCount(suite), 0); - } - - return getSuiteFailureCount(report.testsuite); -} - -/** - * Collect every testcase across every suite into a flat array. - */ -function collectAllCases(report) { - if (!report || typeof report !== 'object') { - return []; - } - const suites = report.testsuites ? - asArray(report.testsuites.testsuite) : - asArray(report.testsuite); - const all = []; - for (const suite of suites) { - if (!suite || typeof suite !== 'object') { - continue; - } - for (const tc of asArray(suite.testcase)) { - all.push(tc); - } - } - return all; -} - -/** - * Compute pass / fail / skip / total at the UNIQUE-TEST level (collapsing - * retries into a single outcome per test, mirroring how Playwright's HTML - * report reports stats). A test that failed once then passed on retry counts - * once as "passed" — not as both. - */ -function getOutcomeCounts(report) { - const cases = collectAllCases(report); - if (cases.length === 0) { - return {passed: 0, failed: 0, skipped: 0, total: 0}; - } - - // Group every case (including retries) by its base name. - const byBase = new Map(); - for (const tc of cases) { - const name = tc.name || ''; - const m = name.match(/^(.*) \(retry #\d+\)$/); - const base = m ? m[1] : name; - if (!byBase.has(base)) { - byBase.set(base, []); - } - byBase.get(base).push(tc); - } - - let passed = 0; - let failed = 0; - let skipped = 0; - for (const attempts of byBase.values()) { - // Empty self-closing tags parse to "" — check for property presence. - const anyPass = attempts.some( - (tc) => - tc.failure === undefined && - tc.error === undefined && - tc.skipped === undefined, - ); - const anyFail = attempts.some( - (tc) => tc.failure !== undefined || tc.error !== undefined, - ); - const allSkipped = attempts.every((tc) => tc.skipped !== undefined); - - if (anyPass) { - passed += 1; - } else if (anyFail) { - failed += 1; - } else if (allSkipped) { - skipped += 1; - } - } - - return {passed, failed, skipped, total: passed + failed + skipped}; -} - -function buildAnalysisResult({failureCount, passCount, skipCount, totalCount}) { - const collectedCount = passCount + failureCount + skipCount; - - // Playwright can exit 0 when test collection finds nothing (e.g. a broken - // import aborts discovery). Treat that as an infrastructure failure so the - // PR status check does not go green with "No tests ran". - if (collectedCount === 0) { - return { - failureCount: 1, - passCount: 0, - skipCount, - totalCount, - newFailedTests: ['no-tests-collected'], - os: process.platform, - testStatus: 'failure', - collectionFailed: true, - }; - } - - return { - failureCount, - passCount, - skipCount, - totalCount, - newFailedTests: new Array(failureCount).fill('failed'), - os: process.platform, - testStatus: failureCount > 0 ? 'failure' : 'success', - collectionFailed: false, - }; -} - -function analyzeFlakyTests() { - const hasJunit = fs.existsSync(JUNIT_REPORT_PATH); - - if (!hasJunit) { - if (process.env.JOB_STATUS === 'cancelled') { - return { - failureCount: 0, - passCount: 0, - skipCount: 0, - totalCount: 0, - newFailedTests: [], - os: process.platform, - testStatus: 'error', - collectionFailed: false, - }; - } - - return buildAnalysisResult({ - failureCount: 0, - passCount: 0, - skipCount: 0, - totalCount: 0, - }); - } - - const XMLParser = getXMLParserClass(); - const parser = new XMLParser({ - ignoreAttributes: false, - attributeNamePrefix: '', - }); - - const report = parser.parse(fs.readFileSync(JUNIT_REPORT_PATH, 'utf8')); - const failureCount = getFailureCountFromReport(report); - const outcomes = getOutcomeCounts(report); - - // `failureCount` is the authoritative number (it applies the retry-pass - // filter + honours PLAYWRIGHT_EXIT_CODE). When they disagree (rare — - // summary-only junit, or exit-code 0 with stale aggregates), trust - // `failureCount` and reconcile the rest. - const reconciledFailed = failureCount; - const reconciledPassed = Math.max(0, outcomes.total - reconciledFailed - outcomes.skipped); - - return buildAnalysisResult({ - failureCount: reconciledFailed, - passCount: reconciledPassed, - skipCount: outcomes.skipped, - totalCount: reconciledFailed + reconciledPassed + outcomes.skipped, - }); -} - -module.exports = { - analyzeFlakyTests, - buildAnalysisResult, -}; diff --git a/e2e/utils/github-actions.js b/e2e/utils/github-actions.js index f399f5cd41b..987a2f6ca9b 100644 --- a/e2e/utils/github-actions.js +++ b/e2e/utils/github-actions.js @@ -2,167 +2,33 @@ // See LICENSE.txt for license information. /* eslint-disable no-console -- Logging is intentional in CI utility scripts */ -const E2E_STATUS_CONTEXTS = [ - 'e2e/linux', - 'e2e/macos', - 'e2e/windows', - 'policy-test/macos', - 'policy-test/windows', -]; +const E2E_STATUS_CONTEXT = 'e2e-test/desktop-playwright'; const E2E_WORKFLOW_NAME = 'Electron Playwright Tests'; const ACTIVE_RUN_STATUSES = ['in_progress', 'queued', 'waiting']; const CANCELLED_STATUS_DESCRIPTION = 'E2E cancelled — tests skipped'; /** - * Update initial pending status for all platforms - * @param {Object} params - Parameters object - * @param {Object} params.github - GitHub API client from actions/github-script - * @param {Object} params.context - GitHub Actions context - * @param {Array} params.platforms - Array of platform objects from matrix - */ -async function updateInitialStatus({github, context, platforms}) { - const workflowUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; - - await Promise.all(platforms.map((platform) => - github.rest.repos.createCommitStatus({ - owner: context.repo.owner, - repo: context.repo.repo, - sha: context.sha, - state: 'pending', - context: `e2e/${platform.platform}`, - description: `E2E tests for Mattermost desktop app on ${platform.platform} have started...`, - target_url: workflowUrl, - }), - )); -} - -/** - * Build the short description shown in the PR status check. - * Only counts tests that actually ran on this platform (passed + failed). - * Skipped tests are omitted — they are cross-platform guards, not real - * failures, and inflate the denominator making results look worse. - * - * - all pass: "All 161 ran, 161 passed" - * - any failure: "161 ran, 157 passed, 4 failed" - */ -function formatStatusDescription({passed, failed, collectionFailed}) { - if (collectionFailed) { - return 'No tests ran (collection failed)'; - } - - const ran = passed + failed; - if (ran === 0) { - return failed > 0 ? `0 ran, ${failed} failed` : 'No tests ran'; - } - if (failed === 0) { - return `All ${ran} ran, ${passed} passed`; - } - return `${ran} ran, ${passed} passed, ${failed} failed`; -} - -async function resolveStatusSha({github, context, prNumber}) { - // Commit statuses must target the SHA this workflow run was dispatched for. - // PR HEAD moves when a new push cancels an in-flight run; using it would mark - // the new commit cancelled instead of the superseded one. - if (context.sha) { - return context.sha; - } - - if (prNumber) { - const {data: pr} = await github.rest.pulls.get({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: prNumber, - }); - return pr.head.sha; - } - - return context.payload.pull_request?.head?.sha; -} - -/** - * Update final status for all platforms based on test results - * @param {Object} params - Parameters object - * @param {Object} params.github - GitHub API client from actions/github-script - * @param {Object} params.context - GitHub Actions context - * @param {Array} params.platforms - Array of platform objects from matrix - * @param {Object} params.outputs - Test outputs from e2e-tests job - * @param {string} [params.e2eTestsResult] - needs.e2e-tests.result from the workflow - * @param {number} [params.prNumber] - PR number for status SHA lookup - */ -async function updateFinalStatus({github, context, platforms, outputs, e2eTestsResult, prNumber}) { - const workflowUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; - const sha = await resolveStatusSha({github, context, prNumber}); - const workflowCancelled = e2eTestsResult === 'cancelled'; - - await Promise.all(platforms.map((platform) => { - let osKey; - if (platform.runner.includes('ubuntu')) { - osKey = 'LINUX'; - } else if (platform.runner.includes('macos')) { - osKey = 'MACOS'; - } else { - osKey = 'WINDOWS'; - } - - const failed = Number(outputs[`NEW_FAILURES_${osKey}`] || 0); - const passed = Number(outputs[`PASSED_${osKey}`] || 0); - const collectionFailed = outputs[`COLLECTION_FAILED_${osKey}`] === 'true'; - const platformStatus = outputs[`STATUS_${osKey}`] || ''; - const reportLink = outputs[`REPORT_LINK_${osKey}`] || workflowUrl; - const ran = passed + failed; - - let state; - let description; - - if (platformStatus === 'error' || (workflowCancelled && ran === 0)) { - state = 'error'; - description = CANCELLED_STATUS_DESCRIPTION; - } else if (ran === 0 && (platformStatus === 'success' || platformStatus === '')) { - state = 'error'; - description = workflowCancelled ? CANCELLED_STATUS_DESCRIPTION : 'E2E incomplete — no tests ran'; - } else if (failed > 0 || platformStatus === 'failure') { - state = 'failure'; - description = formatStatusDescription({passed, failed, collectionFailed}); - } else { - state = 'success'; - description = formatStatusDescription({passed, failed, collectionFailed}); - } - - return github.rest.repos.createCommitStatus({ - owner: context.repo.owner, - repo: context.repo.repo, - sha, - state, - context: `e2e/${platform.platform}`, - description, - target_url: reportLink, - }); - })); -} - -/** - * Mark standard E2E commit statuses as cancelled/skipped on a SHA. + * Mark the E2E commit status as cancelled/skipped on a SHA. * GitHub commit statuses have no "skipped" state — `error` matches mobile E2E. */ async function markE2EStatusesCancelled({github, context, sha, reason = CANCELLED_STATUS_DESCRIPTION}) { const description = String(reason).substring(0, 140); const targetUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; - await Promise.all(E2E_STATUS_CONTEXTS.map((statusContext) => - github.rest.repos.createCommitStatus({ + try { + await github.rest.repos.createCommitStatus({ owner: context.repo.owner, repo: context.repo.repo, sha, state: 'error', - context: statusContext, + context: E2E_STATUS_CONTEXT, description, target_url: targetUrl, - }).catch((error) => { - console.log(`Could not update ${statusContext} on ${sha}: ${error.message}`); - }), - )); + }); + } catch (error) { + console.log(`Could not update ${E2E_STATUS_CONTEXT} on ${sha}: ${error.message}`); + } } /** @@ -308,12 +174,9 @@ async function removeE2ELabel({github, context}) { } module.exports = { - updateInitialStatus, - updateFinalStatus, removeE2ELabel, - formatStatusDescription, markE2EStatusesCancelled, cancelActiveE2ERuns, - E2E_STATUS_CONTEXTS, + E2E_STATUS_CONTEXT, CANCELLED_STATUS_DESCRIPTION, }; diff --git a/e2e/utils/tsio-report-status.js b/e2e/utils/tsio-report-status.js new file mode 100644 index 00000000000..eaeba16bb9b --- /dev/null +++ b/e2e/utils/tsio-report-status.js @@ -0,0 +1,254 @@ +// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. +/* eslint-disable no-console -- Logging is intentional in CI utility scripts */ + +const PRODUCTION_URL = 'https://test-io.test.mattermost.com'; +const STAGING_URL = 'https://staging-test-io.test.mattermost.com'; + +const TERMINAL_STATUSES = ['completed', 'incomplete']; + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +function intEnv(name, fallback) { + const raw = process.env[name]; + if (raw === undefined || raw === '') { + return fallback; + } + const n = Number(raw); + return Number.isInteger(n) && n > 0 ? n : fallback; +} + +function positiveInt(value, fallback) { + const n = Number(value); + return Number.isInteger(n) && n > 0 ? n : fallback; +} + +/** + * Commit-level rollup URL: /reports/{repo}/{branch}/{shortSha}/{name} + * e.g. https://test-io.test.mattermost.com/reports/desktop/tsio-spike/cff190a/desktop-pr + */ +function buildDisplayReportUrl(baseUrl, compositeIdentity) { + const repoTrailing = (compositeIdentity.repository || '').split('/').pop() || compositeIdentity.repository; + const repo = encodeURIComponent(repoTrailing); + const branch = encodeURIComponent( + (compositeIdentity.branch || 'main').replace(/^refs\/heads\//, '').replace(/^refs\/tags\//, ''), + ); + const shortSha = (compositeIdentity.commit_sha || '').slice(0, 7); + const name = encodeURIComponent(compositeIdentity.name); + return `${baseUrl}/reports/${repo}/${branch}/${shortSha}/${name}`; +} + +/** + * Recover a report group's id via the idempotent begin endpoint, poll the + * public status endpoint until the group leaves in_progress, render a step + * summary, and flip a commit status. + * @param {Object} params - Parameters object + * @param {Object} params.core - @actions/core from actions/github-script + * @param {Object} params.context - GitHub Actions context + * @param {Object} params.github - GitHub API client from actions/github-script + * @param {Object} params.compositeIdentity - {repository, commit_sha, gh_run_id, name, gh_run_attempt, branch, gh_pr_number} + * @param {number} params.totalReportsExpected - Number of per-leg reports expected in this group + * @param {string} params.commitStatusContext - Commit-status context to flip on completion + * @param {boolean} [params.failOnTestFailures] - When true (default), throw if the group didn't complete cleanly + * @param {boolean} [params.useStaging] - Target TSIO staging instead of production + * @param {string} [params.oidcAudience] - OIDC audience claim TSIO expects + * @param {boolean} [params.upstreamJobsSucceeded] - When false (default true), force the + * commit status to failure regardless of TSIO's test stats. TSIO only sees test-case-level + * results, so a job-level failure with no failing test attached to it (e.g. a hung worker + * teardown, a crashed runner, npm ci failing before any test ran) would otherwise still + * read as "100% passed" here even though the actual CI run failed. + * @param {number} [params.pollAttempts] - How many times to poll report group status (default + * 12, or TSIO_POLL_ATTEMPTS env). CMT runs with many legs should pass a higher value. + * @param {number} [params.pollDelayMs] - Delay between polls in ms (default 5000, or + * TSIO_POLL_DELAY_MS env). + * @returns {Promise<{reportUrl: string, status: string, stats: Object}>} + */ +async function reportTsioStatus({ + core, + context, + github, + compositeIdentity, + totalReportsExpected, + commitStatusContext, + failOnTestFailures = true, + useStaging = false, + oidcAudience = 'mattermost-test-system-io', + upstreamJobsSucceeded = true, + pollAttempts, + pollDelayMs, +}) { + const resolvedPollAttempts = positiveInt( + pollAttempts ?? intEnv('TSIO_POLL_ATTEMPTS', 12), + 12, + ); + const resolvedPollDelayMs = positiveInt( + pollDelayMs ?? intEnv('TSIO_POLL_DELAY_MS', 5000), + 5000, + ); + + const baseUrl = useStaging ? STAGING_URL : PRODUCTION_URL; + + // Fallback target for the commit status when no TSIO report ever gets + // created (begin/poll failed outright) — the reviewer still needs + // somewhere to click instead of a stuck `pending` row. + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + + let reportId; + let displayReportUrl; + let groupReportUrl; + let detail; + try { + const idToken = await core.getIDToken(oidcAudience); + core.setSecret(idToken); + + const beginRes = await fetch(`${baseUrl}/api/v1/reports/begin`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${idToken}`, + }, + body: JSON.stringify({ + repository: compositeIdentity.repository, + commit: compositeIdentity.commit_sha, + gh_run_id: compositeIdentity.gh_run_id, + gh_run_attempt: compositeIdentity.gh_run_attempt, + framework: 'playwright', + name: compositeIdentity.name, + branch: compositeIdentity.branch, + total_reports_expected: totalReportsExpected, + ...(compositeIdentity.gh_pr_number ? {gh_pr_number: parseInt(compositeIdentity.gh_pr_number, 10)} : {}), + }), + }); + if (!beginRes.ok) { + throw new Error(`reports/begin failed: ${beginRes.status} ${await beginRes.text()}`); + } + ({report_id: reportId} = await beginRes.json()); + + displayReportUrl = buildDisplayReportUrl(baseUrl, compositeIdentity); + groupReportUrl = `${baseUrl}/reports/g/${reportId}`; + + for (let attempt = 0; attempt < resolvedPollAttempts; attempt++) { + const statusRes = await fetch(`${baseUrl}/api/v1/reports/${reportId}`); + if (!statusRes.ok) { + throw new Error(`reports/${reportId} failed: ${statusRes.status} ${await statusRes.text()}`); + } + detail = await statusRes.json(); + if (TERMINAL_STATUSES.includes(detail.status)) { + break; + } + if (attempt < resolvedPollAttempts - 1) { + await sleep(resolvedPollDelayMs); + } + } + } catch (error) { + core.error(`TSIO reporting error: ${error.message}`); + try { + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: compositeIdentity.commit_sha, + state: 'failure', + context: commitStatusContext, + description: 'TSIO reporting error — see workflow run for details', + target_url: groupReportUrl || displayReportUrl || runUrl, + }); + } catch (statusError) { + core.warning(`Failed to create failure commit status: ${statusError.message}`); + } + throw error; + } + + if (!detail) { + throw new Error('TSIO report status never returned after polling'); + } + + const stats = detail.test_stats || {}; + const isComplete = detail.status === 'completed'; + const isIncomplete = detail.status === 'incomplete'; + const uploadedShards = Array.isArray(detail.reports) ? detail.reports.length : 0; + const failedShards = []; + if (Array.isArray(detail.reports)) { + for (const report of detail.reports) { + if (report.status === 'failed') { + failedShards.push(report.display_name || report.gh_job_name || report.id); + } + } + } + const hasFailures = (stats.failed || 0) > 0 || failedShards.length > 0; + + let overallState = 'failure'; + if (isComplete && !hasFailures && upstreamJobsSucceeded) { + overallState = 'success'; + } + + let targetUrl = runUrl; + if (isComplete || isIncomplete) { + targetUrl = displayReportUrl || groupReportUrl; + } + + const summaryLines = [ + `### Test System IO — ${compositeIdentity.name}`, + '', + `**Status:** ${detail.status} · **Report:** [this run](${groupReportUrl}) · [all runs for commit](${displayReportUrl})`, + `**Tests:** ${stats.passed ?? '?'} passed, ${stats.failed ?? '?'} failed, ${stats.flaky ?? 0} flaky, ` + + `${stats.skipped ?? '?'} skipped (of ${stats.total ?? '?'})`, + ]; + + if (uploadedShards > 0) { + summaryLines.push(`**Shards uploaded:** ${uploadedShards}/${totalReportsExpected}`); + } + + if (failedShards.length > 0) { + summaryLines.push(`**Failed shards:** ${failedShards.join(', ')}`); + } + + if (!upstreamJobsSucceeded && !hasFailures) { + summaryLines.push( + '', + ':warning: One or more CI jobs failed outside of any tracked test (e.g. a hung worker, a crashed runner) — forcing this status to failure even though the test stats above may show no failures.', + ); + } + + if (isIncomplete) { + summaryLines.push( + '', + `:warning: Report finalized as \`incomplete\` (${uploadedShards}/${totalReportsExpected} shards) — partial results are in the [TSIO report](${displayReportUrl}); see the [workflow run](${runUrl}) for missing legs.`, + ); + } else if (!isComplete) { + summaryLines.push( + '', + `:warning: Report never reached a terminal state (stuck at \`${detail.status}\`) after ${resolvedPollAttempts} polls — see the [workflow run](${runUrl}).`, + ); + } + + summaryLines.push(''); + await core.summary.addRaw(summaryLines.join('\n')).write(); + + const descriptionPrefix = !upstreamJobsSucceeded && !hasFailures ? 'CI job failed (untracked by TSIO), ' : ''; + const description = `${descriptionPrefix}${stats.passed ?? 0}/${stats.total ?? 0} passed, ${stats.failed ?? 0} failed, ${stats.skipped ?? 0} skipped`.slice(0, 140); + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: compositeIdentity.commit_sha, + state: overallState, + context: commitStatusContext, + description, + target_url: targetUrl, + }); + + if (failOnTestFailures && overallState === 'failure') { + let reason; + if (!upstreamJobsSucceeded && !hasFailures) { + reason = 'an upstream CI job failed with no corresponding test failure'; + } else if (failedShards.length > 0 && (stats.failed || 0) === 0) { + reason = `shard(s) failed: ${failedShards.join(', ')}`; + } else { + reason = `status=${detail.status}, failed=${stats.failed || 0}`; + } + throw new Error(`TSIO report ${reportId} did not pass: ${reason}`); + } + + return {reportUrl: displayReportUrl || groupReportUrl, status: detail.status, stats}; +} + +module.exports = reportTsioStatus; diff --git a/e2e/utils/write-tsio-failure-stub.mjs b/e2e/utils/write-tsio-failure-stub.mjs new file mode 100644 index 00000000000..468d635b41a --- /dev/null +++ b/e2e/utils/write-tsio-failure-stub.mjs @@ -0,0 +1,49 @@ +// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +const OUTPUT = path.join('e2e', 'test-results', 'results.json'); + +const reason = process.env.TSIO_STUB_REASON || + (process.env.PLAYWRIGHT_EXIT_CODE && process.env.PLAYWRIGHT_EXIT_CODE !== '0' ? + `Playwright exited with code ${process.env.PLAYWRIGHT_EXIT_CODE} and no results.json was written` : + 'CI job failed before Playwright JSON results were available'); + +const jobLabel = process.env.TSIO_GH_JOB_NAME || 'unknown-job'; + +const report = { + config: { + rootDir: path.resolve('e2e'), + version: '', + }, + suites: [{ + title: '', + file: 'ci/tsio-shard-failure.ts', + column: 0, + line: 0, + specs: [{ + title: `[${jobLabel}] CI shard failure`, + ok: false, + tags: ['@tsio-stub'], + tests: [{ + timeout: 0, + annotations: [], + expectedStatus: 'passed', + projectName: jobLabel, + results: [{ + workerIndex: 0, + status: 'failed', + duration: 0, + error: {message: reason}, + }], + status: 'failed', + }], + }], + }], +}; + +fs.mkdirSync(path.dirname(OUTPUT), {recursive: true}); +fs.writeFileSync(OUTPUT, JSON.stringify(report)); +console.log(`Wrote TSIO failure stub to ${OUTPUT}: ${reason}`); diff --git a/package.json b/package.json index ae1da3c57d4..566e8bc85f9 100644 --- a/package.json +++ b/package.json @@ -81,7 +81,7 @@ ], "testPathIgnorePatterns": [ "/node_modules/", - "/e2e/", + "/e2e/", "/webpack.config.test.js" ], "globals": { diff --git a/src/app/mainWindow/mainWindow.ts b/src/app/mainWindow/mainWindow.ts index e314de3a7b3..7907e0fb394 100644 --- a/src/app/mainWindow/mainWindow.ts +++ b/src/app/mainWindow/mainWindow.ts @@ -41,6 +41,7 @@ import {DEFAULT_WINDOW_HEIGHT, DEFAULT_WINDOW_WIDTH} from 'common/utils/constant import * as Validator from 'common/Validator'; import ViewManager from 'common/views/viewManager'; import {boundsInfoPath} from 'main/constants'; +import {registerMainWindowE2EReadiness} from 'main/e2e/appReady'; import {localizeMessage} from 'main/i18nManager'; import performanceMonitor from 'main/performanceMonitor'; import ThemeManager from 'main/themeManager'; @@ -90,6 +91,8 @@ export class MainWindow extends EventEmitter { throw new Error('unable to create main window'); } + registerMainWindowE2EReadiness(this.win.browserWindow); + this.win.browserWindow.webContents.once('did-finish-load', () => { if (!this.win || this.win.browserWindow.isDestroyed()) { return; diff --git a/src/main/app/initialize.test.js b/src/main/app/initialize.test.js index 047dc06f2bb..7dad008945c 100644 --- a/src/main/app/initialize.test.js +++ b/src/main/app/initialize.test.js @@ -125,6 +125,9 @@ jest.mock('main/app/config', () => ({ handleConfigUpdate: jest.fn(), handleUpdateTheme: jest.fn(), })); +jest.mock('main/e2e/appReady', () => ({ + registerMainWindowE2EReadiness: jest.fn(), +})); jest.mock('main/app/intercom', () => ({ handleMainWindowIsShown: jest.fn(), })); diff --git a/src/main/app/intercom.test.js b/src/main/app/intercom.test.js index e793dd96b6d..78603b14157 100644 --- a/src/main/app/intercom.test.js +++ b/src/main/app/intercom.test.js @@ -79,35 +79,6 @@ describe('main/app/intercom', () => { }); describe('handleMainWindowIsShown', () => { - // Helper: build a BrowserWindow mock whose `once` records listeners so - // tests can fire them on demand. - const makeWindow = (initialVisible) => { - const listeners = {}; - let visible = initialVisible; - return { - listeners, - setVisible: (v) => { - visible = v; - }, - isVisible: jest.fn(() => visible), - once: jest.fn((event, cb) => { - listeners[event] = cb; - }), - removeListener: jest.fn((event, cb) => { - if (listeners[event] === cb) { - delete listeners[event]; - } - }), - fire: (event) => listeners[event] && listeners[event](), - }; - }; - - afterEach(() => { - delete global.__e2eAppReady; - jest.useRealTimers(); - jest.clearAllMocks(); - }); - it('MM-48079 should not show onboarding screen or server screen if GPO server is pre-configured', () => { getLocalPreload.mockReturnValue('/some/preload.js'); MainWindow.get.mockReturnValue({ @@ -119,57 +90,6 @@ describe('main/app/intercom', () => { handleMainWindowIsShown(); expect(ModalManager.addModal).not.toHaveBeenCalled(); }); - - it('should mark __e2eAppReady synchronously without attaching listeners when the main window is already visible', () => { - ServerManager.hasServers.mockReturnValue(true); - const win = makeWindow(true); - MainWindow.get.mockReturnValue(win); - - handleMainWindowIsShown(); - - expect(global.__e2eAppReady).toBe(true); - expect(win.once).not.toHaveBeenCalled(); - expect(MainWindow.once).not.toHaveBeenCalled(); - }); - - it('should set __e2eAppReady via a `show` listener (not `ready-to-show`) when the window is not yet visible', () => { - ServerManager.hasServers.mockReturnValue(true); - const win = makeWindow(false); - MainWindow.get.mockReturnValue(win); - - handleMainWindowIsShown(); - - // We listen to `show` only — never `ready-to-show`, which fires - // *before* the window is visible. - expect(win.once).toHaveBeenCalledWith('show', expect.any(Function)); - expect(win.once).not.toHaveBeenCalledWith('ready-to-show', expect.any(Function)); - expect(global.__e2eAppReady).toBeUndefined(); - - // The window becomes visible and the `show` event fires. - win.fire('show'); - expect(global.__e2eAppReady).toBe(true); - }); - - it('should defer to MAIN_WINDOW_CREATED if no main window exists yet, then mark ready when it appears', () => { - ServerManager.hasServers.mockReturnValue(true); - MainWindow.get.mockReturnValue(undefined); - - handleMainWindowIsShown(); - - // No window yet — should have registered a one-shot listener. - expect(MainWindow.once).toHaveBeenCalledWith('main-window-created', expect.any(Function)); - expect(global.__e2eAppReady).toBeUndefined(); - - // Window comes into existence (and happens to already be visible). - const win = makeWindow(true); - MainWindow.get.mockReturnValue(win); - - // Invoke the captured listener (simulating MainWindow emitting). - const createdCb = MainWindow.once.mock.calls[0][1]; - createdCb(); - - expect(global.__e2eAppReady).toBe(true); - }); }); describe('handleShowSettingsModal', () => { diff --git a/src/main/app/intercom.ts b/src/main/app/intercom.ts index 079fc03cbf3..b47aeb090ac 100644 --- a/src/main/app/intercom.ts +++ b/src/main/app/intercom.ts @@ -13,7 +13,6 @@ import {Logger} from 'common/log'; import ServerManager from 'common/servers/serverManager'; import {ping} from 'common/utils/requests'; import {parseURL} from 'common/utils/url'; -import {signalE2EAppReadyWhenShown} from 'main/e2e/appReady'; import NotificationManager from 'main/notifications'; import {getLocalPreload} from 'main/utils'; @@ -87,8 +86,6 @@ export function handleMainWindowIsShown() { handleShowOnboardingScreens(showWelcomeScreen(), showNewServerModal(), false); }); } - - signalE2EAppReadyWhenShown(); } export function handleWelcomeScreenModal(prefillURL?: string) { diff --git a/src/main/e2e/appReady.test.js b/src/main/e2e/appReady.test.js new file mode 100644 index 00000000000..b97b3f34af3 --- /dev/null +++ b/src/main/e2e/appReady.test.js @@ -0,0 +1,58 @@ +// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +import Config from 'common/config'; + +import {registerMainWindowE2EReadiness} from 'main/e2e/appReady'; + +jest.mock('common/config', () => ({ + hideOnStart: false, +})); + +describe('main/e2e/appReady', () => { + afterEach(() => { + delete global.__e2eAppReady; + Config.hideOnStart = false; + }); + + const makeWindow = (visible) => { + const listeners = {}; + return { + isDestroyed: () => false, + isVisible: () => visible, + once: jest.fn((event, cb) => { + listeners[event] = cb; + }), + webContents: { + getURL: () => 'mattermost-desktop://renderer/index.html', + once: jest.fn((event, cb) => { + listeners[`wc:${event}`] = cb; + }), + }, + fire: (event) => listeners[event]?.(), + fireWebContents: (event) => listeners[`wc:${event}`]?.(), + }; + }; + + it('should mark ready on show when hideOnStart is false', () => { + const win = makeWindow(false); + registerMainWindowE2EReadiness(win); + + expect(global.__e2eAppReady).toBeUndefined(); + expect(win.once).toHaveBeenCalledWith('show', expect.any(Function)); + expect(win.webContents.once).not.toHaveBeenCalled(); + + win.fire('show'); + expect(global.__e2eAppReady).toBe(true); + }); + + it('should mark ready on index did-finish-load when hideOnStart is true', () => { + Config.hideOnStart = true; + const win = makeWindow(false); + registerMainWindowE2EReadiness(win); + + expect(win.webContents.once).toHaveBeenCalledWith('did-finish-load', expect.any(Function)); + win.fireWebContents('did-finish-load'); + expect(global.__e2eAppReady).toBe(true); + }); +}); diff --git a/src/main/e2e/appReady.ts b/src/main/e2e/appReady.ts index b0c2608570d..777aa6c7fce 100644 --- a/src/main/e2e/appReady.ts +++ b/src/main/e2e/appReady.ts @@ -3,38 +3,29 @@ import type {BrowserWindow} from 'electron'; -import MainWindow from 'app/mainWindow/mainWindow'; -import {MAIN_WINDOW_CREATED} from 'common/communication'; +import Config from 'common/config'; import {setTestField} from 'common/utils/util'; /** - * Signals `__e2eAppReady` once the main window is visible so Playwright can wait - * on app readiness. No-op outside NODE_ENV=test. + * Register listeners on the main BrowserWindow so `__e2eAppReady` is set at the + * first reliable lifecycle point. Must run from MainWindow.init() before + * index.html finishes loading — otherwise a fast `show` during later startup + * work can fire before any listener is attached. */ -export function signalE2EAppReadyWhenShown(): void { +export function registerMainWindowE2EReadiness(win: BrowserWindow): void { if (process.env.NODE_ENV !== 'test') { return; } const markReady = () => setTestField('__e2eAppReady', true); - const whenVisible = (win: BrowserWindow) => { - if (win.isVisible()) { - markReady(); - } else { - win.once('show', markReady); - } - }; - const win = MainWindow.get(); - if (win) { - whenVisible(win); - return; - } + win.once('show', markReady); - MainWindow.once(MAIN_WINDOW_CREATED, () => { - const created = MainWindow.get(); - if (created) { - whenVisible(created); - } - }); + if (Config.hideOnStart) { + win.webContents.once('did-finish-load', () => { + if (!win.isDestroyed() && win.webContents.getURL().includes('index')) { + markReady(); + } + }); + } }