From 0caa4168e0d5853d6a60550cee6d02b5d2fc5c46 Mon Sep 17 00:00:00 2001 From: Inomdzhon Mirdzhamolov Date: Mon, 19 Jun 2023 17:20:01 +0300 Subject: [PATCH 1/6] ci(e2e): use playwright shard feature MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Изменения в `reusable_workflow_test_e2e.yml`: - Используем `strategy.matrix` и `--shard` для распараллеливания джоб. - Объединяем отчёты через `npx playwright merge-reports` см. https://playwright.dev/docs/test-sharding#github-actions-example - Пробуем использовать `@actions/cache` для кеширования `playwright/.cache`. - Удалили загрузку артефакта `e2e-output` за ненадобностью. - Удалили `TODO Playwright реализовать покрытие`, т.к. покрытие мы уже собираем jest'ом. Изменения в `playwright-ct.config.ts`: - Так как в CI мы распараллеливаем тесты через `--shard`, то ставим `workers` на 1. - Пробуем выставить `maxFailures: 10` в CI, чтобы в лишний раз не гонять все тесты, если 10 из них упали. - Используем для CI blob для отчёта, чтобы потом можно было смержить. --- .eslintignore | 2 + .github/workflows/pull_request_packages.yml | 9 ++-- .../workflows/reusable_workflow_test_e2e.yml | 48 ++++++++++--------- .gitignore | 2 + .prettierignore | 2 + packages/vkui/playwright-ct.config.ts | 13 +++-- 6 files changed, 43 insertions(+), 33 deletions(-) diff --git a/.eslintignore b/.eslintignore index 55058ece812..8bb41d4fcdf 100644 --- a/.eslintignore +++ b/.eslintignore @@ -7,3 +7,5 @@ target/ fixture/ storybook-static/ playwright-report/ +blob-report/ +all-blob-reports/ diff --git a/.github/workflows/pull_request_packages.yml b/.github/workflows/pull_request_packages.yml index d98a080b979..31c7c83f45b 100644 --- a/.github/workflows/pull_request_packages.yml +++ b/.github/workflows/pull_request_packages.yml @@ -140,13 +140,16 @@ jobs: with: name: test-output - - name: Download Playwright Report artifact + - name: Download Playwright blob reports from GitHub Actions Artifacts # см. [Примечание 1] if: ${{ needs.changed_files.outputs.package_vkui == 'true' }} uses: actions/download-artifact@v3 with: - name: playwright-report - path: playwright-report/ + name: all-blob-reports + path: all-blob-reports + + - name: Merge Playwright blob reports into HTML Report + run: npx playwright merge-reports --reporter html ./all-blob-reports - name: Upload Playwright Report if: ${{ needs.changed_files.outputs.package_vkui == 'true' }} diff --git a/.github/workflows/reusable_workflow_test_e2e.yml b/.github/workflows/reusable_workflow_test_e2e.yml index 10c03802134..d7d9e389577 100644 --- a/.github/workflows/reusable_workflow_test_e2e.yml +++ b/.github/workflows/reusable_workflow_test_e2e.yml @@ -16,7 +16,12 @@ jobs: # см. https://github.com/microsoft/playwright/blob/main/utils/docker/Dockerfile.focal image: mcr.microsoft.com/playwright:v1.37.0-focal options: --ipc=host - name: Run visual (aka e2e) tests + strategy: + fail-fast: false + matrix: + shardIndex: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + shardTotal: [10] + name: 'Run e2e tests for (shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }})' steps: - name: Checkout uses: actions/checkout@v3 @@ -54,32 +59,29 @@ jobs: - name: Install dependencies run: yarn install --frozen-lockfile --ignore-scripts + - name: Restore cached Playwright build + id: cache_playwright_build_restore + uses: actions/cache/restore@v3 + with: + path: packages/vkui/playwright/.cache + key: playwright-build-cache-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.sha }} + restore-keys: playwright-build-cache-${{ github.event.pull_request.number }}- + - name: Run e2e tests for @vkontakte/vkui workspace # Почему HOME=/root см. https://github.com/microsoft/playwright/issues/6500 - run: HOME=/root yarn run test:e2e:ci + run: HOME=/root yarn run test:e2e:ci --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} - # TODO Playwright реализовать покрытие - # - name: Upload coverage to Codecov - # uses: codecov/codecov-action@v3 - # with: - # token: ${{ secrets.CODECOV_TOKEN }} - # flags: e2e-${{ matrix.browser }}-${{ matrix.platform }}-${{ matrix.appearance }} - # files: .nyc_output/coverage.json - # fail_ci_if_error: true - - - name: Upload test artifact - uses: actions/upload-artifact@v3 - if: always() + - name: Save Playwright build cache + id: playwright-build-cache + uses: actions/cache/save@v3 with: - name: e2e-output - path: | - packages/vkui/__diff_output__/ - packages/vkui/e2e-results.json + path: packages/vkui/playwright/.cache + key: ${{ steps.cache_playwright_build_restore.outputs.cache-primary-key }} - - name: Upload Playwright HTML report - uses: actions/upload-artifact@v3 + - name: Upload Playwright blob report to GitHub Actions Artifacts if: always() + uses: actions/upload-artifact@v3 with: - name: playwright-report - path: packages/vkui/playwright-report/ - retention-days: 30 + name: all-blob-reports + path: packages/vkui/blob-report + retention-days: 1 diff --git a/.gitignore b/.gitignore index d8d7df4944a..a8f7173ba0e 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,8 @@ lint-results.json e2e-results.json yarn-error.log playwright-report/ +blob-report/ +all-blob-reports/ # Caches .cache diff --git a/.prettierignore b/.prettierignore index 289cb1a4149..c784a5c25db 100644 --- a/.prettierignore +++ b/.prettierignore @@ -17,3 +17,5 @@ target/ fixture/ storybook-static/ playwright-report/ +blob-report/ +all-blob-reports/ diff --git a/packages/vkui/playwright-ct.config.ts b/packages/vkui/playwright-ct.config.ts index 4532bcfdef5..2e9dcd3c374 100644 --- a/packages/vkui/playwright-ct.config.ts +++ b/packages/vkui/playwright-ct.config.ts @@ -22,10 +22,7 @@ const TS_CONFIG_ALIASES = Object.entries(tsconfig.compilerOptions.paths).reduce< return aliases; }, {}); -const DEFAULT_REPORTERS: ReporterDescription[] = [ - ['html', { open: 'never' }], - ['json', { outputFile: 'e2e-results.json' }], -]; +const DEFAULT_REPORTERS: ReporterDescription[] = [['json', { outputFile: 'e2e-results.json' }]]; /** * See https://playwright.dev/docs/test-configuration. @@ -56,16 +53,18 @@ export default defineConfig({ forbidOnly: !!process.env.CI, /* Retry on CI only */ retries: 0, + /* Limit the number of failures on CI to save resources. */ + maxFailures: process.env.CI ? 10 : undefined, /* Opt out of parallel tests on CI. */ workers: process.env.CI - ? 4 + ? 1 : typeof process.env.PLAYWRIGHT_WORKERS === 'string' ? Number(process.env.PLAYWRIGHT_WORKERS) : undefined, /* Reporter to use. See https://playwright.dev/docs/test-reporters */ reporter: process.env.CI - ? [['github'], ['dot'], ...DEFAULT_REPORTERS] - : [['list'], ...DEFAULT_REPORTERS], + ? [['github'], ['dot'], ['blob'], ...DEFAULT_REPORTERS] + : [['list'], ['html', { open: 'never' }], ...DEFAULT_REPORTERS], /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { From 66c8e15889de051bc414387ea151c28beedc86f1 Mon Sep 17 00:00:00 2001 From: Inomdzhon Mirdzhamolov Date: Mon, 14 Aug 2023 15:32:33 +0300 Subject: [PATCH 2/6] fix: use playwright merge-reports by yarn cmd --- .github/workflows/pull_request_packages.yml | 2 +- package.json | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pull_request_packages.yml b/.github/workflows/pull_request_packages.yml index 31c7c83f45b..9d770df1ee8 100644 --- a/.github/workflows/pull_request_packages.yml +++ b/.github/workflows/pull_request_packages.yml @@ -149,7 +149,7 @@ jobs: path: all-blob-reports - name: Merge Playwright blob reports into HTML Report - run: npx playwright merge-reports --reporter html ./all-blob-reports + run: yarn run playwright:cmd:merge-reports --reporter html ./all-blob-reports - name: Upload Playwright Report if: ${{ needs.changed_files.outputs.package_vkui == 'true' }} diff --git a/package.json b/package.json index b4db31f1b7f..31a6dd8235e 100644 --- a/package.json +++ b/package.json @@ -125,6 +125,7 @@ "prepare": "husky install", "pre-commit": "lint-staged", "playwright:install": "playwright install --with-deps", + "playwright:cmd:merge-reports": "playwright merge-reports", "codesandbox:install": "PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 yarn install", "deduplicate": "yarn-deduplicate --list && yarn-deduplicate --fail" }, From a0570eca257cf9bde4fa126a6215e75fd790426d Mon Sep 17 00:00:00 2001 From: Inomdzhon Mirdzhamolov Date: Mon, 14 Aug 2023 16:23:08 +0300 Subject: [PATCH 3/6] fix: add test_e2e_prepare_report job --- .github/workflows/pull_request_packages.yml | 43 +++++++++++++++++---- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pull_request_packages.yml b/.github/workflows/pull_request_packages.yml index 9d770df1ee8..d4b3d4eb91d 100644 --- a/.github/workflows/pull_request_packages.yml +++ b/.github/workflows/pull_request_packages.yml @@ -88,6 +88,37 @@ jobs: with: ref: refs/pull/${{ github.event.pull_request.number }}/merge + test_e2e_prepare_report: + needs: test_e2e + name: Prepare e2e's HTML report + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Setup NodeJS + uses: actions/setup-node@v3 + with: + node-version: 18 + cache: 'yarn' + + - name: Download Playwright blob reports from GitHub Actions Artifacts + uses: actions/download-artifact@v3 + with: + name: all-blob-reports + path: all-blob-reports + + - name: Merge Playwright blob reports into HTML Report + run: yarn run playwright:cmd:merge-reports --reporter html ./all-blob-reports + + - name: Upload Playwright HTML report to GitHub Actions Artifacts + if: always() + uses: actions/upload-artifact@v3 + with: + name: playwright-report + path: playwright-report + retention-days: 1 + analyze_bundle_size: if: ${{ needs.changed_files.outputs.package_vkui == 'true' }} needs: changed_files @@ -126,7 +157,7 @@ jobs: - changed_files - linters - test - - test_e2e + - test_e2e_prepare_report runs-on: ubuntu-latest name: Report CI results steps: @@ -140,16 +171,12 @@ jobs: with: name: test-output - - name: Download Playwright blob reports from GitHub Actions Artifacts - # см. [Примечание 1] + - name: Download Playwright HTML report from GitHub Actions Artifacts if: ${{ needs.changed_files.outputs.package_vkui == 'true' }} uses: actions/download-artifact@v3 with: - name: all-blob-reports - path: all-blob-reports - - - name: Merge Playwright blob reports into HTML Report - run: yarn run playwright:cmd:merge-reports --reporter html ./all-blob-reports + name: playwright-report + path: playwright-report - name: Upload Playwright Report if: ${{ needs.changed_files.outputs.package_vkui == 'true' }} From 894da0afd9c6ab7d70fc4a39556fb01177fdb1bc Mon Sep 17 00:00:00 2001 From: Inomdzhon Mirdzhamolov Date: Mon, 14 Aug 2023 17:30:54 +0300 Subject: [PATCH 4/6] Update .github/workflows/pull_request_packages.yml Co-authored-by: Daniil Suvorov --- .github/workflows/pull_request_packages.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pull_request_packages.yml b/.github/workflows/pull_request_packages.yml index d4b3d4eb91d..a3bdff5ea41 100644 --- a/.github/workflows/pull_request_packages.yml +++ b/.github/workflows/pull_request_packages.yml @@ -102,6 +102,9 @@ jobs: node-version: 18 cache: 'yarn' + - name: Install dependencies + run: yarn install --frozen-lockfile --ignore-scripts + - name: Download Playwright blob reports from GitHub Actions Artifacts uses: actions/download-artifact@v3 with: From 0cd2815ced9674edde810349686a24e77cf98e33 Mon Sep 17 00:00:00 2001 From: Inomdzhon Mirdzhamolov Date: Mon, 14 Aug 2023 17:41:40 +0300 Subject: [PATCH 5/6] fix(reusable_workflow_test_e2e): rm @actions/cache --- .github/workflows/reusable_workflow_test_e2e.yml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/.github/workflows/reusable_workflow_test_e2e.yml b/.github/workflows/reusable_workflow_test_e2e.yml index d7d9e389577..604dd594f01 100644 --- a/.github/workflows/reusable_workflow_test_e2e.yml +++ b/.github/workflows/reusable_workflow_test_e2e.yml @@ -59,25 +59,10 @@ jobs: - name: Install dependencies run: yarn install --frozen-lockfile --ignore-scripts - - name: Restore cached Playwright build - id: cache_playwright_build_restore - uses: actions/cache/restore@v3 - with: - path: packages/vkui/playwright/.cache - key: playwright-build-cache-${{ github.event.pull_request.number }}-${{ github.run_id }}-${{ github.sha }} - restore-keys: playwright-build-cache-${{ github.event.pull_request.number }}- - - name: Run e2e tests for @vkontakte/vkui workspace # Почему HOME=/root см. https://github.com/microsoft/playwright/issues/6500 run: HOME=/root yarn run test:e2e:ci --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} - - name: Save Playwright build cache - id: playwright-build-cache - uses: actions/cache/save@v3 - with: - path: packages/vkui/playwright/.cache - key: ${{ steps.cache_playwright_build_restore.outputs.cache-primary-key }} - - name: Upload Playwright blob report to GitHub Actions Artifacts if: always() uses: actions/upload-artifact@v3 From e373ae983b64885ddddc083345eaf62f3dea4d0a Mon Sep 17 00:00:00 2001 From: Inomdzhon Mirdzhamolov Date: Tue, 15 Aug 2023 12:47:39 +0300 Subject: [PATCH 6/6] fix: use retention days: 30 for playwright-report --- .github/workflows/pull_request_packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull_request_packages.yml b/.github/workflows/pull_request_packages.yml index a3bdff5ea41..1ef5a053194 100644 --- a/.github/workflows/pull_request_packages.yml +++ b/.github/workflows/pull_request_packages.yml @@ -120,7 +120,7 @@ jobs: with: name: playwright-report path: playwright-report - retention-days: 1 + retention-days: 30 analyze_bundle_size: if: ${{ needs.changed_files.outputs.package_vkui == 'true' }}