Merge pull request #7203 from LedgerHQ/test/parallelization

[QAA-120] Review and optimize e2e testing workflow
LedgerHQ · Jul 3, 2024 · b54085f · b54085f
2 parents c37944f + 4e6b25d
commit b54085f
Show file tree

Hide file tree

Showing 38 changed files with 308 additions and 377 deletions.
diff --git a/.github/workflows/test-ui-e2e-only-desktop.yml b/.github/workflows/test-ui-e2e-only-desktop.yml
@@ -22,14 +22,6 @@ on:
         description: Ignore test having name pattern (entered in the previous field)
         type: boolean
         default: false
-      speculos_tests:
-        description: Run the speculos tests (if false, mocked tests will be run instead)
-        type: boolean
-        default: true
-      enable_broadcast:
-        description: Enable transaction broadcast
-        type: boolean
-        default: false
 
       report_path:
         description: Path where you want to store this execution's results
@@ -63,6 +55,11 @@ jobs:
       # DEBUG: "pw:browser*"
       # DEBUG_LOGS: 1
     runs-on: [ledger-live-4xlarge]
+    strategy:
+      fail-fast: false
+      matrix:
+        shardIndex: [1, 2, 3]
+        shardTotal: [3]
     steps:
       - uses: actions/checkout@v4
         with:
@@ -103,33 +100,23 @@ jobs:
         id: tests
         run: |
           export COINAPPS=$PWD/coin-apps
-          if [ "$ENABLE_BROADCAST" = true ]; then export ENABLE_TRANSACTION_BROADCAST=1; fi
-          if [ "$SPECULOS_TESTS" = true ]; then export MOCK=0 && speculos_tests=":speculos"; fi
+          export ENABLE_TRANSACTION_BROADCAST=1
+          export MOCK=0
           if [ "${{ inputs.invert_filter }}" = true ]; then invert_filter="-invert"; fi
-          xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- pnpm desktop test:playwright$speculos_tests ${INPUTS_TEST_FILTER:+--grep$invert_filter} "${{ inputs.test_filter }}"
+          xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- pnpm desktop test:playwright:speculos ${INPUTS_TEST_FILTER:+--grep$invert_filter} "${{ inputs.test_filter }}" --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
         env:
           INPUTS_TEST_FILTER: ${{ inputs.test_filter }}
           SEED: ${{ secrets.SEED_QAA_B2C }}
-          ENABLE_BROADCAST: ${{ contains(inputs.enable_broadcast, 'true') }}
-          SPECULOS_TESTS: ${{ !contains(inputs.speculos_tests, 'false') }}
 
-      - name: upload diffs to s3
-        if: ${{ !cancelled() }}
-        uses: LedgerHQ/ledger-live/tools/actions/upload-images@develop
-        id: s3
-        with:
-          path: apps/ledger-live-desktop/tests/artifacts/test-results
-          workspace: ${{ github.workspace }}
-          os: linux
-          group-name: ${{ github.ref_name }}-${{ github.run_id }}-${{ github.run_number }}
-      - name: upload ci suggested screenshots
+      - name: Upload Allure Results
         if: ${{ !cancelled() }}
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v4.3.0
         with:
-          name: images
-          path: images-linux.json
+          name: allure-results-${{ matrix.shardIndex }}
+          path: "apps/ledger-live-desktop/allure-results"
+
       - name: Upload playwright test results [On Failure]
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v4.3.0
         if: failure() && !cancelled()
         with:
           name: playwright-results-linux
@@ -140,18 +127,23 @@ jobs:
             apps/ledger-live-desktop/tests/artifacts/videos
             apps/ledger-live-desktop/tests/artifacts/logs
             apps/ledger-live-desktop/tests/artifacts/*.log
-      - name: Upload Allure Report
-        if: ${{ !cancelled() }}
-        uses: actions/upload-artifact@v4
+
+  report-and-notify:
+    name: "Report and Notify"
+    runs-on: [ledger-live-medium]
+    needs: e2e-tests-linux
+    if: always()
+    steps:
+      - uses: actions/checkout@v4
         with:
-          name: allure-results-linux
-          path: apps/ledger-live-desktop/allure-results
+          ref: ${{ inputs.ref || github.sha }}
 
-      - name: Upload Allure Results
-        uses: actions/upload-artifact@v4.3.0
+      - name: Download Allure Results
+        uses: actions/download-artifact@v4.1.1
         with:
-          name: "allure-results"
           path: "apps/ledger-live-desktop/allure-results"
+          pattern: allure-results-*
+          merge-multiple: true
 
       - name: Publish report on Allure Server
         id: allure-server
@@ -189,9 +181,9 @@ jobs:
         if: ${{ !cancelled() }}
         shell: bash
         run: >
-          if ${{ steps.tests.outcome == 'success' }};
+          if ${{ needs.e2e-tests-linux.outputs.status == 'success' }};
           then echo "STATUS_COLOR=#33FF39" >> $GITHUB_ENV;
-          elif ${{ steps.tests.outcome == 'failure' }};
+          elif ${{ needs.e2e-tests-linux.outputs.status == 'failure' }};
           then echo "STATUS_COLOR=#FF333C" >> $GITHUB_ENV;
           else echo "STATUS_COLOR=#F3FF33" >> $GITHUB_ENV;
           fi
@@ -227,8 +219,8 @@ jobs:
 
             const report = {
               linux: {
-                pass: ${{ steps.tests.outcome == 'success' }},
-                status: "${{ steps.tests.outcome }}",
+                pass: ${{ needs.e2e-tests-linux.outputs.status == 'success' }},
+                status: "${{ needs.e2e-tests-linux.outputs.status }}",
               }
             };
 

diff --git a/apps/ledger-live-desktop/package.json b/apps/ledger-live-desktop/package.json
@@ -160,7 +160,7 @@
     "@ledgerhq/test-utils": "workspace:^",
     "@ledgerhq/speculos-transport": "workspace:^",
     "@octokit/rest": "^18.12.0",
-    "@playwright/test": "^1.41.2",
+    "@playwright/test": "^1.45.0",
     "@sentry/cli": "2.31.0",
     "@sentry/types": "7.109.0",
     "@testing-library/dom": "^9.3.3",
@@ -193,7 +193,7 @@
     "@types/uuid": "^8.3.4",
     "@types/write-file-atomic": "^4.0.0",
     "@vitejs/plugin-react": "^3.1.0",
-    "allure-playwright": "2.12.2",
+    "allure-playwright": "2.15.1",
     "axios": "1.3.4",
     "chalk": "^4.1.2",
     "cross-env": "^7.0.3",

diff --git a/apps/ledger-live-desktop/tests/component/layout.component.ts b/apps/ledger-live-desktop/tests/component/layout.component.ts
@@ -2,42 +2,42 @@ import { step } from "../misc/reporters/step";
 import { Component } from "tests/page/abstractClasses";
 
 export class Layout extends Component {
-  readonly renderError = this.page.locator("data-test-id=render-error");
-  readonly appVersion = this.page.locator("data-test-id=app-version");
+  readonly renderError = this.page.getByTestId("render-error");
+  readonly appVersion = this.page.getByTestId("app-version");
 
   // portfolio && accounts
-  readonly totalBalance = this.page.locator("data-test-id=total-balance");
+  readonly totalBalance = this.page.getByTestId("total-balance");
 
   // drawer
-  readonly drawerCollapseButton = this.page.locator("data-test-id=drawer-collapse-button");
-  readonly drawerPortfolioButton = this.page.locator("data-test-id=drawer-dashboard-button");
-  private drawerMarketButton = this.page.locator("data-test-id=drawer-market-button");
-  private drawerAccountsButton = this.page.locator("data-test-id=drawer-accounts-button");
-  private drawerDiscoverButton = this.page.locator("data-test-id=drawer-catalog-button");
-  private drawerSendButton = this.page.locator("data-test-id=drawer-send-button");
-  private drawerReceiveButton = this.page.locator("data-test-id=drawer-receive-button");
-  private drawerManagerButton = this.page.locator("data-test-id=drawer-manager-button");
-  private drawerBuycryptoButton = this.page.locator("data-test-id=drawer-exchange-button");
-  private drawerEarnButton = this.page.locator("data-test-id=drawer-earn-button");
-  readonly drawerExperimentalButton = this.page.locator("data-test-id=drawer-experimental-button");
-  private bookmarkedAccountsList = this.page.locator("data-test-id=drawer-bookmarked-accounts");
+  readonly drawerCollapseButton = this.page.getByTestId("drawer-collapse-button");
+  readonly drawerPortfolioButton = this.page.getByTestId("drawer-dashboard-button");
+  private drawerMarketButton = this.page.getByTestId("drawer-market-button");
+  private drawerAccountsButton = this.page.getByTestId("drawer-accounts-button");
+  private drawerDiscoverButton = this.page.getByTestId("drawer-catalog-button");
+  private drawerSendButton = this.page.getByTestId("drawer-send-button");
+  private drawerReceiveButton = this.page.getByTestId("drawer-receive-button");
+  private drawerManagerButton = this.page.getByTestId("drawer-manager-button");
+  private drawerBuycryptoButton = this.page.getByTestId("drawer-exchange-button");
+  private drawerEarnButton = this.page.getByTestId("drawer-earn-button");
+  readonly drawerExperimentalButton = this.page.getByTestId("drawer-experimental-button");
+  private bookmarkedAccountsList = this.page.getByTestId("drawer-bookmarked-accounts");
   readonly bookmarkedAccounts = this.bookmarkedAccountsList.locator(".bookmarked-account-item");
 
   // topbar
-  private topbarDiscreetButton = this.page.locator("data-test-id=topbar-discreet-button");
-  readonly topbarSynchronizeButton = this.page.locator("data-test-id=topbar-synchronize-button");
-  private topbarSettingsButton = this.page.locator("data-test-id=topbar-settings-button");
-  readonly topbarLockButton = this.page.locator("data-test-id=topbar-password-lock-button");
-  readonly topbarHelpButton = this.page.locator("data-test-id=topbar-help-button");
+  private topbarDiscreetButton = this.page.getByTestId("topbar-discreet-button");
+  readonly topbarSynchronizeButton = this.page.getByTestId("topbar-synchronize-button");
+  private topbarSettingsButton = this.page.getByTestId("topbar-settings-button");
+  readonly topbarLockButton = this.page.getByTestId("topbar-password-lock-button");
+  readonly topbarHelpButton = this.page.getByTestId("topbar-help-button");
   private discreetTooltip = this.page.locator("#tippy-12"); // automatically generated tippy id but it's consistent
 
   // general
   readonly inputError = this.page.locator("id=input-error"); // no data-test-id because css style is applied
-  private loadingSpinner = this.page.locator("data-test-id=loading-spinner");
-  readonly logo = this.page.locator("data-test-id=logo");
+  private loadingSpinner = this.page.getByTestId("loading-spinner");
+  readonly logo = this.page.getByTestId("logo");
 
   // updater
-  readonly appUpdateBanner = this.page.locator("data-test-id=layout-app-update-banner");
+  readonly appUpdateBanner = this.page.getByTestId("layout-app-update-banner");
   // }
 
   @step("Go to Portfolio")

diff --git a/apps/ledger-live-desktop/tests/component/modal.component.ts b/apps/ledger-live-desktop/tests/component/modal.component.ts
@@ -5,27 +5,27 @@ export class Modal extends Component {
   readonly container = this.page.locator(
     '[data-test-id=modal-container][style*="opacity: 1"][style*="transform: scale(1)"]',
   );
-  readonly title = this.page.locator("data-test-id=modal-title");
-  readonly content = this.page.locator("data-test-id=modal-content");
-  protected backdrop = this.page.locator("data-test-id=modal-backdrop");
-  protected continueButton = this.page.locator("data-test-id=modal-continue-button");
-  protected saveButton = this.page.locator("data-test-id=modal-save-button");
-  protected cancelButton = this.page.locator("data-test-id=modal-cancel-button");
-  protected confirmButton = this.page.locator("data-test-id=modal-confirm-button");
-  protected closeButton = this.page.locator("data-test-id=modal-close-button");
-  protected backButton = this.page.locator("data-test-id=modal-back-button");
-  protected titleProvider = this.page.locator("data-test-id=modal-provider-title");
-  protected rowProvider = this.page.locator('[data-test-id="modal-provider-row"]');
+  readonly title = this.page.getByTestId("modal-title");
+  readonly content = this.page.getByTestId("modal-content");
+  protected backdrop = this.page.getByTestId("modal-backdrop");
+  protected continueButton = this.page.getByTestId("modal-continue-button");
+  protected saveButton = this.page.getByTestId("modal-save-button");
+  protected cancelButton = this.page.getByTestId("modal-cancel-button");
+  protected confirmButton = this.page.getByTestId("modal-confirm-button");
+  protected closeButton = this.page.getByTestId("modal-close-button");
+  protected backButton = this.page.getByTestId("modal-back-button");
+  protected titleProvider = this.page.getByTestId("modal-provider-title");
+  protected rowProvider = this.page.getByTestId("modal-provider-row");
   protected delegateContinueButton = this.page.locator("id=delegate-continue-button");
-  protected spendableBanner = this.page.locator("data-test-id=modal-spendable-banner");
-  protected maxAmountCheckbox = this.page.locator("data-test-id=modal-max-checkbox");
-  protected cryptoAmountField = this.page.locator("data-test-id=modal-amount-field");
+  protected spendableBanner = this.page.getByTestId("modal-spendable-banner");
+  protected maxAmountCheckbox = this.page.getByTestId("modal-max-checkbox");
+  protected cryptoAmountField = this.page.getByTestId("modal-amount-field");
   protected continueAmountButton = this.page.locator("id=send-amount-continue-button");
   protected searchOpenButton = this.page.getByText("Show all");
   protected searchCloseButton = this.page.getByText("Show less");
   protected inputSearchField = this.page.getByPlaceholder("Search by name or address...");
   protected stakeProviderContainer = (stakeProviderID: string) =>
-    this.page.locator(`data-test-id=stake-provider-container-${stakeProviderID}`);
+    this.page.getByTestId(`stake-provider-container-${stakeProviderID}`);
   protected signContinueButton = this.page.locator("text=Continue");
   protected confirmText = this.page.locator(
     "text=Please confirm the operation on your device to finalize it",

diff --git a/apps/ledger-live-desktop/tests/component/webviewLayout.component.ts b/apps/ledger-live-desktop/tests/component/webviewLayout.component.ts
@@ -1,7 +1,7 @@
 import { Component } from "tests/page/abstractClasses";
 
 export class WebviewLayout extends Component {
-  readonly selectedAccountButton = this.page.locator(
-    '[data-test-id="web-platform-player-topbar-selected-account"]',
+  readonly selectedAccountButton = this.page.getByTestId(
+    "web-platform-player-topbar-selected-account",
   );
 }
diff --git a/apps/ledger-live-desktop/tests/fixtures/common.ts b/apps/ledger-live-desktop/tests/fixtures/common.ts
@@ -16,6 +16,7 @@ import { startSpeculos, stopSpeculos } from "../utils/speculos";
 import { Spec } from "../utils/speculos";
 
 import { allure } from "allure-playwright";
+import { Application } from "tests/page";
 
 export function generateUUID(): string {
   return crypto.randomBytes(16).toString("hex");
@@ -41,6 +42,7 @@ type TestFixtures = {
   featureFlags: OptionalFeatureMap;
   recordTestNamesForApiResponseLogging: void;
   simulateCamera: string;
+  app: Application;
 };
 
 const IS_NOT_MOCK = process.env.MOCK == "0";
@@ -57,6 +59,12 @@ export const test = base.extend<TestFixtures>({
   speculosCurrency: undefined,
   speculosOffset: undefined,
   testName: undefined,
+
+  app: async ({ page }, use) => {
+    const app = new Application(page);
+    await use(app);
+  },
+
   userdataDestinationPath: async ({}, use) => {
     use(path.join(__dirname, "../artifacts/userdata", generateUUID()));
   },

diff --git a/apps/ledger-live-desktop/tests/page/account.page.ts b/apps/ledger-live-desktop/tests/page/account.page.ts
@@ -4,33 +4,32 @@ import { AppPage } from "tests/page/abstractClasses";
 import { Token } from "tests/enum/Tokens";
 
 export class AccountPage extends AppPage {
-  readonly settingsButton = this.page.locator("data-test-id=account-settings-button");
-  private settingsDeleteButton = this.page.locator("data-test-id=account-settings-delete-button");
-  private settingsConfirmButton = this.page.locator("data-test-id=modal-confirm-button");
-  private swapButton = this.page.locator("data-test-id=swap-account-action-button");
-  private buyButton = this.page.locator("data-test-id=buy-button");
-  private sellButton = this.page.locator("data-test-id=sell-button");
-  private stakeButton = this.page.locator("data-test-id=stake-from-account-action-button");
-  private stakeButtonCosmos = this.page.locator("data-test-id=stake-button-cosmos");
-  readonly stakeBanner = this.page.locator("data-test-id=account-stake-banner");
-  private stakeBannerButton = this.page.locator("data-test-id=account-stake-banner-button");
-  private receiveButton = this.page.locator("data-test-id=receive-account-action-button");
+  readonly settingsButton = this.page.getByTestId("account-settings-button");
+  private settingsDeleteButton = this.page.getByTestId("account-settings-delete-button");
+  private settingsConfirmButton = this.page.getByTestId("modal-confirm-button");
+  private swapButton = this.page.getByTestId("swap-account-action-button");
+  private buyButton = this.page.getByTestId("buy-button");
+  private sellButton = this.page.getByTestId("sell-button");
+  private stakeButton = this.page.getByTestId("stake-from-account-action-button");
+  private stakeButtonCosmos = this.page.getByTestId("stake-button-cosmos");
+  readonly stakeBanner = this.page.getByTestId("account-stake-banner");
+  private stakeBannerButton = this.page.getByTestId("account-stake-banner-button");
+  private receiveButton = this.page.getByTestId("receive-account-action-button");
   private sendButton = this.page.getByRole("button", { name: "Send" });
   private accountName = (name: string) => this.page.locator(`text=${name}`);
   private lastOperation = this.page.locator("text=Latest operations");
   private tokenValue = (tokenName: string) =>
-    this.page.locator(`data-test-id=account-row-${tokenName.toLowerCase()}`);
-  private accountBalance = this.page.locator("data-test-id=total-balance");
+    this.page.getByTestId(`account-row-${tokenName.toLowerCase()}`);
+  private accountBalance = this.page.getByTestId("total-balance");
   private operationList = this.page.locator("id=operation-list");
   private showMoreButton = this.page.getByText("Show more");
   private advancedButton = this.page.getByText("Advanced");
-  private accountAdvancedLogs = this.page.locator("data-test-id=Advanced_Logs");
+  private accountAdvancedLogs = this.page.getByTestId("Advanced_Logs");
   private operationRows = this.page.locator("[data-test-id^='operation-row-']");
-  private closeModal = this.page.locator("data-test-id=modal-close-button");
+  private closeModal = this.page.getByTestId("modal-close-button");
   private accountbutton = (accountName: string) =>
     this.page.getByRole("button", { name: `${accountName}` });
-  private tokenRow = (tokenTicker: string) =>
-    this.page.locator(`data-test-id=token-row-${tokenTicker}`);
+  private tokenRow = (tokenTicker: string) => this.page.getByTestId(`token-row-${tokenTicker}`);
   private addTokenButton = this.page.getByRole("button", { name: "Add token" });
 
   @step("Navigate to token $0")

diff --git a/apps/ledger-live-desktop/tests/page/accounts.page.ts b/apps/ledger-live-desktop/tests/page/accounts.page.ts
@@ -3,14 +3,14 @@ import { step } from "tests/misc/reporters/step";
 import { AppPage } from "tests/page/abstractClasses";
 
 export class AccountsPage extends AppPage {
-  private addAccountButton = this.page.locator("data-test-id=accounts-add-account-button");
+  private addAccountButton = this.page.getByTestId("accounts-add-account-button");
   private accountComponent = (accountName: string) =>
-    this.page.locator(`data-test-id=account-component-${accountName}`);
+    this.page.getByTestId(`account-component-${accountName}`);
   private firstAccount = this.page.locator(".accounts-account-row-item").locator("div").first();
   // Accounts context menu
-  private contextMenuEdit = this.page.locator("data-test-id=accounts-context-menu-edit");
-  private settingsDeleteButton = this.page.locator("data-test-id=account-settings-delete-button");
-  private settingsConfirmButton = this.page.locator("data-test-id=modal-confirm-button");
+  private contextMenuEdit = this.page.getByTestId("accounts-context-menu-edit");
+  private settingsDeleteButton = this.page.getByTestId("account-settings-delete-button");
+  private settingsConfirmButton = this.page.getByTestId("modal-confirm-button");
   private accountListNumber = this.page.locator(`[data-test-id^="account-component-"]`);
 
   async openAddAccountModal() {

diff --git a/apps/ledger-live-desktop/tests/page/asset.page.ts b/apps/ledger-live-desktop/tests/page/asset.page.ts
@@ -1,8 +1,8 @@
 import { AppPage } from "tests/page/abstractClasses";
 
 export class AssetPage extends AppPage {
-  private stakeButton = this.page.locator("data-test-id=asset-page-stake-button");
-  private buyButton = this.page.locator("data-test-id=asset-page-buy-button");
+  private stakeButton = this.page.getByTestId("asset-page-stake-button");
+  private buyButton = this.page.getByTestId("asset-page-buy-button");
 
   async startStakeFlow() {
     await this.stakeButton.click();

diff --git a/apps/ledger-live-desktop/tests/page/discover.page.ts b/apps/ledger-live-desktop/tests/page/discover.page.ts
@@ -2,7 +2,7 @@ import { AppPage } from "tests/page/abstractClasses";
 
 export class DiscoverPage extends AppPage {
   private testAppCatalogItem = this.page.locator("#platform-catalog-app-dummy-live-app");
-  private disclaimerTitle = this.page.locator("data-test-id=live-app-disclaimer-drawer-title");
+  private disclaimerTitle = this.page.getByTestId("live-app-disclaimer-drawer-title");
 
   async openTestApp() {
     await this.testAppCatalogItem.click();