V10/bugfix/create simple package test

build/azure-pipelines.yml

@@ -444,13 +444,12 @@ stages:
           - pwsh: npx playwright install --with-deps
             displayName: Install Playwright
             workingDirectory: tests/Umbraco.Tests.AcceptanceTest
-          - pwsh: npm run test --ignore-certificate-errors --output $(Build.ArtifactStagingDirectory)\test-results
+          - pwsh: npm run test --ignore-certificate-errors
             displayName: Run Playwright (Desktop)
             continueOnError: true
             workingDirectory: tests/Umbraco.Tests.AcceptanceTest
             env:
               CI: true
-              PLAYWRIGHT_HTML_REPORT: $(Build.ArtifactStagingDirectory)\playwright-report
           - pwsh: |
               docker logs $(dockerImageName) > $(Build.ArtifactStagingDirectory)/playwright.log 2>&1
               docker stop $(dockerImageName)
@@ -459,13 +458,27 @@ stages:
           - pwsh: Stop-Process $env:AcceptanceTestProcessId
             condition: eq(variables['Agent.OS'], 'Windows_NT')
             displayName: Stop app (Windows only)
+          - task: PowerShell@2
+            displayName: Check if artifacts folder exists
+            inputs:
+              targetType: inline
+              script: |
+                $MyVariable = Test-Path -Path $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/results
+                Write-Host "##vso[task.setvariable variable=resultFolderExists;]$MyVariable"
+          - task: CopyFiles@2
+            displayName: Prepare artifacts
+            condition: eq(variables.resultFolderExists, 'True')
+            inputs:
+              sourceFolder: $(Build.SourcesDirectory)/tests/Umbraco.Tests.AcceptanceTest/results/
+              targetFolder: $(Build.ArtifactStagingDirectory)/playwright
           - task: PublishPipelineArtifact@1
             condition: always()
             displayName: Publish test artifacts
             inputs:
               targetPath: $(Build.ArtifactStagingDirectory)
               artifact: 'E2E artifacts - $(Agent.OS) - Attempt #$(System.JobAttempt)'
 
+
   ###############################################
   ## Release
   ###############################################

tests/Umbraco.Tests.AcceptanceTest/package.json

@@ -19,7 +19,7 @@
   },
   "dependencies": {
     "@umbraco/json-models-builders": "^1.0.0",
-    "@umbraco/playwright-testhelpers": "^1.0.2",
+    "@umbraco/playwright-testhelpers": "^1.0.3",
     "camelize": "^1.0.0",
     "faker": "^4.1.0",
     "form-data": "^4.0.0",

tests/Umbraco.Tests.AcceptanceTest/playwright.config.ts

@@ -21,11 +21,11 @@ const config: PlaywrightTestConfig = {
   /* Fail the build on CI if you accidentally left test.only in the source code. */
   forbidOnly: !!process.env.CI,
   /* Retry on CI only */
-  retries: process.env.CI ? 2 : 5,
+  retries: process.env.CI ? 5 : 2,
   // We don't want to run parallel, as tests might differ in state
   workers:  1,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
-  reporter: process.env.CI ? 'html' : 'line',
+  reporter: process.env.CI ? 'line' : 'html',
   outputDir : "./results",
 
   /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
@@ -36,7 +36,7 @@ const config: PlaywrightTestConfig = {
     // baseURL: 'http://localhost:44332',
 
     /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
-    trace: 'on-first-retry',
+    trace: 'retain-on-failure',
     ignoreHTTPSErrors: true,    
   },
 

tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Packages/packages.spec.ts

@@ -89,9 +89,17 @@ test.describe('Packages', () => {
     // Navigate pack to packages and Assert the file is created
     // Waits until the button download is visible
     await page.locator('[label-key="general_download"]').isVisible();
+    // Checks if the packages was created
+    const doesExist = await umbracoApi.packages.doesNameExist(packageName);
+    await expect(doesExist).toBe(true);
     await umbracoUi.goToSection(ConstantHelper.sections.packages);
+
+    // Needs to wait until the page has loaded and the button is clickable
+    await page.locator('[data-element="sub-view-umbCreatedPackages"]').isVisible();
     await page.locator('[data-element="sub-view-umbCreatedPackages"]').click();
-    await expect(await page.locator("body", {hasText: packageName})).toBeVisible();
+    // Asserts that the package can be found in the table
+    await expect(await page.locator('.table-hover')).toHaveCount(1);
+    await expect(await page.locator('.table-hover').first()).toContainText(packageName);
 
     // Cleanup
     await umbracoApi.packages.ensureNameNotExists(packageName);