Skip to content

V11/cherry pick flaky test #13506

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
andr317c merged 2 commits into from
Dec 1, 2022
Merged

V11/cherry pick flaky test #13506

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6f7f50e
V10/feature/fixing flaky acceptance test (#13473)
andr317c Nov 30, 2022
251298e
Bumped version
andr317c Nov 30, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions tests/Umbraco.Tests.AcceptanceTest/package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion tests/Umbraco.Tests.AcceptanceTest/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
},
"dependencies": {
"@umbraco/json-models-builders": "^1.0.2",
"@umbraco/playwright-testhelpers": "^1.0.18",
"@umbraco/playwright-testhelpers": "^1.0.19",
"camelize": "^1.0.0",
"faker": "^4.1.0",
"form-data": "^4.0.0",
Expand Down
5 changes: 3 additions & 2 deletions tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Content/content.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -553,8 +553,9 @@ test.describe('Content tests', () => {

// Save and publish
await umbracoUi.clickElement(umbracoUi.getButtonByLabelKey(ConstantHelper.buttons.saveAndPublish));
await umbracoUi.isSuccessNotificationVisible();

// Added additional time because it could fail on pipeline because it's not saving fast enough
await umbracoUi.isSuccessNotificationVisible({timeout:20000});

// Assert
const expectedContent = '<p>Acceptance test</p>'
await expect(await umbracoApi.content.verifyRenderedContent('/contentpickercontent', expectedContent, true)).toBeTruthy();
Expand Down
4 changes: 2 additions & 2 deletions tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/HelpPanel/systemInformation.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ test.describe('System Information', () => {
await umbracoApi.users.setCurrentLanguage(enCulture);
});

async function openSystemInformation(page: Page, umbracoUi : UiHelpers) {
async function openSystemInformation(page: Page, umbracoUi: UiHelpers) {
//We have to wait for page to load, if the site is slow
await umbracoUi.clickElement(umbracoUi.getGlobalHelp());
await expect(page.locator('.umb-help-list-item').last()).toBeVisible();
Expand Down Expand Up @@ -47,4 +47,4 @@ test.describe('System Information', () => {
// Close the help panel
await page.locator('.umb-button__content').last().click();
});
});
});
107 changes: 53 additions & 54 deletions tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Login/login.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,70 +1,69 @@
import {expect } from '@playwright/test';
import {test} from "@umbraco/playwright-testhelpers";

test.describe('Login', () => {

test.beforeEach(async ({ page, umbracoApi }, testInfo) => {
await umbracoApi.report.report(testInfo);
await page.goto(process.env.URL + '/umbraco');
});
test('Login with correct username and password', async ({page}) => {
test.beforeEach(async ({ page, umbracoApi }, testInfo) => {
await umbracoApi.report.report(testInfo);
await page.goto(process.env.URL + '/umbraco');
});
test('Login with correct username and password', async ({page}) => {

let error = page.locator('.text-error');
await expect(error).toBeHidden();
let error = page.locator('.text-error');
await expect(error).toBeHidden();

// Action
await page.fill('#umb-username', process.env.UMBRACO_USER_LOGIN);
await page.fill('#umb-passwordTwo', process.env.UMBRACO_USER_PASSWORD);
await page.locator('[label-key="general_login"]').click();
await page.waitForNavigation();
// Action
await page.fill('#umb-username', process.env.UMBRACO_USER_LOGIN);
await page.fill('#umb-passwordTwo', process.env.UMBRACO_USER_PASSWORD);
await page.locator('[label-key="general_login"]').click();
await page.waitForNavigation();

// Assert
await expect(page).toHaveURL(process.env.URL + '/umbraco#/content');
let usernameField = await page.locator('#umb-username');
let passwordField = await page.locator('#umb-passwordTwo');
await expect(usernameField).toHaveCount(0);
await expect(passwordField).toHaveCount(0);
});
// Assert
await expect(page).toHaveURL(process.env.URL + '/umbraco#/content');
let usernameField = await page.locator('#umb-username');
let passwordField = await page.locator('#umb-passwordTwo');
await expect(usernameField).toHaveCount(0);
await expect(passwordField).toHaveCount(0);
});

test('Login with correct username but wrong password', async({page}) => {
const username = process.env.UMBRACO_USER_LOGIN;
const password = 'wrong';
test('Login with correct username but wrong password', async ({page}) => {
const username = process.env.UMBRACO_USER_LOGIN;
const password = 'wrong';

// Precondition
let error = page.locator('.text-error');
await expect(error).toBeHidden();
// Precondition
let error = page.locator('.text-error');
await expect(error).toBeHidden();

// Action
await page.fill('#umb-username', username);
await page.fill('#umb-passwordTwo', password);
await page.locator('[label-key="general_login"]').click();
// Action
await page.fill('#umb-username', username);
await page.fill('#umb-passwordTwo', password);
await page.locator('[label-key="general_login"]').click();

// Assert
let usernameField = await page.locator('#umb-username');
let passwordField = await page.locator('#umb-passwordTwo');
await expect(error).toBeVisible();
await expect(usernameField).toBeVisible();
await expect(passwordField).toBeVisible();
});
// Assert
let usernameField = await page.locator('#umb-username');
let passwordField = await page.locator('#umb-passwordTwo');
await expect(error).toBeVisible();
await expect(usernameField).toBeVisible();
await expect(passwordField).toBeVisible();
});

test('Login with wrong username and wrong password', async({page}) => {
const username = 'wrong-username';
const password = 'wrong';
test('Login with wrong username and wrong password', async ({page}) => {
const username = 'wrong-username';
const password = 'wrong';

// Precondition
let error = page.locator('.text-error');
await expect(error).toBeHidden();
// Precondition
let error = page.locator('.text-error');
await expect(error).toBeHidden();

// Action
await page.fill('#umb-username', username);
await page.fill('#umb-passwordTwo', password);
await page.locator('[label-key="general_login"]').click();
// Action
await page.fill('#umb-username', username);
await page.fill('#umb-passwordTwo', password);
await page.locator('[label-key="general_login"]').click();

// Assert
let usernameField = await page.locator('#umb-username');
let passwordField = await page.locator('#umb-passwordTwo');
await expect(error).toBeVisible();
await expect(usernameField).toBeVisible();
await expect(passwordField).toBeVisible();
});
// Assert
let usernameField = await page.locator('#umb-username');
let passwordField = await page.locator('#umb-passwordTwo');
await expect(error).toBeVisible();
await expect(usernameField).toBeVisible();
await expect(passwordField).toBeVisible();
});
});
2 changes: 1 addition & 1 deletion tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/Media/mediaSection.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ test.describe('Media', () => {

// Assert
// Needs to wait before refreshing the media tree, otherwise the media files wont be moved to the folder yet
await page.waitForTimeout(1000);
await page.waitForTimeout(2500);
await umbracoUi.refreshMediaTree();
await page.locator('[data-element="tree-item-' + folderToMoveTooName + '"]').click();
for (const names of mediaFileTypes) {
Expand Down
48 changes: 23 additions & 25 deletions tests/Umbraco.Tests.AcceptanceTest/tests/DefaultConfig/ModelsBuilder/modelsbuilder.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ test.describe('Modelsbuilder tests', () => {
.done()
.done()
.build();

await umbracoApi.documentTypes.save(docType);

await umbracoApi.templates.edit(docTypeName, `@using Umbraco.Cms.Web.Common.PublishedModels;
@inherits Umbraco.Cms.Web.Common.Views.UmbracoViewPage<ContentModels.Testdocument>
@using ContentModels = Umbraco.Cms.Web.Common.PublishedModels;
Expand All @@ -47,9 +47,9 @@ test.describe('Modelsbuilder tests', () => {
await umbracoUi.createContentWithDocumentType(docTypeName);
await umbracoUi.setEditorHeaderName(contentName);
// Fortunately for us the input field of a text box has the alias of the property as an id :)
await page.locator("#title").type("Hello world!")
await page.locator("#title").type("Hello world!");
await umbracoUi.clickElement(umbracoUi.getButtonByLabelKey(ConstantHelper.buttons.saveAndPublish));
await umbracoUi.isSuccessNotificationVisible();
await umbracoUi.isSuccessNotificationVisible({timeout:10000});
// Ensure that we can render it on the frontend = we can compile the models and views
await umbracoApi.content.verifyRenderedContent("/", "<h1>Hello world!</h1>", true);

Expand All @@ -62,7 +62,7 @@ test.describe('Modelsbuilder tests', () => {
const docTypeName = "TestDocument";
const docTypeAlias = AliasHelper.toAlias(docTypeName);
const propertyAlias = "title";
const propertyValue = "Hello world!"
const propertyValue = "Hello world!";

await umbracoApi.content.deleteAllContent();
await umbracoApi.documentTypes.ensureNameNotExists(docTypeName);
Expand All @@ -80,8 +80,8 @@ test.describe('Modelsbuilder tests', () => {
.done()
.done()
.build();

const savedDocType = await umbracoApi.documentTypes.save(docType);

await umbracoApi.templates.edit(docTypeName, `@using Umbraco.Cms.Web.Common.PublishedModels;
@inherits Umbraco.Cms.Web.Common.Views.UmbracoViewPage<ContentModels.Testdocument>
@using ContentModels = Umbraco.Cms.Web.Common.PublishedModels;
Expand All @@ -90,7 +90,6 @@ test.describe('Modelsbuilder tests', () => {
}

<h1>@Model.Title</h1>`);

const content = new ContentBuilder()
.withContentTypeAlias(savedDocType["alias"])
.withAction("publishNew")
Expand All @@ -104,7 +103,6 @@ test.describe('Modelsbuilder tests', () => {
.done()
.done()
.build()

await umbracoApi.content.save(content);

// Navigate to the document type
Expand All @@ -118,7 +116,8 @@ test.describe('Modelsbuilder tests', () => {
await page.locator('.umb-card-grid >> [title="Textstring"]').click();
await umbracoUi.clickElement(umbracoUi.getButtonByLabelKey(ConstantHelper.buttons.submit));
await umbracoUi.clickElement(umbracoUi.getButtonByLabelKey(ConstantHelper.buttons.save));
await umbracoUi.isSuccessNotificationVisible();
// Has a long timeout because it can sometimes take longer than 5 sec to save on the pipeline
await umbracoUi.isSuccessNotificationVisible({timeout:10000});

// Now that the content is updated and the models are rebuilt, ensure that we can still render the frontend.
await umbracoApi.content.verifyRenderedContent("/", "<h1>" + propertyValue + "</h1>", true)
Expand All @@ -132,7 +131,7 @@ test.describe('Modelsbuilder tests', () => {
const docTypeName = "TestDocument";
const docTypeAlias = AliasHelper.toAlias(docTypeName);
const propertyAlias = "title";
const propertyValue = "Hello world!"
const propertyValue = "Hello world!";

await umbracoApi.content.deleteAllContent();
await umbracoApi.documentTypes.ensureNameNotExists(docTypeName);
Expand All @@ -150,8 +149,8 @@ test.describe('Modelsbuilder tests', () => {
.done()
.done()
.build();

const savedDocType = await umbracoApi.documentTypes.save(docType);

await umbracoApi.templates.edit(docTypeName, `@using Umbraco.Cms.Web.Common.PublishedModels;
@inherits Umbraco.Cms.Web.Common.Views.UmbracoViewPage<ContentModels.Testdocument>
@using ContentModels = Umbraco.Cms.Web.Common.PublishedModels;
Expand All @@ -173,8 +172,7 @@ test.describe('Modelsbuilder tests', () => {
.withValue(propertyValue)
.done()
.done()
.build()

.build();
await umbracoApi.content.save(content);

// Navigate to the document type
Expand All @@ -183,11 +181,12 @@ test.describe('Modelsbuilder tests', () => {
const editor = await page.locator('.ace_content');
await editor.click();
// We only have to type out the opening tag, the editor adds the closing tag automatically.
await editor.type("<p>Edited")
await umbracoUi.clickElement(umbracoUi.getButtonByLabelKey(ConstantHelper.buttons.save))
await editor.type("<p>Edited");
await umbracoUi.clickElement(umbracoUi.getButtonByLabelKey(ConstantHelper.buttons.save));

await umbracoUi.isSuccessNotificationVisible({timeout:10000});

await umbracoUi.isSuccessNotificationVisible();
await umbracoApi.content.verifyRenderedContent("/", "<h1>" + propertyValue + "</h1><p>Edited</p>", true)
await umbracoApi.content.verifyRenderedContent("/", "<h1>" + propertyValue + "</h1><p>Edited</p>", true);

await umbracoApi.content.deleteAllContent();
await umbracoApi.documentTypes.ensureNameNotExists(docTypeName);
Expand All @@ -198,7 +197,7 @@ test.describe('Modelsbuilder tests', () => {
const docTypeName = "TestDocument";
const docTypeAlias = AliasHelper.toAlias(docTypeName);
const propertyAlias = "title";
const propertyValue = "Hello world!"
const propertyValue = "Hello world!";
const contentName = "Home";

await umbracoApi.content.deleteAllContent();
Expand All @@ -217,8 +216,8 @@ test.describe('Modelsbuilder tests', () => {
.done()
.done()
.build();

const savedDocType = await umbracoApi.documentTypes.save(docType);

await umbracoApi.templates.edit(docTypeName, `@using Umbraco.Cms.Web.Common.PublishedModels;
@inherits Umbraco.Cms.Web.Common.Views.UmbracoViewPage<ContentModels.Testdocument>
@using ContentModels = Umbraco.Cms.Web.Common.PublishedModels;
Expand All @@ -240,8 +239,7 @@ test.describe('Modelsbuilder tests', () => {
.withValue(propertyValue)
.done()
.done()
.build()

.build();
await umbracoApi.content.save(content);

// Navigate to the document type
Expand All @@ -263,22 +261,22 @@ test.describe('Modelsbuilder tests', () => {
const editor = await page.locator('.ace_content');
await editor.click();
// We only have to type out the opening tag, the editor adds the closing tag automatically.
await editor.type("<p>@Model.Bod")
await umbracoUi.clickElement(umbracoUi.getButtonByLabelKey(ConstantHelper.buttons.save))
await umbracoUi.isSuccessNotificationVisible();
await editor.type("<p>@Model.Bod");
await umbracoUi.clickElement(umbracoUi.getButtonByLabelKey(ConstantHelper.buttons.save));
await umbracoUi.isSuccessNotificationVisible({timeout: 10000});
await page.locator('span:has-text("×")').click();

// Navigate to the content section and update the content
await umbracoUi.goToSection(ConstantHelper.sections.content);
await umbracoUi.refreshContentTree();
await umbracoUi.clickElement(umbracoUi.getTreeItem("content", [contentName]));
await page.locator("#bod").type("Fancy body text");
await umbracoUi.clickElement(umbracoUi.getButtonByLabelKey(ConstantHelper.buttons.saveAndPublish))
await umbracoUi.clickElement(umbracoUi.getButtonByLabelKey(ConstantHelper.buttons.saveAndPublish));

await umbracoApi.content.verifyRenderedContent("/", "<h1>" + propertyValue + "</h1><p>Fancy body text</p>", true);

await umbracoApi.content.deleteAllContent();
await umbracoApi.documentTypes.ensureNameNotExists(docTypeName);
await umbracoApi.templates.ensureNameNotExists(docTypeName)
await umbracoApi.templates.ensureNameNotExists(docTypeName);
});
});
Loading