devops: balance Playwright shards by duration

test/e2e/package.json

@@ -21,7 +21,7 @@
 		"url": "https://github.com/WordPress/gutenberg/issues"
 	},
 	"devDependencies": {
-		"@flakiness/playwright": "^1.14.0",
+		"@flakiness/playwright": "^2.0.1",
 		"@playwright/test": "^1.62.1",
 		"@types/node": "^20.19.39",
 		"@wordpress/e2e-test-utils-playwright": "file:../../packages/e2e-test-utils-playwright",

test/e2e/playwright.config.ts

@@ -17,6 +17,13 @@ if ( Array.isArray( baseConfig.testIgnore ) ) {
 	baseTestIgnore.push( baseConfig.testIgnore );
 }
 
+const flakinessOptions = {
+	flakinessProject: 'WordPress/gutenberg',
+	shardBalancing: {
+		timingsFile: './timings.json',
+	},
+};
+
 const config = defineConfig( {
 	...baseConfig,
 	webServer: {
@@ -34,14 +41,11 @@ const config = defineConfig( {
 				 */
 				...( process.env.GITHUB_REPOSITORY === 'WordPress/gutenberg'
 					? ( [
-							[
-								'@flakiness/playwright',
-								{ flakinessProject: 'WordPress/gutenberg' },
-							],
+							[ '@flakiness/playwright', flakinessOptions ],
 					  ] as const )
 					: [] ),
 		  ]
-		: 'list',
+		: [ [ 'list' ], [ '@flakiness/playwright', flakinessOptions ] ],
 	workers: 1,
 	globalSetup: fileURLToPath(
 		new URL( './config/global-setup.ts', 'file:' + __filename ).href

test/e2e/specs/admin/user-taxonomies.spec.js

@@ -319,6 +319,14 @@ test.describe( 'User taxonomies', () => {
 		const PLURAL = 'Xyzzies';
 		const SINGULAR = 'Xyzzy';
 
+		// Checkbox assertions below match by state (`checked: true`) instead
+		// of calling `toBeChecked()` on a name-only locator: WP creates a
+		// missing default term inside `register_taxonomy()` on every
+		// request's `init`, so two requests racing right after the taxonomy
+		// record is created can both insert it (same name, deduped slug). A
+		// name-only locator then resolves to two checkboxes and trips
+		// Playwright's strict mode.
+
 		// Outer `afterEach` deletes the wp_user_taxonomy record; this
 		// additionally cleans up the posts the tests publish. The terms
 		// auto-created by WP's default-term mechanism are intentionally
@@ -421,8 +429,11 @@ test.describe( 'User taxonomies', () => {
 			await expandTaxonomyPanel( page );
 			const genresGroup = page.getByRole( 'group', { name: PLURAL } );
 			await expect(
-				genresGroup.getByRole( 'checkbox', { name: 'Default Xyzzy A' } )
-			).toBeChecked();
+				genresGroup.getByRole( 'checkbox', {
+					name: 'Default Xyzzy A',
+					checked: true,
+				} )
+			).toBeVisible();
 
 			// Disable the default-term toggle via the edit page.
 			const taxonomyId = await getTaxonomyId( requestUtils );
@@ -480,8 +491,11 @@ test.describe( 'User taxonomies', () => {
 			await expandTaxonomyPanel( page );
 			const postAGroup = page.getByRole( 'group', { name: PLURAL } );
 			await expect(
-				postAGroup.getByRole( 'checkbox', { name: 'Original Xyzzy' } )
-			).toBeChecked();
+				postAGroup.getByRole( 'checkbox', {
+					name: 'Original Xyzzy',
+					checked: true,
+				} )
+			).toBeVisible();
 
 			// Rename the default term via the edit page.
 			const taxonomyId = await getTaxonomyId( requestUtils );
@@ -512,11 +526,22 @@ test.describe( 'User taxonomies', () => {
 			await expandTaxonomyPanel( page );
 			const postBGroup = page.getByRole( 'group', { name: PLURAL } );
 			await expect(
-				postBGroup.getByRole( 'checkbox', { name: 'Renamed Xyzzy' } )
-			).toBeChecked();
+				postBGroup.getByRole( 'checkbox', {
+					name: 'Renamed Xyzzy',
+					checked: true,
+				} )
+			).toBeVisible();
+			await expect(
+				postBGroup
+					.getByRole( 'checkbox', { name: 'Original Xyzzy' } )
+					.first()
+			).toBeVisible();
 			await expect(
-				postBGroup.getByRole( 'checkbox', { name: 'Original Xyzzy' } )
-			).not.toBeChecked();
+				postBGroup.getByRole( 'checkbox', {
+					name: 'Original Xyzzy',
+					checked: true,
+				} )
+			).toHaveCount( 0 );
 		} );
 	} );
 } );

test/e2e/specs/editor/collaboration/collaboration-selection.spec.ts

@@ -5,6 +5,12 @@ import { test, expect } from './fixtures';
 import { pressKey, LINE_START_KEY } from './fixtures/keyboard-utils';
 
 test.describe( 'Collaboration - Selection Rendering', () => {
+	test.afterEach( async ( { requestUtils } ) => {
+		// Reset preferences via REST so a mid-test failure doesn't leak
+		// the fixed-toolbar setting (or any other pref) to other tests.
+		await requestUtils.resetPreferences();
+	} );
+
 	test( 'Text selection is visible to other users', async ( {
 		collaborationUtils,
 		requestUtils,

test/e2e/specs/editor/collaboration/collaboration-title-reload.spec.ts

@@ -33,6 +33,7 @@ const test = base.extend< Fixtures >( {
 		await setCollaboration( requestUtils, true );
 		await use( utils );
 		await utils.teardown();
+		await setCollaboration( requestUtils, false );
 	},
 	collaboratorUser: async (
 		{ collaborationUtils, requestUtils },

test/e2e/specs/editor/various/preferences.spec.js

@@ -8,6 +8,12 @@ test.describe( 'Preferences', () => {
 		await admin.createNewPost();
 	} );
 
+	test.afterEach( async ( { requestUtils } ) => {
+		// Reset preferences via REST so the dismissed-sidebar preference
+		// doesn't leak into other tests that expect it open by default.
+		await requestUtils.resetPreferences();
+	} );
+
 	test( 'remembers sidebar dismissal between sessions', async ( {
 		editor,
 		page,

test/e2e/specs/editor/various/preview.spec.js

@@ -3,10 +3,21 @@
  */
 const { test, expect } = require( '@wordpress/e2e-test-utils-playwright' );
 
+/**
+ * Internal dependencies
+ */
+const {
+	setCollaboration,
+} = require( '../collaboration/fixtures/collaboration-utils' );
+
 test.use( {
 	previewUtils: async ( { page }, use ) => {
 		await use( new PreviewUtils( { page } ) );
 	},
+	enableCollaboration: async ( { requestUtils }, use ) => {
+		await use( () => setCollaboration( requestUtils, true ) );
+		await setCollaboration( requestUtils, false );
+	},
 } );
 
 test.describe( 'Preview', () => {
@@ -147,9 +158,19 @@ test.describe( 'Preview', () => {
 	// See: https://github.com/WordPress/gutenberg/issues/33758.
 	test( 'should not use stale autosave data after reverting title', async ( {
 		editor,
+		enableCollaboration,
 		page,
 		previewUtils,
 	} ) => {
+		await enableCollaboration();
+		await Promise.all( [
+			page.waitForResponse(
+				( response ) =>
+					response.url().includes( 'wp-sync' ) &&
+					response.status() === 200
+			),
+			page.reload(),
+		] );
 		const editorPage = page;
 
 		// Create and publish a post.

test/e2e/specs/preload/post-editor.spec.js

@@ -10,11 +10,15 @@ const {
 	recordRequests,
 	waitForRequestsToSettle,
 } = require( './record-requests' );
+const {
+	setCollaboration,
+} = require( '../editor/collaboration/fixtures/collaboration-utils' );
 
 test.describe( 'Preload', () => {
 	let postId;
 
 	test.beforeAll( async ( { requestUtils } ) => {
+		await setCollaboration( requestUtils, true );
 		const post = await requestUtils.createPost( {
 			content:
 				'<!-- wp:heading -->\n<h2 class="wp-block-heading">Hello</h2>\n<!-- /wp:heading -->',
@@ -24,7 +28,11 @@ test.describe( 'Preload', () => {
 	} );
 
 	test.afterAll( async ( { requestUtils } ) => {
-		await requestUtils.deleteAllPosts();
+		try {
+			await requestUtils.deleteAllPosts();
+		} finally {
+			await setCollaboration( requestUtils, false );
+		}
 	} );
 
 	test( 'Should fetch a known set of routes during startup', async ( {

test/e2e/specs/site-editor/template-registration.spec.js

@@ -109,6 +109,7 @@ test.describe( 'Block template registration', () => {
 		} );
 
 		// Change template.
+		await editor.openDocumentSettingsSidebar();
 		await page.getByRole( 'button', { name: 'Post', exact: true } ).click();
 		await page.getByRole( 'button', { name: 'Template options' } ).click();
 		await page.getByRole( 'menuitem', { name: 'Change template' } ).click();
@@ -136,6 +137,7 @@ test.describe( 'Block template registration', () => {
 		} );
 
 		// Change template.
+		await editor.openDocumentSettingsSidebar();
 		await page.getByRole( 'button', { name: 'Post', exact: true } ).click();
 		await page.getByRole( 'button', { name: 'Template options' } ).click();
 		await page.getByRole( 'menuitem', { name: 'Change template' } ).click();
@@ -323,6 +325,8 @@ test.describe( 'Block template registration', () => {
 		await page
 			.locator( '.fields-field__title', { hasText: 'Author: Admin' } )
 			.click();
+		// The Actions button lives in the settings sidebar.
+		await editor.openDocumentSettingsSidebar();
 		const actions = page.getByLabel( 'Actions' );
 		await actions.first().click();
 		await page.getByRole( 'menuitem', { name: 'Reset' } ).click();