diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ee984f8f..731bb790 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -65,6 +65,13 @@ jobs: cd main npm run test:ci + - name: Upload test failure screenshots + if: always() + uses: actions/upload-artifact@v5 + with: + name: test-failure-screenshots-${{ matrix.os }} + path: main/test/webdriverio/test/failures + webdriverio_tests: name: WebdriverIO tests (against pinned v12) # Don't run pinned version checks for PRs. diff --git a/.gitignore b/.gitignore index b4725784..39db7fb0 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ build-debug.log node_modules/ build/ dist/ +test/webdriverio/test/failures/ diff --git a/package.json b/package.json index 663b312d..51741d0d 100644 --- a/package.json +++ b/package.json @@ -22,9 +22,9 @@ "wdio:build": "npm run wdio:build:app && npm run wdio:build:tests", "wdio:build:app": "cd test/webdriverio && webpack", "wdio:build:tests": "tsc -p ./test/webdriverio/test/tsconfig.json", - "wdio:clean": "cd test/webdriverio/test && rm -rf dist", - "wdio:run": "npm run wdio:build && cd test/webdriverio/test && npx mocha dist", - "wdio:run:ci": "npm run wdio:build && cd test/webdriverio/test && npx mocha --timeout 30000 dist" + "wdio:clean": "cd test/webdriverio/test && rm -rf dist && rm -rf failures", + "wdio:run": "npm run wdio:build && cd test/webdriverio/test && mkdir -p failures && npx mocha dist", + "wdio:run:ci": "npm run wdio:build && cd test/webdriverio/test && mkdir -p failures && npx mocha --timeout 30000 dist" }, "main": "./dist/index.js", "module": "./src/index.js", diff --git a/test/webdriverio/test/actions_test.ts b/test/webdriverio/test/actions_test.ts index 56c5b85b..557c6dc6 100644 --- a/test/webdriverio/test/actions_test.ts +++ b/test/webdriverio/test/actions_test.ts @@ -20,6 +20,8 @@ import { sendKeyAndWait, keyRight, contextMenuItems, + checkForFailures, + pause, } from './test_setup.js'; const isDarwin = process.platform === 'darwin'; @@ -96,13 +98,21 @@ suite('Menus test', function () { testFileLocations.MORE_BLOCKS, this.timeout(), ); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); + }); + + teardown(async function () { + await checkForFailures( + this.browser, + this.currentTest?.title, + this.currentTest?.state, + ); }); test('Menu action via keyboard on block opens menu', async function () { // Navigate to draw_circle_1. await focusOnBlock(this.browser, 'draw_circle_1'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await sendKeyAndWait(this.browser, [Key.Ctrl, Key.Return]); chai.assert.deepEqual( @@ -126,7 +136,7 @@ suite('Menus test', function () { await focusOnBlock(this.browser, 'text_print_1'); await this.browser.keys(Key.ArrowRight); await this.browser.keys([Key.Ctrl, Key.Return]); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); chai.assert.deepEqual( await contextMenuItems(this.browser), @@ -154,9 +164,9 @@ suite('Menus test', function () { await moveToToolboxCategory(this.browser, 'Math'); // Move to flyout. await keyRight(this.browser); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await rightClickOnFlyoutBlockType(this.browser, 'math_number'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); chai.assert.deepEqual( await contextMenuItems(this.browser), @@ -192,11 +202,11 @@ suite('Menus test', function () { test('Escape key dismisses menu', async function () { await tabNavigateToWorkspace(this.browser); await focusOnBlock(this.browser, 'draw_circle_1'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await this.browser.keys([Key.Ctrl, Key.Return]); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await this.browser.keys(Key.Escape); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); chai.assert.isTrue( await contextMenuExists(this.browser, 'Duplicate', /* reverse= */ true), @@ -207,9 +217,9 @@ suite('Menus test', function () { test('Clicking workspace dismisses menu', async function () { await tabNavigateToWorkspace(this.browser); await clickBlock(this.browser, 'draw_circle_1', {button: 'right'}); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await focusWorkspace(this.browser); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); chai.assert.isTrue( await contextMenuExists(this.browser, 'Duplicate', /* reverse= */ true), diff --git a/test/webdriverio/test/basic_test.ts b/test/webdriverio/test/basic_test.ts index 276f6835..ec228db6 100644 --- a/test/webdriverio/test/basic_test.ts +++ b/test/webdriverio/test/basic_test.ts @@ -22,6 +22,8 @@ import { keyRight, keyUp, keyDown, + checkForFailures, + pause, } from './test_setup.js'; import {Key} from 'webdriverio'; @@ -37,6 +39,14 @@ suite('Keyboard navigation on Blocks', function () { ); }); + teardown(async function () { + await checkForFailures( + this.browser, + this.currentTest?.title, + this.currentTest?.state, + ); + }); + test('Default workspace', async function () { const blockCount = await this.browser.execute(() => { return Blockly.getMainWorkspace().getAllBlocks(false).length; @@ -47,7 +57,7 @@ suite('Keyboard navigation on Blocks', function () { test('Selected block', async function () { await tabNavigateToWorkspace(this.browser); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await keyDown(this.browser, 14); @@ -58,7 +68,7 @@ suite('Keyboard navigation on Blocks', function () { test('Down from statement block selects next block across stacks', async function () { await focusOnBlock(this.browser, 'p5_canvas_1'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await keyDown(this.browser); chai @@ -68,7 +78,7 @@ suite('Keyboard navigation on Blocks', function () { test('Up from statement block selects previous block', async function () { await focusOnBlock(this.browser, 'simple_circle_1'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await keyUp(this.browser); chai @@ -78,7 +88,7 @@ suite('Keyboard navigation on Blocks', function () { test('Down from parent block selects first child block', async function () { await focusOnBlock(this.browser, 'p5_setup_1'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await keyDown(this.browser); chai .expect(await getCurrentFocusedBlockId(this.browser)) @@ -87,7 +97,7 @@ suite('Keyboard navigation on Blocks', function () { test('Up from child block selects parent block', async function () { await focusOnBlock(this.browser, 'p5_canvas_1'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await keyUp(this.browser); chai .expect(await getCurrentFocusedBlockId(this.browser)) @@ -96,7 +106,7 @@ suite('Keyboard navigation on Blocks', function () { test('Right from block selects first field', async function () { await focusOnBlock(this.browser, 'p5_canvas_1'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await keyRight(this.browser); chai @@ -108,7 +118,7 @@ suite('Keyboard navigation on Blocks', function () { test('Right from block selects first inline input', async function () { await focusOnBlock(this.browser, 'simple_circle_1'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await keyRight(this.browser); chai.assert.equal( @@ -119,7 +129,7 @@ suite('Keyboard navigation on Blocks', function () { test('Up from inline input selects statement block', async function () { await focusOnBlock(this.browser, 'math_number_2'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await keyUp(this.browser); chai.assert.equal( @@ -130,7 +140,7 @@ suite('Keyboard navigation on Blocks', function () { test('Left from first inline input selects block', async function () { await focusOnBlock(this.browser, 'math_number_2'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await keyLeft(this.browser); chai.assert.equal( @@ -141,7 +151,7 @@ suite('Keyboard navigation on Blocks', function () { test('Right from first inline input selects second inline input', async function () { await focusOnBlock(this.browser, 'math_number_2'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await keyRight(this.browser); chai.assert.equal( @@ -152,7 +162,7 @@ suite('Keyboard navigation on Blocks', function () { test('Left from second inline input selects first inline input', async function () { await focusOnBlock(this.browser, 'math_number_3'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await keyLeft(this.browser); chai.assert.equal( @@ -163,7 +173,7 @@ suite('Keyboard navigation on Blocks', function () { test('Right from last inline input selects next block', async function () { await focusOnBlock(this.browser, 'colour_picker_1'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await keyRight(this.browser); chai @@ -173,7 +183,7 @@ suite('Keyboard navigation on Blocks', function () { test('Down from inline input selects next block', async function () { await focusOnBlock(this.browser, 'colour_picker_1'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await keyDown(this.browser); chai @@ -183,7 +193,7 @@ suite('Keyboard navigation on Blocks', function () { test("Down from inline input selects block's child block", async function () { await focusOnBlock(this.browser, 'logic_boolean_1'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await keyDown(this.browser); chai @@ -193,7 +203,7 @@ suite('Keyboard navigation on Blocks', function () { test('Right from text block selects shadow block then field', async function () { await focusOnBlock(this.browser, 'text_print_1'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await keyRight(this.browser); chai.assert.equal(await getCurrentFocusedBlockId(this.browser), 'text_1'); @@ -235,9 +245,17 @@ suite('Keyboard navigation on Fields', function () { ); }); + teardown(async function () { + await checkForFailures( + this.browser, + this.currentTest?.title, + this.currentTest?.state, + ); + }); + test('Up from first field selects block', async function () { await focusOnBlockField(this.browser, 'p5_canvas_1', 'WIDTH'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await keyUp(this.browser); chai.assert.equal( @@ -248,7 +266,7 @@ suite('Keyboard navigation on Fields', function () { test('Left from first field selects block', async function () { await focusOnBlockField(this.browser, 'p5_canvas_1', 'WIDTH'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await keyLeft(this.browser); chai.assert.equal( @@ -259,7 +277,7 @@ suite('Keyboard navigation on Fields', function () { test('Right from first field selects second field', async function () { await focusOnBlockField(this.browser, 'p5_canvas_1', 'WIDTH'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await keyRight(this.browser); chai @@ -271,7 +289,7 @@ suite('Keyboard navigation on Fields', function () { test('Left from second field selects first field', async function () { await focusOnBlockField(this.browser, 'p5_canvas_1', 'HEIGHT'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await keyLeft(this.browser); chai @@ -283,7 +301,7 @@ suite('Keyboard navigation on Fields', function () { test('Right from second field selects next block', async function () { await focusOnBlockField(this.browser, 'p5_canvas_1', 'HEIGHT'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await keyRight(this.browser); chai @@ -293,7 +311,7 @@ suite('Keyboard navigation on Fields', function () { test('Down from field selects next block', async function () { await focusOnBlockField(this.browser, 'p5_canvas_1', 'WIDTH'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await keyDown(this.browser); chai @@ -303,7 +321,7 @@ suite('Keyboard navigation on Fields', function () { test("Down from field selects block's child block", async function () { await focusOnBlockField(this.browser, 'controls_repeat_1', 'TIMES'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await keyDown(this.browser); chai @@ -314,7 +332,7 @@ suite('Keyboard navigation on Fields', function () { test('Do not navigate while field editor is open', async function () { // Open a field editor dropdown await focusOnBlockField(this.browser, 'logic_boolean_1', 'BOOL'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await sendKeyAndWait(this.browser, Key.Enter); // Try to navigate to a different block @@ -327,7 +345,7 @@ suite('Keyboard navigation on Fields', function () { test('Do not reopen field editor when handling enter to make a choice inside the editor', async function () { // Open colour picker await focusOnBlockField(this.browser, 'colour_picker_1', 'COLOUR'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await sendKeyAndWait(this.browser, Key.Enter); // Move right to pick a new colour. diff --git a/test/webdriverio/test/block_comment_test.ts b/test/webdriverio/test/block_comment_test.ts index f4c50169..aa5feca3 100644 --- a/test/webdriverio/test/block_comment_test.ts +++ b/test/webdriverio/test/block_comment_test.ts @@ -14,6 +14,7 @@ import { testFileLocations, keyRight, PAUSE_TIME, + checkForFailures, } from './test_setup.js'; import {Key} from 'webdriverio'; @@ -34,6 +35,14 @@ suite('Block comment navigation', function () { }); }); + teardown(async function () { + await checkForFailures( + this.browser, + this.currentTest?.title, + this.currentTest?.state, + ); + }); + test('Activating a block comment icon focuses the comment', async function () { await focusOnBlock(this.browser, 'p5_canvas_1'); await keyRight(this.browser); diff --git a/test/webdriverio/test/clipboard_test.ts b/test/webdriverio/test/clipboard_test.ts index b204a2ee..870a1bb4 100644 --- a/test/webdriverio/test/clipboard_test.ts +++ b/test/webdriverio/test/clipboard_test.ts @@ -18,6 +18,8 @@ import { blockIsPresent, getFocusedBlockType, sendKeyAndWait, + checkForFailures, + pause, } from './test_setup.js'; import {Key, KeyAction, PointerAction, WheelAction} from 'webdriverio'; @@ -28,7 +30,15 @@ suite('Clipboard test', function () { // Clear the workspace and load start blocks. setup(async function () { this.browser = await testSetup(testFileLocations.BASE, this.timeout()); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); + }); + + teardown(async function () { + await checkForFailures( + this.browser, + this.currentTest?.title, + this.currentTest?.state, + ); }); test('Copy and paste while block selected', async function () { @@ -112,7 +122,7 @@ suite('Clipboard test', function () { test('Do not cut block while field editor is open', async function () { // Open a field editor await focusOnBlockField(this.browser, 'draw_circle_1_color', 'COLOUR'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await sendKeyAndWait(this.browser, Key.Enter); // Try to cut block while field editor is open @@ -166,7 +176,7 @@ async function performActionWhileDraggingBlock( .move(blockX, blockY), action, ]); - await browser.pause(PAUSE_TIME); + await pause(browser); } /** diff --git a/test/webdriverio/test/delete_test.ts b/test/webdriverio/test/delete_test.ts index a379c273..433fef2e 100644 --- a/test/webdriverio/test/delete_test.ts +++ b/test/webdriverio/test/delete_test.ts @@ -18,6 +18,8 @@ import { sendKeyAndWait, keyRight, focusOnBlockField, + checkForFailures, + pause, } from './test_setup.js'; import {Key} from 'webdriverio'; @@ -31,12 +33,20 @@ suite('Deleting Blocks', function () { testFileLocations.NAVIGATION_TEST_BLOCKS, this.timeout(), ); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); + }); + + teardown(async function () { + await checkForFailures( + this.browser, + this.currentTest?.title, + this.currentTest?.state, + ); }); test('Deleting block selects parent block', async function () { await focusOnBlock(this.browser, 'controls_if_2'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); chai .expect(await blockIsPresent(this.browser, 'controls_if_2')) @@ -55,7 +65,7 @@ suite('Deleting Blocks', function () { test('Cutting block selects parent block', async function () { await focusOnBlock(this.browser, 'controls_if_2'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); chai .expect(await blockIsPresent(this.browser, 'controls_if_2')) @@ -74,7 +84,7 @@ suite('Deleting Blocks', function () { test('Deleting block also deletes children and inputs', async function () { await focusOnBlock(this.browser, 'controls_if_2'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); chai .expect(await blockIsPresent(this.browser, 'logic_boolean_1')) @@ -93,7 +103,7 @@ suite('Deleting Blocks', function () { test('Cutting block also removes children and inputs', async function () { await focusOnBlock(this.browser, 'controls_if_2'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); chai .expect(await blockIsPresent(this.browser, 'logic_boolean_1')) @@ -112,7 +122,7 @@ suite('Deleting Blocks', function () { test('Deleting inline input selects parent block', async function () { await focusOnBlock(this.browser, 'logic_boolean_1'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); chai .expect(await blockIsPresent(this.browser, 'logic_boolean_1')) @@ -131,7 +141,7 @@ suite('Deleting Blocks', function () { test('Cutting inline input selects parent block', async function () { await focusOnBlock(this.browser, 'logic_boolean_1'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); chai .expect(await blockIsPresent(this.browser, 'logic_boolean_1')) @@ -155,11 +165,11 @@ suite('Deleting Blocks', function () { // We want deleting a block to focus the workspace, whatever that // means at the time. await tabNavigateToWorkspace(this.browser); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); // The test workspace doesn't already contain a stranded block, so add one. await moveToToolboxCategory(this.browser, 'Math'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); // Move to flyout. await keyRight(this.browser); // Select number block. @@ -179,11 +189,11 @@ suite('Deleting Blocks', function () { test('Cutting stranded block selects top block', async function () { await tabNavigateToWorkspace(this.browser); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); // The test workspace doesn't already contain a stranded block, so add one. await moveToToolboxCategory(this.browser, 'Math'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); // Move to flyout. await keyRight(this.browser); // Select number block. @@ -204,7 +214,7 @@ suite('Deleting Blocks', function () { test('Do not delete block while field editor is open', async function () { // Open a field editor await focusOnBlockField(this.browser, 'colour_picker_1', 'COLOUR'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await sendKeyAndWait(this.browser, Key.Enter); // Try to delete block while field editor is open diff --git a/test/webdriverio/test/duplicate_test.ts b/test/webdriverio/test/duplicate_test.ts index 879ef964..eaeeb48d 100644 --- a/test/webdriverio/test/duplicate_test.ts +++ b/test/webdriverio/test/duplicate_test.ts @@ -15,6 +15,8 @@ import { testFileLocations, testSetup, sendKeyAndWait, + checkForFailures, + pause, } from './test_setup.js'; suite('Duplicate test', function () { @@ -24,7 +26,15 @@ suite('Duplicate test', function () { // Clear the workspace and load start blocks. setup(async function () { this.browser = await testSetup(testFileLocations.BASE, this.timeout()); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); + }); + + teardown(async function () { + await checkForFailures( + this.browser, + this.currentTest?.title, + this.currentTest?.state, + ); }); test('Duplicate block', async function () { @@ -61,7 +71,7 @@ suite('Duplicate test', function () { (workspace as Blockly.WorkspaceSvg).getTopComments()[0], ); }, text); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); // Duplicate. await sendKeyAndWait(this.browser, 'd'); diff --git a/test/webdriverio/test/flyout_test.ts b/test/webdriverio/test/flyout_test.ts index 4e438f6d..ad29017c 100644 --- a/test/webdriverio/test/flyout_test.ts +++ b/test/webdriverio/test/flyout_test.ts @@ -20,6 +20,9 @@ import { getCurrentFocusNodeId, getCurrentFocusedBlockId, tabNavigateToToolbox, + checkForFailures, + pause, + setSynchronizeCoreBlocklyRendering, } from './test_setup.js'; suite('Toolbox and flyout test', function () { @@ -29,7 +32,15 @@ suite('Toolbox and flyout test', function () { // Clear the workspace and load start blocks. setup(async function () { this.browser = await testSetup(testFileLocations.BASE, this.timeout()); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); + }); + + teardown(async function () { + await checkForFailures( + this.browser, + this.currentTest?.title, + this.currentTest?.state, + ); }); test('Tab navigating to toolbox should open flyout', async function () { @@ -144,7 +155,7 @@ suite('Toolbox and flyout test', function () { test('Tabbing to the workspace should close the flyout', async function () { await tabNavigateToWorkspace(this.browser); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); // The flyout should be closed since it lost focus. const flyoutIsOpen = await checkIfFlyoutIsOpen(this.browser); @@ -291,8 +302,11 @@ suite('Toolbox and flyout test', function () { } }); test('callbackkey is activated with enter', async function () { + // Rendering synchronization must be disabled since this test opens an + // alert dialog. See the function's documentation for more specifics. + setSynchronizeCoreBlocklyRendering(false); await tabNavigateToToolbox(this.browser); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); // First thing in the toolbox is the first button // Press Enter to activate it. @@ -304,8 +318,11 @@ suite('Toolbox and flyout test', function () { }); test('callbackKey is activated with enter', async function () { + // Rendering synchronization must be disabled since this test opens an + // alert dialog. See the function's documentation for more specifics. + setSynchronizeCoreBlocklyRendering(false); await tabNavigateToToolbox(this.browser); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); // Navigate to second button. // Press Enter to activate it. diff --git a/test/webdriverio/test/insert_test.ts b/test/webdriverio/test/insert_test.ts index 6517cb46..c7000934 100644 --- a/test/webdriverio/test/insert_test.ts +++ b/test/webdriverio/test/insert_test.ts @@ -22,6 +22,8 @@ import { blockIsPresent, keyUp, tabNavigateToToolbox, + checkForFailures, + pause, } from './test_setup.js'; suite('Insert test', function () { @@ -31,7 +33,15 @@ suite('Insert test', function () { // Clear the workspace and load start blocks. setup(async function () { this.browser = await testSetup(testFileLocations.BASE, this.timeout()); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); + }); + + teardown(async function () { + await checkForFailures( + this.browser, + this.currentTest?.title, + this.currentTest?.state, + ); }); test('Insert and cancel with block selection', async function () { @@ -145,7 +155,15 @@ suite('Insert test with more blocks', function () { testFileLocations.MORE_BLOCKS, this.timeout(), ); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); + }); + + teardown(async function () { + await checkForFailures( + this.browser, + this.currentTest?.title, + this.currentTest?.state, + ); }); test('Does not bump immovable input blocks on insert', async function () { diff --git a/test/webdriverio/test/keyboard_mode_test.ts b/test/webdriverio/test/keyboard_mode_test.ts index 1c0f4d8e..84c52f7b 100644 --- a/test/webdriverio/test/keyboard_mode_test.ts +++ b/test/webdriverio/test/keyboard_mode_test.ts @@ -15,6 +15,8 @@ import { tabNavigateToWorkspace, clickBlock, sendKeyAndWait, + checkForFailures, + pause, } from './test_setup.js'; import {Key} from 'webdriverio'; @@ -37,7 +39,7 @@ suite( this.timeout(), ); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); // Reset the keyboard navigation state between tests. await this.browser.execute(() => { @@ -48,8 +50,16 @@ suite( await tabNavigateToWorkspace(this.browser); }); + teardown(async function () { + await checkForFailures( + this.browser, + this.currentTest?.title, + this.currentTest?.state, + ); + }); + test('T to open toolbox enables keyboard mode', async function () { - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await sendKeyAndWait(this.browser, 't'); chai.assert.isTrue(await isKeyboardNavigating(this.browser)); @@ -57,14 +67,14 @@ suite( test('M for move mode enables keyboard mode', async function () { await focusOnBlock(this.browser, 'controls_if_2'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await sendKeyAndWait(this.browser, 'm'); chai.assert.isTrue(await isKeyboardNavigating(this.browser)); }); test('W for workspace cursor enables keyboard mode', async function () { - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await sendKeyAndWait(this.browser, 'w'); chai.assert.isTrue(await isKeyboardNavigating(this.browser)); @@ -72,7 +82,7 @@ suite( test('X to disconnect enables keyboard mode', async function () { await focusOnBlock(this.browser, 'controls_if_2'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await sendKeyAndWait(this.browser, 'x'); chai.assert.isTrue(await isKeyboardNavigating(this.browser)); @@ -81,7 +91,7 @@ suite( test('Copy does not change keyboard mode state', async function () { // Make sure we're on a copyable block so that copy occurs await focusOnBlock(this.browser, 'controls_if_2'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await sendKeyAndWait(this.browser, [Key.Ctrl, 'c']); chai.assert.isFalse(await isKeyboardNavigating(this.browser)); @@ -90,7 +100,7 @@ suite( Blockly.keyboardNavigationController.setIsActive(true); }); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await sendKeyAndWait(this.browser, [Key.Ctrl, 'c']); chai.assert.isTrue(await isKeyboardNavigating(this.browser)); @@ -99,7 +109,7 @@ suite( test('Delete does not change keyboard mode state', async function () { // Make sure we're on a deletable block so that delete occurs await focusOnBlock(this.browser, 'controls_if_2'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await sendKeyAndWait(this.browser, Key.Backspace); chai.assert.isFalse(await isKeyboardNavigating(this.browser)); @@ -110,7 +120,7 @@ suite( // Focus a different deletable block await focusOnBlock(this.browser, 'controls_if_1'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); await sendKeyAndWait(this.browser, Key.Backspace); chai.assert.isTrue(await isKeyboardNavigating(this.browser)); @@ -121,10 +131,10 @@ suite( Blockly.keyboardNavigationController.setIsActive(true); }); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); // Right click a block await clickBlock(this.browser, 'controls_if_1', {button: 'right'}); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); chai.assert.isFalse(await isKeyboardNavigating(this.browser)); }); @@ -134,7 +144,7 @@ suite( Blockly.keyboardNavigationController.setIsActive(true); }); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); // Drag a block const element = await getBlockElementById(this.browser, 'controls_if_1'); @@ -147,7 +157,7 @@ suite( ); }); await element.dragAndDrop({x: 10, y: 10}); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); chai.assert.isFalse(await isKeyboardNavigating(this.browser)); }); diff --git a/test/webdriverio/test/move_test.ts b/test/webdriverio/test/move_test.ts index 72212051..d9cd2aea 100644 --- a/test/webdriverio/test/move_test.ts +++ b/test/webdriverio/test/move_test.ts @@ -16,12 +16,14 @@ import { sendKeyAndWait, keyDown, contextMenuItems, + checkForFailures, + pause, } from './test_setup.js'; suite('Move start tests', function () { - // Increase timeout to 10s for this longer test (but disable - // timeouts if when non-zero PAUSE_TIME is used to watch tests) run. - this.timeout(PAUSE_TIME ? 0 : 10000); + // Increase timeout for this longer test (but disable timeouts if when + // non-zero PAUSE_TIME is used to watch tests) run. + this.timeout(PAUSE_TIME ? 0 : 30000); // Clear the workspace and load start blocks. setup(async function () { @@ -29,7 +31,15 @@ suite('Move start tests', function () { testFileLocations.MOVE_START_TEST_BLOCKS, this.timeout(), ); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); + }); + + teardown(async function () { + await checkForFailures( + this.browser, + this.currentTest?.title, + this.currentTest?.state, + ); }); // When a move of a statement block begins, it is expected that only @@ -181,9 +191,9 @@ suite('Move start tests', function () { }); suite('Statement move tests', function () { - // Increase timeout to 10s for this longer test (but disable - // timeouts if when non-zero PAUSE_TIME is used to watch tests) run. - this.timeout(PAUSE_TIME ? 0 : 10000); + // Increase timeout for this longer test (but disable timeouts if when + // non-zero PAUSE_TIME is used to watch tests) run. + this.timeout(PAUSE_TIME ? 0 : 30000); // Clear the workspace and load start blocks. setup(async function () { @@ -191,7 +201,15 @@ suite('Statement move tests', function () { testFileLocations.MOVE_STATEMENT_TEST_BLOCKS, this.timeout(), ); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); + }); + + teardown(async function () { + await checkForFailures( + this.browser, + this.currentTest?.title, + this.currentTest?.state, + ); }); /** Serialized simple statement block with no statement inputs. */ @@ -362,9 +380,9 @@ suite('Statement move tests', function () { }); suite(`Value expression move tests`, function () { - // Increase timeout to 10s for this longer test (but disable - // timeouts if when non-zero PAUSE_TIME is used to watch tests) run. - this.timeout(PAUSE_TIME ? 0 : 10000); + // Increase timeout for this longer test (but disable timeouts if when + // non-zero PAUSE_TIME is used to watch tests) run. + this.timeout(PAUSE_TIME ? 0 : 30000); /** Serialized simple reporter value block with no inputs. */ const VALUE_SIMPLE = { @@ -474,7 +492,15 @@ suite(`Value expression move tests`, function () { ), this.timeout(), ); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); + }); + + teardown(async function () { + await checkForFailures( + this.browser, + this.currentTest?.title, + this.currentTest?.state, + ); }); suite('Constrained moves of a simple reporter block', function () { diff --git a/test/webdriverio/test/mutator_test.ts b/test/webdriverio/test/mutator_test.ts index b604c752..9714b1bf 100644 --- a/test/webdriverio/test/mutator_test.ts +++ b/test/webdriverio/test/mutator_test.ts @@ -17,6 +17,8 @@ import { sendKeyAndWait, keyRight, keyDown, + checkForFailures, + pause, } from './test_setup.js'; import {Key} from 'webdriverio'; @@ -32,7 +34,7 @@ suite('Mutator navigation', function () { ); this.openMutator = async () => { await focusOnBlock(this.browser, 'controls_if_1'); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); // Navigate to the mutator icon await keyRight(this.browser); // Activate the icon @@ -40,6 +42,14 @@ suite('Mutator navigation', function () { }; }); + teardown(async function () { + await checkForFailures( + this.browser, + this.currentTest?.title, + this.currentTest?.state, + ); + }); + test('Enter opens mutator', async function () { await this.openMutator(); @@ -96,7 +106,7 @@ suite('Mutator navigation', function () { await sendKeyAndWait(this.browser, 't'); // Navigate down to the second block in the flyout await keyDown(this.browser); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); // Hit enter to enter insert mode await sendKeyAndWait(this.browser, Key.Enter); // Hit enter again to lock it into place on the connection diff --git a/test/webdriverio/test/scroll_test.ts b/test/webdriverio/test/scroll_test.ts index c340714b..f43d9a1f 100644 --- a/test/webdriverio/test/scroll_test.ts +++ b/test/webdriverio/test/scroll_test.ts @@ -15,6 +15,8 @@ import { tabNavigateToWorkspace, testFileLocations, testSetup, + checkForFailures, + pause, } from './test_setup.js'; suite('Scrolling into view', function () { @@ -28,7 +30,7 @@ suite('Scrolling into view', function () { this.browser = await testSetup(testFileLocations.BASE, this.timeout()); this.windowSize = await this.browser.getWindowSize(); await this.browser.setWindowSize(800, 600); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); }); // Restore original browser window size. @@ -44,9 +46,17 @@ suite('Scrolling into view', function () { await testSetup(testFileLocations.BASE, this.timeout()); }); + teardown(async function () { + await checkForFailures( + this.browser, + this.currentTest?.title, + this.currentTest?.state, + ); + }); + test('Insert scrolls new block into view', async function () { - // Increase timeout to 10s for this longer test. - this.timeout(PAUSE_TIME ? 0 : 10000); + // Increase timeout for this longer test. + this.timeout(PAUSE_TIME ? 0 : 30000); await tabNavigateToWorkspace(this.browser); @@ -64,6 +74,8 @@ suite('Scrolling into view', function () { ).getBoundingRectangleWithoutChildren(), ); }); + // Pause to allow scrolling to stabilize before proceeding. + await pause(this.browser); // Insert and confirm the test block which should be scrolled into view. await sendKeyAndWait(this.browser, 't'); @@ -71,7 +83,7 @@ suite('Scrolling into view', function () { await sendKeyAndWait(this.browser, Key.Enter, 2); // Assert new block has been scrolled into the viewport. - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); const inViewport = await this.browser.execute(() => { const workspace = Blockly.getMainWorkspace() as Blockly.WorkspaceSvg; const block = workspace.getBlocksByType( diff --git a/test/webdriverio/test/stack_navigation.ts b/test/webdriverio/test/stack_navigation.ts index b26f273f..50d48361 100644 --- a/test/webdriverio/test/stack_navigation.ts +++ b/test/webdriverio/test/stack_navigation.ts @@ -8,18 +8,27 @@ import * as chai from 'chai'; import { getCurrentFocusedBlockId, getCurrentFocusNodeId, - PAUSE_TIME, tabNavigateToWorkspace, testFileLocations, testSetup, sendKeyAndWait, + checkForFailures, + pause, } from './test_setup.js'; suite('Stack navigation', function () { // Clear the workspace and load start blocks. setup(async function () { this.browser = await testSetup(testFileLocations.COMMENTS, this.timeout()); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); + }); + + teardown(async function () { + await checkForFailures( + this.browser, + this.currentTest?.title, + this.currentTest?.state, + ); }); test('Next', async function () { diff --git a/test/webdriverio/test/styling_test.ts b/test/webdriverio/test/styling_test.ts index 78a99018..60a5b980 100644 --- a/test/webdriverio/test/styling_test.ts +++ b/test/webdriverio/test/styling_test.ts @@ -14,6 +14,8 @@ import { tabNavigateToWorkspace, testFileLocations, testSetup, + checkForFailures, + pause, } from './test_setup.js'; import * as chai from 'chai'; @@ -24,7 +26,15 @@ suite('Styling test', function () { // Clear the workspace and load start blocks. setup(async function () { this.browser = await testSetup(testFileLocations.BASE, this.timeout()); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); + }); + + teardown(async function () { + await checkForFailures( + this.browser, + this.currentTest?.title, + this.currentTest?.state, + ); }); async function strokeColorEquals( diff --git a/test/webdriverio/test/test_setup.ts b/test/webdriverio/test/test_setup.ts index 1ddbac97..ccaaa0cf 100644 --- a/test/webdriverio/test/test_setup.ts +++ b/test/webdriverio/test/test_setup.ts @@ -44,6 +44,8 @@ let driver: webdriverio.Browser | null = null; */ export const PAUSE_TIME = 0; +let synchronizeCoreBlocklyRendering = true; + /** * Start up WebdriverIO and load the test page. This should only be * done once, to avoid constantly popping browser windows open and @@ -119,8 +121,16 @@ export async function testSetup( playgroundUrl: string, wdioWaitTimeoutMs: number, ): Promise { + // Reset back to default state between tests. + synchronizeCoreBlocklyRendering = true; + if (!driver) { driver = await driverSetup(wdioWaitTimeoutMs); + } else if (process.env.CI) { + // If running in CI force a session reload to ensure no browser state can + // leak across tests (since this can sometimes cause complex combined + // failures in CI). + await driver.reloadSession(); } await driver.url(playgroundUrl); // Wait for the workspace to exist and be rendered. @@ -130,6 +140,28 @@ export async function testSetup( return driver; } +/** + * Checks whether the current test has finished in a 'failing state' and, if it + * did, save a screenshot to the 'failures' directory with a name corresponding + * to the current test's title. + * + * @param browser The active WebdriverIO Browser object. + * @param testTitle The current running test's title, if known. + * @param testState The current running test's completion state, if known. + */ +export async function checkForFailures( + browser: WebdriverIO.Browser, + testTitle: string | undefined, + testState: string | undefined, +) { + if (testState === 'failed') { + if (!testTitle) { + throw new Error('Test failed and finished with no test title.'); + } + await browser.saveScreenshot(`failures/${testTitle}.png`); + } +} + /** * Replaces OS-specific path with POSIX style path. * @@ -192,6 +224,64 @@ export async function getSelectedBlockId(browser: WebdriverIO.Browser) { }); } +/** + * Pauses the browser for PAUSE_TIME, also possibly synchronizing rendering in + * core Blockly. + * + * This generally should always be preferred over calling browser.pause() + * directly. + * + * See setSynchronizeCoreBlocklyRendering() for additional details on + * configuring how this function behaves. + * + * @param browser The active WebdriverIO Browser object. + */ +export async function pause(browser: WebdriverIO.Browser) { + if (synchronizeCoreBlocklyRendering) { + // First, attempt to synchronize on rendering to ensure that Blockly is + // fully rendered before pausing for browser execution. This works around + // potential bugs when running in headless mode that can cause + // requestAnimationFrame to not call back (and cause state inconsistencies + // in block positions and sizes per #770). + await browser.execute(() => { + const workspace = Blockly.getMainWorkspace() as Blockly.WorkspaceSvg; + // Queue re-rendering all blocks. + workspace.render(); + // Flush the rendering queue (this is a slight hack to leverage + // BlockSvg.render() directly blocking on rendering finishing). + const blocks = workspace.getTopBlocks(); + if (blocks.length > 0) { + blocks[0].render(); + } + }); + } + await browser.pause(PAUSE_TIME); +} + +/** + * Configures whether to synchronize core Blockly's rendering system when trying + * to pause tests to wait for operations to complete. + * + * This is enabled by default and changes the behavior of pause() which is used + * both directly in tests and indirectly via the many test helpers in this file. + * + * Synchronization is useful because it ensures Blockly is fully rendered and + * stable before proceeding with the test (which can sometimes be desynchronized + * if the headless test environment drops a request for animation rendering + * frame which has been observed in CI environments). + * + * Synchronization must be disabled in certain tests, particularly those that + * can trigger alert dialogs since, when open, these will always cause any + * browser.execute() calls to fail the test (and rendering synchronization + * relies on this WebdriverIO mechanism). + * + * @param syncRendering Whether to synchronize Blockly rendering when pausing + * test execution using pause(). + */ +export function setSynchronizeCoreBlocklyRendering(syncRendering: boolean) { + synchronizeCoreBlocklyRendering = syncRendering; +} + /** * Clicks in the workspace to focus it. * @@ -202,7 +292,7 @@ export async function focusWorkspace(browser: WebdriverIO.Browser) { '#blocklyDiv > div > svg.blocklySvg > g', ); await workspaceElement.click({x: 100}); - await browser.pause(PAUSE_TIME); + await pause(browser); } /** @@ -291,7 +381,7 @@ export async function focusOnBlock( if (!block) throw new Error(`No block found with ID: ${blockId}.`); Blockly.getFocusManager().focusNode(block); }, blockId); - await browser.pause(PAUSE_TIME); + await pause(browser); } /** @@ -314,7 +404,7 @@ export async function focusOnWorkspaceComment( } Blockly.getFocusManager().focusNode(comment); }, commentId); - await browser.pause(PAUSE_TIME); + await pause(browser); } /** @@ -346,7 +436,7 @@ export async function focusOnBlockField( blockId, fieldName, ); - await browser.pause(PAUSE_TIME); + await pause(browser); } /** @@ -479,7 +569,7 @@ export async function tabNavigateToWorkspace( // there's no straightforward way to do that; see // https://stackoverflow.com/q/51518855/4969945 await browser.execute(() => document.getElementById('focusableDiv')?.focus()); - await browser.pause(PAUSE_TIME); + await pause(browser); // Navigate to workspace. if (hasToolbox) await tabNavigateForward(browser); if (hasFlyout) await tabNavigateForward(browser); @@ -576,10 +666,11 @@ export async function sendKeyAndWait( // Send all keys in one call if no pauses needed. keys = Array(times).fill(keys).flat(); await browser.keys(keys); + await pause(browser); } else { for (let i = 0; i < times; i++) { await browser.keys(keys); - await browser.pause(PAUSE_TIME); + await pause(browser); } } } @@ -725,13 +816,13 @@ export async function clickBlock( blockId, findableId, ); - await browser.pause(PAUSE_TIME); + await pause(browser); // In the test context, get the WebdriverIO Element that we've identified. const elem = await browser.$(`#${findableId}`); await elem.click(clickOptions); - await browser.pause(PAUSE_TIME); + await pause(browser); // In the browser context, remove the ID. await browser.execute((elemId) => { @@ -751,5 +842,5 @@ export async function rightClickOnFlyoutBlockType( ) { const elem = await browser.$(`.blocklyFlyout .${blockType}`); await elem.click({button: 'right'}); - await browser.pause(PAUSE_TIME); + await pause(browser); } diff --git a/test/webdriverio/test/toast_test.ts b/test/webdriverio/test/toast_test.ts index d73ca3d6..0a5e9a87 100644 --- a/test/webdriverio/test/toast_test.ts +++ b/test/webdriverio/test/toast_test.ts @@ -6,7 +6,13 @@ import * as chai from 'chai'; import * as Blockly from 'blockly/core'; -import {PAUSE_TIME, testFileLocations, testSetup} from './test_setup.js'; +import { + PAUSE_TIME, + testFileLocations, + testSetup, + checkForFailures, + pause, +} from './test_setup.js'; suite('HTML toasts', function () { // Disable timeouts when non-zero PAUSE_TIME is used to watch tests run. @@ -15,7 +21,15 @@ suite('HTML toasts', function () { // Clear the workspace and load start blocks. setup(async function () { this.browser = await testSetup(testFileLocations.BASE, this.timeout()); - await this.browser.pause(PAUSE_TIME); + await pause(this.browser); + }); + + teardown(async function () { + await checkForFailures( + this.browser, + this.currentTest?.title, + this.currentTest?.state, + ); }); test('Can be displayed', async function () { diff --git a/test/webdriverio/test/workspace_comment_test.ts b/test/webdriverio/test/workspace_comment_test.ts index 7f9e4a75..f57f9ab9 100644 --- a/test/webdriverio/test/workspace_comment_test.ts +++ b/test/webdriverio/test/workspace_comment_test.ts @@ -20,6 +20,7 @@ import { keyUp, contextMenuItems, PAUSE_TIME, + checkForFailures, } from './test_setup.js'; import {Key} from 'webdriverio'; @@ -69,6 +70,14 @@ suite('Workspace comment navigation', function () { }; }); + teardown(async function () { + await checkForFailures( + this.browser, + this.currentTest?.title, + this.currentTest?.state, + ); + }); + test('Navigate forward from block to workspace comment', async function () { await focusOnBlock(this.browser, 'p5_canvas_1'); await keyDown(this.browser); @@ -151,7 +160,7 @@ suite('Workspace comment navigation', function () { return Blockly.getMainWorkspace() .getCommentById(commentId) ?.isCollapsed(); - }, this.commentId1); + }, this.commentId1 as string); chai.assert.isTrue(collapsed); }); @@ -172,7 +181,7 @@ suite('Workspace comment navigation', function () { const commentText = await this.browser.execute((commentId) => { return Blockly.getMainWorkspace().getCommentById(commentId)?.getText(); - }, this.commentId1); + }, this.commentId1 as string); chai.assert.equal(commentText, 'Comment oneHello world'); });