diff --git a/test/runner/browserstack/browsers.js b/test/runner/browsers.js similarity index 67% rename from test/runner/browserstack/browsers.js rename to test/runner/browsers.js index 7b703ceb..1ddccdf7 100644 --- a/test/runner/browserstack/browsers.js +++ b/test/runner/browsers.js @@ -1,6 +1,11 @@ import chalk from "chalk"; -import { getBrowserString } from "../lib/getBrowserString.js"; -import { createWorker, deleteWorker, getAvailableSessions } from "./api.js"; +import { getBrowserString } from "./lib/getBrowserString.js"; +import { + createWorker, + deleteWorker, + getAvailableSessions +} from "./browserstack/api.js"; +import createDriver from "./selenium/createDriver.js"; const workers = Object.create( null ); @@ -8,12 +13,13 @@ const workers = Object.create( null ); * Keys are browser strings * Structure of a worker: * { - * debug: boolean, // Stops the worker from being cleaned up when finished - * id: string, - * lastTouch: number, // The last time a request was received - * url: string, - * browser: object, // The browser object + * browser: object // The browser object + * debug: boolean // Stops the worker from being cleaned up when finished + * lastTouch: number // The last time a request was received + * restarts: number // The number of times the worker has been restarted * options: object // The options to create the worker + * url: string // The URL the worker is on + * quit: function // A function to stop the worker * } */ @@ -31,70 +37,8 @@ const RUN_WORKER_TIMEOUT = 60 * 1000 * 2; const WORKER_WAIT_TIME = 30000; -export function touchBrowser( browser ) { - const fullBrowser = getBrowserString( browser ); - const worker = workers[ fullBrowser ]; - if ( worker ) { - worker.lastTouch = Date.now(); - } -} - -async function waitForAck( worker, { fullBrowser, verbose } ) { - delete worker.lastTouch; - return new Promise( ( resolve, reject ) => { - const interval = setInterval( () => { - if ( worker.lastTouch ) { - if ( verbose ) { - console.log( `\n${ fullBrowser } acknowledged.` ); - } - clearTimeout( timeout ); - clearInterval( interval ); - resolve(); - } - }, ACKNOWLEDGE_INTERVAL ); - - const timeout = setTimeout( () => { - clearInterval( interval ); - reject( - new Error( - `${ fullBrowser } not acknowledged after ${ - ACKNOWLEDGE_TIMEOUT / 1000 / 60 - }min.` - ) - ); - }, ACKNOWLEDGE_TIMEOUT ); - } ); -} - -async function restartWorker( worker ) { - await cleanupWorker( worker, worker.options ); - await createBrowserWorker( - worker.url, - worker.browser, - worker.options, - worker.restarts + 1 - ); -} - -export async function restartBrowser( browser ) { - const fullBrowser = getBrowserString( browser ); - const worker = workers[ fullBrowser ]; - if ( worker ) { - await restartWorker( worker ); - } -} - -async function ensureAcknowledged( worker ) { - const fullBrowser = getBrowserString( worker.browser ); - const verbose = worker.options.verbose; - try { - await waitForAck( worker, { fullBrowser, verbose } ); - return worker; - } catch ( error ) { - console.error( error.message ); - await restartWorker( worker ); - } -} +// Limit concurrency to 8 by default in selenium +const MAX_SELENIUM_CONCURRENCY = 8; export async function createBrowserWorker( url, browser, options, restarts = 0 ) { if ( restarts > MAX_WORKER_RESTARTS ) { @@ -104,37 +48,51 @@ export async function createBrowserWorker( url, browser, options, restarts = 0 ) ) }` ); } - const verbose = options.verbose; - while ( ( await getAvailableSessions() ) <= 0 ) { + const { browserstack, debug, headless, runId, tunnelId, verbose } = options; + while ( await maxWorkersReached( options ) ) { if ( verbose ) { console.log( "\nWaiting for available sessions..." ); } await new Promise( ( resolve ) => setTimeout( resolve, WORKER_WAIT_TIME ) ); } - const { debug, runId, tunnelId } = options; const fullBrowser = getBrowserString( browser ); - const worker = await createWorker( { - ...browser, - url: encodeURI( url ), - project: "jquery-migrate", - build: `Run ${ runId }`, - - // This is the maximum timeout allowed - // by BrowserStack. We do this because - // we control the timeout in the runner. - // See https://github.com/browserstack/api/blob/b324a6a5bc1b6052510d74e286b8e1c758c308a7/README.md#timeout300 - timeout: 1800, - - // Not documented in the API docs, - // but required to make local testing work. - // See https://www.browserstack.com/docs/automate/selenium/manage-multiple-connections#nodejs - "browserstack.local": true, - "browserstack.localIdentifier": tunnelId - } ); + let worker; + + if ( browserstack ) { + worker = await createWorker( { + ...browser, + url: encodeURI( url ), + project: "jquery", + build: `Run ${ runId }`, + + // This is the maximum timeout allowed + // by BrowserStack. We do this because + // we control the timeout in the runner. + // See https://github.com/browserstack/api/blob/b324a6a5bc1b6052510d74e286b8e1c758c308a7/README.md#timeout300 + timeout: 1800, + + // Not documented in the API docs, + // but required to make local testing work. + // See https://www.browserstack.com/docs/automate/selenium/manage-multiple-connections#nodejs + "browserstack.local": true, + "browserstack.localIdentifier": tunnelId + } ); + worker.quit = () => deleteWorker( worker.id ); + } else { + const driver = await createDriver( { + browserName: browser.browser, + headless, + url, + verbose + } ); + worker = { + quit: () => driver.quit() + }; + } - browser.debug = !!debug; + worker.debug = !!debug; worker.url = url; worker.browser = browser; worker.restarts = restarts; @@ -147,6 +105,14 @@ export async function createBrowserWorker( url, browser, options, restarts = 0 ) return ensureAcknowledged( worker ); } +export function touchBrowser( browser ) { + const fullBrowser = getBrowserString( browser ); + const worker = workers[ fullBrowser ]; + if ( worker ) { + worker.lastTouch = Date.now(); + } +} + export async function setBrowserWorkerUrl( browser, url ) { const fullBrowser = getBrowserString( browser ); const worker = workers[ fullBrowser ]; @@ -155,6 +121,14 @@ export async function setBrowserWorkerUrl( browser, url ) { } } +export async function restartBrowser( browser ) { + const fullBrowser = getBrowserString( browser ); + const worker = workers[ fullBrowser ]; + if ( worker ) { + await restartWorker( worker ); + } +} + /** * Checks that all browsers have received * a response in the given amount of time. @@ -176,27 +150,12 @@ export async function checkLastTouches() { } } -export async function cleanupWorker( worker, { verbose } ) { - for ( const [ fullBrowser, w ] of Object.entries( workers ) ) { - if ( w === worker ) { - delete workers[ fullBrowser ]; - await deleteWorker( worker.id ); - if ( verbose ) { - console.log( `\nStopped ${ fullBrowser }.` ); - } - return; - } - } -} - export async function cleanupAllBrowsers( { verbose } ) { const workersRemaining = Object.values( workers ); const numRemaining = workersRemaining.length; if ( numRemaining ) { try { - await Promise.all( - workersRemaining.map( ( worker ) => deleteWorker( worker.id ) ) - ); + await Promise.all( workersRemaining.map( ( worker ) => worker.quit() ) ); if ( verbose ) { console.log( `Stopped ${ numRemaining } browser${ numRemaining > 1 ? "s" : "" }.` @@ -209,3 +168,75 @@ export async function cleanupAllBrowsers( { verbose } ) { } } } + +async function maxWorkersReached( { + browserstack, + concurrency = MAX_SELENIUM_CONCURRENCY +} ) { + if ( browserstack ) { + return ( await getAvailableSessions() ) <= 0; + } + return workers.length >= concurrency; +} + +async function waitForAck( worker, { fullBrowser, verbose } ) { + delete worker.lastTouch; + return new Promise( ( resolve, reject ) => { + const interval = setInterval( () => { + if ( worker.lastTouch ) { + if ( verbose ) { + console.log( `\n${ fullBrowser } acknowledged.` ); + } + clearTimeout( timeout ); + clearInterval( interval ); + resolve(); + } + }, ACKNOWLEDGE_INTERVAL ); + + const timeout = setTimeout( () => { + clearInterval( interval ); + reject( + new Error( + `${ fullBrowser } not acknowledged after ${ + ACKNOWLEDGE_TIMEOUT / 1000 / 60 + }min.` + ) + ); + }, ACKNOWLEDGE_TIMEOUT ); + } ); +} + +async function ensureAcknowledged( worker ) { + const fullBrowser = getBrowserString( worker.browser ); + const verbose = worker.options.verbose; + try { + await waitForAck( worker, { fullBrowser, verbose } ); + return worker; + } catch ( error ) { + console.error( error.message ); + await restartWorker( worker ); + } +} + +async function cleanupWorker( worker, { verbose } ) { + for ( const [ fullBrowser, w ] of Object.entries( workers ) ) { + if ( w === worker ) { + delete workers[ fullBrowser ]; + await worker.quit(); + if ( verbose ) { + console.log( `\nStopped ${ fullBrowser }.` ); + } + return; + } + } +} + +async function restartWorker( worker ) { + await cleanupWorker( worker, worker.options ); + await createBrowserWorker( + worker.url, + worker.browser, + worker.options, + worker.restarts + 1 + ); +} diff --git a/test/runner/command.js b/test/runner/command.js index fccc324d..462dba6e 100644 --- a/test/runner/command.js +++ b/test/runner/command.js @@ -2,10 +2,10 @@ import yargs from "yargs/yargs"; import { browsers } from "./flags/browsers.js"; import { getPlan, listBrowsers, stopWorkers } from "./browserstack/api.js"; import { buildBrowserFromString } from "./browserstack/buildBrowserFromString.js"; -import { modules } from "./modules.js"; +import { modules } from "./flags/modules.js"; import { run } from "./run.js"; -import { jqueryMigrate } from "./jquery-migrate.js"; -import { jquery } from "./jquery.js"; +import { jqueryMigrate } from "./flags/jquery-migrate.js"; +import { jquery } from "./flags/jquery.js"; const argv = yargs( process.argv.slice( 2 ) ) .version( false ) @@ -68,15 +68,23 @@ const argv = yargs( process.argv.slice( 2 ) ) "Leave the browser open for debugging. Cannot be used with --headless.", conflicts: [ "headless" ] } ) + .option( "retries", { + alias: "r", + type: "number", + description: "Number of times to retry failed tests by refreshing the URL." + } ) + .option( "hard-retries", { + type: "number", + description: + "Number of times to retry failed tests by restarting the worker. " + + "This is in addition to the normal retries " + + "and are only used when the normal retries are exhausted." + } ) .option( "verbose", { alias: "v", type: "boolean", description: "Log additional information." } ) - .option( "run-id", { - type: "string", - description: "A unique identifier for this run." - } ) .option( "isolate", { type: "boolean", description: "Run each module by itself in the test page. This can extend testing time." @@ -93,19 +101,9 @@ const argv = yargs( process.argv.slice( 2 ) ) "Otherwise, the --browser option will be used, " + "with the latest version/device for that browser, on a matching OS." } ) - .option( "retries", { - alias: "r", - type: "number", - description: "Number of times to retry failed tests in BrowserStack.", - implies: [ "browserstack" ] - } ) - .option( "hard-retries", { - type: "number", - description: - "Number of times to retry failed tests in BrowserStack " + - "by restarting the worker. This is in addition to the normal retries " + - "and are only used when the normal retries are exhausted.", - implies: [ "browserstack" ] + .option( "run-id", { + type: "string", + description: "A unique identifier for the run in BrowserStack." } ) .option( "list-browsers", { type: "string", diff --git a/test/runner/jquery-migrate.js b/test/runner/flags/jquery-migrate.js similarity index 100% rename from test/runner/jquery-migrate.js rename to test/runner/flags/jquery-migrate.js diff --git a/test/runner/jquery.js b/test/runner/flags/jquery.js similarity index 100% rename from test/runner/jquery.js rename to test/runner/flags/jquery.js diff --git a/test/runner/modules.js b/test/runner/flags/modules.js similarity index 100% rename from test/runner/modules.js rename to test/runner/flags/modules.js diff --git a/test/runner/listeners.js b/test/runner/listeners.js index cca2bbd6..61a98e7c 100644 --- a/test/runner/listeners.js +++ b/test/runner/listeners.js @@ -53,6 +53,17 @@ return nu; } } + + // Serialize Symbols as string representations so they are + // sent over the wire after being stringified. + if ( typeof value === "symbol" ) { + + // We can *describe* unique symbols, but note that their identity + // (e.g., `Symbol() !== Symbol()`) is lost + var ctor = Symbol.keyFor( value ) !== undefined ? "Symbol.for" : "Symbol"; + return ctor + "(" + JSON.stringify( value.description ) + ")"; + } + return value; } return derez( object ); diff --git a/test/runner/browserstack/queue.js b/test/runner/queue.js similarity index 89% rename from test/runner/browserstack/queue.js rename to test/runner/queue.js index 6d1c8d51..843d5672 100644 --- a/test/runner/browserstack/queue.js +++ b/test/runner/queue.js @@ -1,5 +1,5 @@ import chalk from "chalk"; -import { getBrowserString } from "../lib/getBrowserString.js"; +import { getBrowserString } from "./lib/getBrowserString.js"; import { checkLastTouches, createBrowserWorker, @@ -45,7 +45,7 @@ export function retryTest( reportId, maxRetries ) { test.retries++; if ( test.retries <= maxRetries ) { console.log( - `Retrying test ${ reportId } for ${ chalk.yellow( + `\nRetrying test ${ reportId } for ${ chalk.yellow( test.options.modules.join( ", " ) ) }...${ test.retries }` ); @@ -63,7 +63,7 @@ export async function hardRetryTest( reportId, maxHardRetries ) { test.hardRetries++; if ( test.hardRetries <= maxHardRetries ) { console.log( - `Hard retrying test ${ reportId } for ${ chalk.yellow( + `\nHard retrying test ${ reportId } for ${ chalk.yellow( test.options.modules.join( ", " ) ) }...${ test.hardRetries }` ); @@ -74,7 +74,7 @@ export async function hardRetryTest( reportId, maxHardRetries ) { return false; } -export function addBrowserStackRun( url, browser, options ) { +export function addRun( url, browser, options ) { queue.push( { browser, fullBrowser: getBrowserString( browser ), @@ -87,7 +87,7 @@ export function addBrowserStackRun( url, browser, options ) { } ); } -export async function runAllBrowserStack() { +export async function runAll() { return new Promise( async( resolve, reject ) => { while ( queue.length ) { try { diff --git a/test/runner/reporter.js b/test/runner/reporter.js index f3942ce9..91cdd297 100644 --- a/test/runner/reporter.js +++ b/test/runner/reporter.js @@ -3,6 +3,18 @@ import { getBrowserString } from "./lib/getBrowserString.js"; import { prettyMs } from "./lib/prettyMs.js"; import * as Diff from "diff"; +function serializeForDiff( value ) { + + // Use naive serialization for everything except types with confusable values + if ( typeof value === "string" ) { + return JSON.stringify( value ); + } + if ( typeof value === "bigint" ) { + return `${ value }n`; + } + return `${ value }`; +} + export function reportTest( test, reportId, { browser, headless } ) { if ( test.status === "passed" ) { @@ -25,9 +37,16 @@ export function reportTest( test, reportId, { browser, headless } ) { message += `\n${ error.message }`; } message += `\n${ chalk.gray( error.stack ) }`; - if ( "expected" in error && "actual" in error ) { - message += `\nexpected: ${ JSON.stringify( error.expected ) }`; - message += `\nactual: ${ JSON.stringify( error.actual ) }`; + + // Show expected and actual values + // if either is defined and non-null. + // error.actual is set to null for failed + // assert.expect() assertions, so skip those as well. + // This should be fine because error.expected would + // have to also be null for this to be skipped. + if ( error.expected != null || error.actual != null ) { + message += `\nexpected: ${ chalk.red( JSON.stringify( error.expected ) ) }`; + message += `\nactual: ${ chalk.green( JSON.stringify( error.actual ) ) }`; let diff; if ( Array.isArray( error.expected ) && Array.isArray( error.actual ) ) { @@ -43,7 +62,7 @@ export function reportTest( test, reportId, { browser, headless } ) { diff = Diff.diffJson( error.expected, error.actual ); } else if ( typeof error.expected === "number" && - typeof error.expected === "number" + typeof error.actual === "number" ) { // Diff numbers directly @@ -54,30 +73,35 @@ export function reportTest( test, reportId, { browser, headless } ) { diff = [ { removed: true, value: `${ value }` } ]; } } else if ( - typeof error.expected === "boolean" && - typeof error.actual === "boolean" + typeof error.expected === "string" && + typeof error.actual === "string" ) { - // Show the actual boolean in red - diff = [ { removed: true, value: `${ error.actual }` } ]; + // Diff the characters of strings + diff = Diff.diffChars( error.expected, error.actual ); } else { - // Diff everything else as characters - diff = Diff.diffChars( `${ error.expected }`, `${ error.actual }` ); + // Diff everything else as words + diff = Diff.diffWords( + serializeForDiff( error.expected ), + serializeForDiff( error.actual ) + ); } - message += "\n"; - message += diff - .map( ( part ) => { - if ( part.added ) { - return chalk.green( part.value ); - } - if ( part.removed ) { - return chalk.red( part.value ); - } - return chalk.gray( part.value ); - } ) - .join( "" ); + if ( diff ) { + message += "\n"; + message += diff + .map( ( part ) => { + if ( part.added ) { + return chalk.green( part.value ); + } + if ( part.removed ) { + return chalk.red( part.value ); + } + return chalk.gray( part.value ); + } ) + .join( "" ); + } } } } diff --git a/test/runner/run.js b/test/runner/run.js index f3676e5d..69ef86bd 100644 --- a/test/runner/run.js +++ b/test/runner/run.js @@ -8,16 +8,15 @@ import { createTestServer } from "./createTestServer.js"; import { buildTestUrl } from "./lib/buildTestUrl.js"; import { generateHash, printModuleHashes } from "./lib/generateHash.js"; import { getBrowserString } from "./lib/getBrowserString.js"; -import { modules as allModules } from "./modules.js"; -import { cleanupAllBrowsers, touchBrowser } from "./browserstack/browsers.js"; +import { modules as allModules } from "./flags/modules.js"; +import { cleanupAllBrowsers, touchBrowser } from "./browsers.js"; import { - addBrowserStackRun, + addRun, getNextBrowserTest, hardRetryTest, retryTest, - runAllBrowserStack -} from "./browserstack/queue.js"; -import { addSeleniumRun, runAllSelenium } from "./selenium/queue.js"; + runAll +} from "./queue.js"; const EXIT_HOOK_WAIT_TIMEOUT = 60 * 1000; @@ -271,6 +270,8 @@ export async function run( { } ); const options = { + browserstack, + concurrency, debug, headless, jquery, @@ -282,11 +283,7 @@ export async function run( { verbose }; - if ( browserstack ) { - addBrowserStackRun( url, browser, options ); - } else { - addSeleniumRun( url, browser, options ); - } + addRun( url, browser, options ); } } } @@ -303,11 +300,7 @@ export async function run( { try { console.log( `Starting Run ${ runId }...` ); - if ( browserstack ) { - await runAllBrowserStack( { verbose } ); - } else { - await runAllSelenium( { concurrency, verbose } ); - } + await runAll(); } catch ( error ) { console.error( error ); if ( !debug ) { diff --git a/test/runner/selenium/createDriver.js b/test/runner/selenium/createDriver.js index d1680b22..095c1221 100644 --- a/test/runner/selenium/createDriver.js +++ b/test/runner/selenium/createDriver.js @@ -7,7 +7,7 @@ import { browserSupportsHeadless } from "../lib/getBrowserString.js"; // Set script timeout to 10min const DRIVER_SCRIPT_TIMEOUT = 1000 * 60 * 10; -export default async function createDriver( { browserName, headless, verbose } ) { +export default async function createDriver( { browserName, headless, url, verbose } ) { const capabilities = Capabilities[ browserName ](); const prefs = new logging.Preferences(); prefs.setLevel( logging.Type.BROWSER, logging.Level.ALL ); @@ -77,5 +77,8 @@ export default async function createDriver( { browserName, headless, verbose } ) // Increase script timeout to 10min await driver.manage().setTimeouts( { script: DRIVER_SCRIPT_TIMEOUT } ); + // Set the first URL for the browser + await driver.get( url ); + return driver; } diff --git a/test/runner/selenium/queue.js b/test/runner/selenium/queue.js deleted file mode 100644 index 2d62f695..00000000 --- a/test/runner/selenium/queue.js +++ /dev/null @@ -1,66 +0,0 @@ -// Build a queue that runs both browsers and modules -// in parallel when the length reaches the concurrency limit -// and refills the queue when one promise resolves. - -import chalk from "chalk"; -import { getBrowserString } from "../lib/getBrowserString.js"; -import { runSelenium } from "./runSelenium.js"; - -const promises = []; -const queue = []; - -const SELENIUM_WAIT_TIME = 100; - -// Limit concurrency to 8 by default in selenium -// BrowserStack defaults to the max allowed by the plan -// More than this will log MaxListenersExceededWarning -const MAX_CONCURRENCY = 8; - -export function addSeleniumRun( url, browser, options ) { - queue.push( { url, browser, options } ); -} - -export async function runAllSelenium( { concurrency = MAX_CONCURRENCY, verbose } ) { - while ( queue.length ) { - const next = queue.shift(); - const { url, browser, options } = next; - - const fullBrowser = getBrowserString( browser, options.headless ); - console.log( - `\nRunning ${ chalk.yellow( options.modules.join( ", " ) ) } tests ` + - `for jQuery Migrate ${ chalk.yellow( options.jqueryMigrate ) } ` + - `and jQuery ${ chalk.yellow( options.jquery ) } ` + - `in ${ chalk.yellow( fullBrowser ) } (${ chalk.bold( options.reportId ) })...` - ); - - // Wait enough time between requests - // to give concurrency a chance to update. - // In selenium, this helps avoid undici connect timeout errors. - await new Promise( ( resolve ) => setTimeout( resolve, SELENIUM_WAIT_TIME ) ); - - if ( verbose ) { - console.log( `\nTests remaining: ${ queue.length + 1 }.` ); - } - - const promise = runSelenium( url, browser, options ); - - // Remove the promise from the list when it resolves - promise.then( () => { - const index = promises.indexOf( promise ); - if ( index !== -1 ) { - promises.splice( index, 1 ); - } - } ); - - // Add the promise to the list - promises.push( promise ); - - // Wait until at least one promise resolves - // if we've reached the concurrency limit - if ( promises.length >= concurrency ) { - await Promise.any( promises ); - } - } - - await Promise.all( promises ); -} diff --git a/test/runner/selenium/runSelenium.js b/test/runner/selenium/runSelenium.js deleted file mode 100644 index 848db36c..00000000 --- a/test/runner/selenium/runSelenium.js +++ /dev/null @@ -1,30 +0,0 @@ -import createDriver from "./createDriver.js"; - -export async function runSelenium( - url, - { browser }, - { debug, headless, verbose } = {} -) { - if ( debug && headless ) { - throw new Error( "Cannot debug in headless mode." ); - } - - const driver = await createDriver( { - browserName: browser, - headless, - verbose - } ); - - try { - await driver.get( url ); - await driver.executeScript( -`return new Promise( ( resolve ) => { - QUnit.on( "runEnd", resolve ); -} )` - ); - } finally { - if ( !debug || headless ) { - await driver.quit(); - } - } -}