Skip to content

Commit

Permalink
Tests: share queue/browser handling for all worker types
Browse files Browse the repository at this point in the history 
- one queue to rule them all: browserstack and selenium
- retries and hard retries are now supported in selenium
- selenium tests now re-use browsers in the same way as browserstack
  • Loading branch information
@timmywil
timmywil committed Apr 4, 2024
1 parent 1426c56 commit 8bec395
Show file tree
Hide file tree
Showing 12 changed files with 233 additions and 269 deletions.
249 changes: 140 additions & 109 deletions test/runner/browserstack/browsers.js → test/runner/browsers.js
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,25 @@
import chalk from "chalk";
import { getBrowserString } from "../lib/getBrowserString.js";
import { createWorker, deleteWorker, getAvailableSessions } from "./api.js";
import { getBrowserString } from "./lib/getBrowserString.js";
import {
createWorker,
deleteWorker,
getAvailableSessions
} from "./browserstack/api.js";
import createDriver from "./selenium/createDriver.js";

const workers = Object.create( null );

/**
* Keys are browser strings
* Structure of a worker:
* {
* debug: boolean, // Stops the worker from being cleaned up when finished
* id: string,
* lastTouch: number, // The last time a request was received
* url: string,
* browser: object, // The browser object
* browser: object // The browser object
* debug: boolean // Stops the worker from being cleaned up when finished
* lastTouch: number // The last time a request was received
* restarts: number // The number of times the worker has been restarted
* options: object // The options to create the worker
* url: string // The URL the worker is on
* quit: function // A function to stop the worker
* }
*/

Expand All @@ -31,70 +37,8 @@ const RUN_WORKER_TIMEOUT = 60 * 1000 * 2;

const WORKER_WAIT_TIME = 30000;

export function touchBrowser( browser ) {
const fullBrowser = getBrowserString( browser );
const worker = workers[ fullBrowser ];
if ( worker ) {
worker.lastTouch = Date.now();
}
}

async function waitForAck( worker, { fullBrowser, verbose } ) {
delete worker.lastTouch;
return new Promise( ( resolve, reject ) => {
const interval = setInterval( () => {
if ( worker.lastTouch ) {
if ( verbose ) {
console.log( `\n${ fullBrowser } acknowledged.` );
}
clearTimeout( timeout );
clearInterval( interval );
resolve();
}
}, ACKNOWLEDGE_INTERVAL );

const timeout = setTimeout( () => {
clearInterval( interval );
reject(
new Error(
`${ fullBrowser } not acknowledged after ${
ACKNOWLEDGE_TIMEOUT / 1000 / 60
}min.`
)
);
}, ACKNOWLEDGE_TIMEOUT );
} );
}

async function restartWorker( worker ) {
await cleanupWorker( worker, worker.options );
await createBrowserWorker(
worker.url,
worker.browser,
worker.options,
worker.restarts + 1
);
}

export async function restartBrowser( browser ) {
const fullBrowser = getBrowserString( browser );
const worker = workers[ fullBrowser ];
if ( worker ) {
await restartWorker( worker );
}
}

async function ensureAcknowledged( worker ) {
const fullBrowser = getBrowserString( worker.browser );
const verbose = worker.options.verbose;
try {
await waitForAck( worker, { fullBrowser, verbose } );
return worker;
} catch ( error ) {
console.error( error.message );
await restartWorker( worker );
}
}
// Limit concurrency to 8 by default in selenium
const MAX_SELENIUM_CONCURRENCY = 8;

export async function createBrowserWorker( url, browser, options, restarts = 0 ) {
if ( restarts > MAX_WORKER_RESTARTS ) {
Expand All @@ -104,37 +48,51 @@ export async function createBrowserWorker( url, browser, options, restarts = 0 )
) }`
);
}
const verbose = options.verbose;
while ( ( await getAvailableSessions() ) <= 0 ) {
const { browserstack, debug, headless, runId, tunnelId, verbose } = options;
while ( await maxWorkersReached( options ) ) {
if ( verbose ) {
console.log( "\nWaiting for available sessions..." );
}
await new Promise( ( resolve ) => setTimeout( resolve, WORKER_WAIT_TIME ) );
}

const { debug, runId, tunnelId } = options;
const fullBrowser = getBrowserString( browser );

const worker = await createWorker( {
...browser,
url: encodeURI( url ),
project: "jquery-migrate",
build: `Run ${ runId }`,

// This is the maximum timeout allowed
// by BrowserStack. We do this because
// we control the timeout in the runner.
// See https://github.com/browserstack/api/blob/b324a6a5bc1b6052510d74e286b8e1c758c308a7/README.md#timeout300
timeout: 1800,

// Not documented in the API docs,
// but required to make local testing work.
// See https://www.browserstack.com/docs/automate/selenium/manage-multiple-connections#nodejs
"browserstack.local": true,
"browserstack.localIdentifier": tunnelId
} );
let worker;

if ( browserstack ) {
worker = await createWorker( {
...browser,
url: encodeURI( url ),
project: "jquery",
build: `Run ${ runId }`,

// This is the maximum timeout allowed
// by BrowserStack. We do this because
// we control the timeout in the runner.
// See https://github.com/browserstack/api/blob/b324a6a5bc1b6052510d74e286b8e1c758c308a7/README.md#timeout300
timeout: 1800,

// Not documented in the API docs,
// but required to make local testing work.
// See https://www.browserstack.com/docs/automate/selenium/manage-multiple-connections#nodejs
"browserstack.local": true,
"browserstack.localIdentifier": tunnelId
} );
worker.quit = () => deleteWorker( worker.id );
} else {
const driver = await createDriver( {
browserName: browser.browser,
headless,
url,
verbose
} );
worker = {
quit: () => driver.quit()
};
}

browser.debug = !!debug;
worker.debug = !!debug;
worker.url = url;
worker.browser = browser;
worker.restarts = restarts;
Expand All @@ -147,6 +105,14 @@ export async function createBrowserWorker( url, browser, options, restarts = 0 )
return ensureAcknowledged( worker );
}

export function touchBrowser( browser ) {
const fullBrowser = getBrowserString( browser );
const worker = workers[ fullBrowser ];
if ( worker ) {
worker.lastTouch = Date.now();
}
}

export async function setBrowserWorkerUrl( browser, url ) {
const fullBrowser = getBrowserString( browser );
const worker = workers[ fullBrowser ];
Expand All @@ -155,6 +121,14 @@ export async function setBrowserWorkerUrl( browser, url ) {
}
}

export async function restartBrowser( browser ) {
const fullBrowser = getBrowserString( browser );
const worker = workers[ fullBrowser ];
if ( worker ) {
await restartWorker( worker );
}
}

/**
* Checks that all browsers have received
* a response in the given amount of time.
Expand All @@ -176,27 +150,12 @@ export async function checkLastTouches() {
}
}

export async function cleanupWorker( worker, { verbose } ) {
for ( const [ fullBrowser, w ] of Object.entries( workers ) ) {
if ( w === worker ) {
delete workers[ fullBrowser ];
await deleteWorker( worker.id );
if ( verbose ) {
console.log( `\nStopped ${ fullBrowser }.` );
}
return;
}
}
}

export async function cleanupAllBrowsers( { verbose } ) {
const workersRemaining = Object.values( workers );
const numRemaining = workersRemaining.length;
if ( numRemaining ) {
try {
await Promise.all(
workersRemaining.map( ( worker ) => deleteWorker( worker.id ) )
);
await Promise.all( workersRemaining.map( ( worker ) => worker.quit() ) );
if ( verbose ) {
console.log(
`Stopped ${ numRemaining } browser${ numRemaining > 1 ? "s" : "" }.`
Expand All @@ -209,3 +168,75 @@ export async function cleanupAllBrowsers( { verbose } ) {
}
}
}

async function maxWorkersReached( {
browserstack,
concurrency = MAX_SELENIUM_CONCURRENCY
} ) {
if ( browserstack ) {
return ( await getAvailableSessions() ) <= 0;
}
return workers.length >= concurrency;
}

async function waitForAck( worker, { fullBrowser, verbose } ) {
delete worker.lastTouch;
return new Promise( ( resolve, reject ) => {
const interval = setInterval( () => {
if ( worker.lastTouch ) {
if ( verbose ) {
console.log( `\n${ fullBrowser } acknowledged.` );
}
clearTimeout( timeout );
clearInterval( interval );
resolve();
}
}, ACKNOWLEDGE_INTERVAL );

const timeout = setTimeout( () => {
clearInterval( interval );
reject(
new Error(
`${ fullBrowser } not acknowledged after ${
ACKNOWLEDGE_TIMEOUT / 1000 / 60
}min.`
)
);
}, ACKNOWLEDGE_TIMEOUT );
} );
}

async function ensureAcknowledged( worker ) {
const fullBrowser = getBrowserString( worker.browser );
const verbose = worker.options.verbose;
try {
await waitForAck( worker, { fullBrowser, verbose } );
return worker;
} catch ( error ) {
console.error( error.message );
await restartWorker( worker );
}
}

async function cleanupWorker( worker, { verbose } ) {
for ( const [ fullBrowser, w ] of Object.entries( workers ) ) {
if ( w === worker ) {
delete workers[ fullBrowser ];
await worker.quit();
if ( verbose ) {
console.log( `\nStopped ${ fullBrowser }.` );
}
return;
}
}
}

async function restartWorker( worker ) {
await cleanupWorker( worker, worker.options );
await createBrowserWorker(
worker.url,
worker.browser,
worker.options,
worker.restarts + 1
);
}
38 changes: 18 additions & 20 deletions test/runner/command.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@ import yargs from "yargs/yargs";
import { browsers } from "./flags/browsers.js";
import { getPlan, listBrowsers, stopWorkers } from "./browserstack/api.js";
import { buildBrowserFromString } from "./browserstack/buildBrowserFromString.js";
import { modules } from "./modules.js";
import { modules } from "./flags/modules.js";
import { run } from "./run.js";
import { jqueryMigrate } from "./jquery-migrate.js";
import { jquery } from "./jquery.js";
import { jqueryMigrate } from "./flags/jquery-migrate.js";
import { jquery } from "./flags/jquery.js";

const argv = yargs( process.argv.slice( 2 ) )
.version( false )
Expand Down Expand Up @@ -68,15 +68,23 @@ const argv = yargs( process.argv.slice( 2 ) )
"Leave the browser open for debugging. Cannot be used with --headless.",
conflicts: [ "headless" ]
} )
.option( "retries", {
alias: "r",
type: "number",
description: "Number of times to retry failed tests by refreshing the URL."
} )
.option( "hard-retries", {
type: "number",
description:
"Number of times to retry failed tests by restarting the worker. " +
"This is in addition to the normal retries " +
"and are only used when the normal retries are exhausted."
} )
.option( "verbose", {
alias: "v",
type: "boolean",
description: "Log additional information."
} )
.option( "run-id", {
type: "string",
description: "A unique identifier for this run."
} )
.option( "isolate", {
type: "boolean",
description: "Run each module by itself in the test page. This can extend testing time."
Expand All @@ -93,19 +101,9 @@ const argv = yargs( process.argv.slice( 2 ) )
"Otherwise, the --browser option will be used, " +
"with the latest version/device for that browser, on a matching OS."
} )
.option( "retries", {
alias: "r",
type: "number",
description: "Number of times to retry failed tests in BrowserStack.",
implies: [ "browserstack" ]
} )
.option( "hard-retries", {
type: "number",
description:
"Number of times to retry failed tests in BrowserStack " +
"by restarting the worker. This is in addition to the normal retries " +
"and are only used when the normal retries are exhausted.",
implies: [ "browserstack" ]
.option( "run-id", {
type: "string",
description: "A unique identifier for the run in BrowserStack."
} )
.option( "list-browsers", {
type: "string",
Expand Down
0 test/runner/jquery-migrate.js → test/runner/flags/jquery-migrate.js
View file
File renamed without changes.
0 test/runner/jquery.js → test/runner/flags/jquery.js
View file
File renamed without changes.
0 test/runner/modules.js → test/runner/flags/modules.js
View file
File renamed without changes.
11 changes: 11 additions & 0 deletions test/runner/listeners.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,17 @@
return nu;
}
}

// Serialize Symbols as string representations so they are
// sent over the wire after being stringified.
if ( typeof value === "symbol" ) {

// We can *describe* unique symbols, but note that their identity
// (e.g., `Symbol() !== Symbol()`) is lost
var ctor = Symbol.keyFor( value ) !== undefined ? "Symbol.for" : "Symbol";
return ctor + "(" + JSON.stringify( value.description ) + ")";
}

return value;
}
return derez( object );
Expand Down
Loading

0 comments on commit 8bec395

Please sign in to comment.