From 844c09a9a7900cab9af8d3aeffa53de8a44f2b3d Mon Sep 17 00:00:00 2001 From: "alexey.kamaev" Date: Mon, 20 Aug 2018 12:36:44 +0300 Subject: [PATCH] [WIP]Restart browser if it became unresponsive (closes #1815) --- Gulpfile.js | 2 +- src/browser/connection/index.js | 32 ++++++++++- src/runner/test-run-controller.js | 5 ++ src/test-run/index.js | 8 +++ .../browser-reconnect/pages/index.html | 10 ++++ .../browser-reconnect/test.js | 57 +++++++++++++++++++ .../testcafe-fixtures/index-test.js | 28 +++++++++ 7 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 test/functional/fixtures/browser-provider/browser-reconnect/pages/index.html create mode 100644 test/functional/fixtures/browser-provider/browser-reconnect/test.js create mode 100644 test/functional/fixtures/browser-provider/browser-reconnect/testcafe-fixtures/index-test.js diff --git a/Gulpfile.js b/Gulpfile.js index cca9ad70cca..c298f294768 100644 --- a/Gulpfile.js +++ b/Gulpfile.js @@ -625,7 +625,7 @@ function testFunctional (fixturesDir, testingEnvironmentName, browserProviderNam .pipe(mocha({ ui: 'bdd', reporter: 'spec', - timeout: typeof v8debug === 'undefined' ? 30000 : Infinity // NOTE: disable timeouts in debug + timeout: typeof v8debug === 'undefined' ? 180000 : Infinity // NOTE: disable timeouts in debug })); } diff --git a/src/browser/connection/index.js b/src/browser/connection/index.js index 07611e1aa21..6b0e3d3f4ae 100644 --- a/src/browser/connection/index.js +++ b/src/browser/connection/index.js @@ -10,6 +10,7 @@ import COMMAND from './command'; import STATUS from './status'; import { GeneralError } from '../../errors/runtime'; import MESSAGE from '../../errors/runtime/message'; +import testRunTracker from '../../api/test-run-tracker'; const IDLE_PAGE_TEMPLATE = read('../../client/browser/idle-page/index.html.mustache'); @@ -98,6 +99,8 @@ export default class BrowserConnection extends EventEmitter { try { await this.provider.closeBrowser(this.id); + + this.opened = false; } catch (err) { // NOTE: A warning would be really nice here, but it can't be done while log is stored in a task. @@ -112,9 +115,30 @@ export default class BrowserConnection extends EventEmitter { } } + _restartBrowser () { + this.ready = false; + + this._forceIdle(); + + this._closeBrowser() + .then(() => { + const testRun = this._getActiveTestRun(); + + testRun.stop(new GeneralError(MESSAGE.browserDisconnected, this.userAgent)); + + this._runBrowser(); + }); + } + _waitForHeartbeat () { this.heartbeatTimeout = setTimeout(() => { - this.emit('error', new GeneralError(MESSAGE.browserDisconnected, this.userAgent)); + const needRestartBrowser = true; // option + + if (needRestartBrowser) + this._restartBrowser(); + else + this.emit('error', new GeneralError(MESSAGE.browserDisconnected, this.userAgent)); + }, this.HEARTBEAT_TIMEOUT); } @@ -125,6 +149,12 @@ export default class BrowserConnection extends EventEmitter { return this.pendingTestRunUrl; } + _getActiveTestRun () { + const testRuns = Object.values(testRunTracker.activeTestRuns); + + return testRuns.find(tr => tr.browserConnection.id === this.id); + } + async _popNextTestRunUrl () { while (this.hasQueuedJobs && !this.currentJob.hasQueuedTestRuns) this.jobQueue.shift(); diff --git a/src/runner/test-run-controller.js b/src/runner/test-run-controller.js index 184cb820423..ef4be8f3b5f 100644 --- a/src/runner/test-run-controller.js +++ b/src/runner/test-run-controller.js @@ -91,6 +91,10 @@ export default class TestRunController extends EventEmitter { } _keepInQuarantine () { + this._restart(); + } + + _restart () { this.emit('test-run-restart'); } @@ -135,6 +139,7 @@ export default class TestRunController extends EventEmitter { testRun.once('start', () => this.emit('test-run-start')); testRun.once('done', () => this._testRunDone()); + testRun.once('stop', () => this._restart()); testRun.start(); diff --git a/src/test-run/index.js b/src/test-run/index.js index 247ee5beb8b..7858d1ae3de 100644 --- a/src/test-run/index.js +++ b/src/test-run/index.js @@ -376,6 +376,9 @@ export default class TestRun extends EventEmitter { } _rejectCurrentDriverTask (err) { + if (!this.currentDriverTask) + return; + err.callsite = err.callsite || this.driverTaskQueue[0].callsite; err.isRejectedDriverTask = true; @@ -666,6 +669,11 @@ export default class TestRun extends EventEmitter { return await getLocation(); } + + stop (err) { + this._rejectCurrentDriverTask(err); + this.emit('stop'); + } } diff --git a/test/functional/fixtures/browser-provider/browser-reconnect/pages/index.html b/test/functional/fixtures/browser-provider/browser-reconnect/pages/index.html new file mode 100644 index 00000000000..9fa7bf611d3 --- /dev/null +++ b/test/functional/fixtures/browser-provider/browser-reconnect/pages/index.html @@ -0,0 +1,10 @@ + + + + + Title + + +
+ + diff --git a/test/functional/fixtures/browser-provider/browser-reconnect/test.js b/test/functional/fixtures/browser-provider/browser-reconnect/test.js new file mode 100644 index 00000000000..84ef1fa2c79 --- /dev/null +++ b/test/functional/fixtures/browser-provider/browser-reconnect/test.js @@ -0,0 +1,57 @@ +const path = require('path'); +const Promise = require('pinkie'); +const expect = require('chai').expect; +const config = require('../../../config'); +const browserProviderPool = require('../../../../../lib/browser/provider/pool'); +const BrowserConnection = require('../../../../../lib/browser/connection'); + +let hasErrors = true; + +function customReporter () { + return { + reportTestDone (name, testRunInfo) { + hasErrors = !!testRunInfo.errs.length; + }, + reportFixtureStart () { + }, + reportTaskStart () { + }, + reportTaskDone () { + } + }; +} + +if (config.useLocalBrowsers) { + describe('Browser reconnect', function () { + async function run (pathToTest) { + const src = path.join(__dirname, pathToTest); + const aliases = config.currentEnvironment.browsers.map(browser => browser.alias); + + return Promise.all(aliases.map(alias => browserProviderPool.getBrowserInfo(alias))) + .then(browsers => { + const connections = browsers.map(browser => new BrowserConnection(testCafe.browserConnectionGateway, browser, true)); + + connections.forEach(connection => { + connection.HEARTBEAT_TIMEOUT = 4000; + }); + + return connections; + }) + .then(connection => { + return testCafe + .createRunner() + .src(src) + .reporter(customReporter) + .browsers(connection) + .run().then(() => { + expect(hasErrors).to.be.false; + }); + }); + } + + it.only('Should restart browser when it does not respond', function () { + return run('./testcafe-fixtures/index-test.js'); + }); + }); +} + diff --git a/test/functional/fixtures/browser-provider/browser-reconnect/testcafe-fixtures/index-test.js b/test/functional/fixtures/browser-provider/browser-reconnect/testcafe-fixtures/index-test.js new file mode 100644 index 00000000000..f5c813a22c2 --- /dev/null +++ b/test/functional/fixtures/browser-provider/browser-reconnect/testcafe-fixtures/index-test.js @@ -0,0 +1,28 @@ +import { ClientFunction } from 'testcafe'; + +fixture `Browser reconnect` + .page `http://localhost:3000/fixtures/browser-provider/browser-reconnect/pages/index.html`; + +const counter = {}; +const getUserAgent = ClientFunction(() => navigator.userAgent.toString()); + +const hang = ClientFunction(() => { + const now = Date.now(); + + while (Date.now() < now + 5000) { + // hang for 5s + } +}); + +test('Should restart browser when it does not respond', async t => { + const userAgent = await getUserAgent(); + + counter[userAgent] = counter[userAgent] || 0; + + counter[userAgent]++; + + if (counter[userAgent] < 3) + await hang(); + + await t.expect(counter[userAgent]).eql(3); +});