[WIP]Restart browser if it became unresponsive (closes DevExpress#1815)

AlexKamaev · Aug 20, 2018 · 844c09a · 844c09a
1 parent 19fd15d
commit 844c09a
Show file tree

Hide file tree

Showing 7 changed files with 140 additions and 2 deletions.
diff --git a/Gulpfile.js b/Gulpfile.js
@@ -625,7 +625,7 @@ function testFunctional (fixturesDir, testingEnvironmentName, browserProviderNam
         .pipe(mocha({
             ui:       'bdd',
             reporter: 'spec',
-            timeout:  typeof v8debug === 'undefined' ? 30000 : Infinity // NOTE: disable timeouts in debug
+            timeout:  typeof v8debug === 'undefined' ? 180000 : Infinity // NOTE: disable timeouts in debug
         }));
 }
 

diff --git a/src/browser/connection/index.js b/src/browser/connection/index.js
@@ -10,6 +10,7 @@ import COMMAND from './command';
 import STATUS from './status';
 import { GeneralError } from '../../errors/runtime';
 import MESSAGE from '../../errors/runtime/message';
+import testRunTracker from '../../api/test-run-tracker';
 
 const IDLE_PAGE_TEMPLATE = read('../../client/browser/idle-page/index.html.mustache');
 
@@ -98,6 +99,8 @@ export default class BrowserConnection extends EventEmitter {
 
         try {
             await this.provider.closeBrowser(this.id);
+
+            this.opened = false;
         }
         catch (err) {
             // NOTE: A warning would be really nice here, but it can't be done while log is stored in a task.
@@ -112,9 +115,30 @@ export default class BrowserConnection extends EventEmitter {
         }
     }
 
+    _restartBrowser () {
+        this.ready = false;
+
+        this._forceIdle();
+
+        this._closeBrowser()
+            .then(() => {
+                const testRun = this._getActiveTestRun();
+
+                testRun.stop(new GeneralError(MESSAGE.browserDisconnected, this.userAgent));
+
+                this._runBrowser();
+            });
+    }
+
     _waitForHeartbeat () {
         this.heartbeatTimeout = setTimeout(() => {
-            this.emit('error', new GeneralError(MESSAGE.browserDisconnected, this.userAgent));
+            const needRestartBrowser = true; // option
+
+            if (needRestartBrowser)
+                this._restartBrowser();
+            else
+                this.emit('error', new GeneralError(MESSAGE.browserDisconnected, this.userAgent));
+
         }, this.HEARTBEAT_TIMEOUT);
     }
 
@@ -125,6 +149,12 @@ export default class BrowserConnection extends EventEmitter {
         return this.pendingTestRunUrl;
     }
 
+    _getActiveTestRun () {
+        const testRuns = Object.values(testRunTracker.activeTestRuns);
+
+        return testRuns.find(tr => tr.browserConnection.id === this.id);
+    }
+
     async _popNextTestRunUrl () {
         while (this.hasQueuedJobs && !this.currentJob.hasQueuedTestRuns)
             this.jobQueue.shift();

diff --git a/src/runner/test-run-controller.js b/src/runner/test-run-controller.js
@@ -91,6 +91,10 @@ export default class TestRunController extends EventEmitter {
     }
 
     _keepInQuarantine () {
+        this._restart();
+    }
+
+    _restart () {
         this.emit('test-run-restart');
     }
 
@@ -135,6 +139,7 @@ export default class TestRunController extends EventEmitter {
 
         testRun.once('start', () => this.emit('test-run-start'));
         testRun.once('done', () => this._testRunDone());
+        testRun.once('stop', () => this._restart());
 
         testRun.start();
 

diff --git a/src/test-run/index.js b/src/test-run/index.js
@@ -376,6 +376,9 @@ export default class TestRun extends EventEmitter {
     }
 
     _rejectCurrentDriverTask (err) {
+        if (!this.currentDriverTask)
+            return;
+
         err.callsite             = err.callsite || this.driverTaskQueue[0].callsite;
         err.isRejectedDriverTask = true;
 
@@ -666,6 +669,11 @@ export default class TestRun extends EventEmitter {
 
         return await getLocation();
     }
+
+    stop (err) {
+        this._rejectCurrentDriverTask(err);
+        this.emit('stop');
+    }
 }
 
 

diff --git a/test/functional/fixtures/browser-provider/browser-reconnect/pages/index.html b/test/functional/fixtures/browser-provider/browser-reconnect/pages/index.html
@@ -0,0 +1,10 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Title</title>
+</head>
+<body>
+    <div id="log"></div>
+</body>
+</html>
diff --git a/test/functional/fixtures/browser-provider/browser-reconnect/test.js b/test/functional/fixtures/browser-provider/browser-reconnect/test.js
@@ -0,0 +1,57 @@
+const path                = require('path');
+const Promise             = require('pinkie');
+const expect              = require('chai').expect;
+const config              = require('../../../config');
+const browserProviderPool = require('../../../../../lib/browser/provider/pool');
+const BrowserConnection   = require('../../../../../lib/browser/connection');
+
+let hasErrors = true;
+
+function customReporter () {
+    return {
+        reportTestDone (name, testRunInfo) {
+            hasErrors = !!testRunInfo.errs.length;
+        },
+        reportFixtureStart () {
+        },
+        reportTaskStart () {
+        },
+        reportTaskDone () {
+        }
+    };
+}
+
+if (config.useLocalBrowsers) {
+    describe('Browser reconnect', function () {
+        async function run (pathToTest) {
+            const src     = path.join(__dirname, pathToTest);
+            const aliases = config.currentEnvironment.browsers.map(browser => browser.alias);
+
+            return Promise.all(aliases.map(alias => browserProviderPool.getBrowserInfo(alias)))
+                .then(browsers => {
+                    const connections = browsers.map(browser => new BrowserConnection(testCafe.browserConnectionGateway, browser, true));
+
+                    connections.forEach(connection => {
+                        connection.HEARTBEAT_TIMEOUT = 4000;
+                    });
+
+                    return connections;
+                })
+                .then(connection => {
+                    return testCafe
+                        .createRunner()
+                        .src(src)
+                        .reporter(customReporter)
+                        .browsers(connection)
+                        .run().then(() => {
+                            expect(hasErrors).to.be.false;
+                        });
+                });
+        }
+
+        it.only('Should restart browser when it does not respond', function () {
+            return run('./testcafe-fixtures/index-test.js');
+        });
+    });
+}
+
diff --git a/test/functional/fixtures/browser-provider/browser-reconnect/testcafe-fixtures/index-test.js b/test/functional/fixtures/browser-provider/browser-reconnect/testcafe-fixtures/index-test.js
@@ -0,0 +1,28 @@
+import { ClientFunction } from 'testcafe';
+
+fixture `Browser reconnect`
+    .page `http://localhost:3000/fixtures/browser-provider/browser-reconnect/pages/index.html`;
+
+const counter      = {};
+const getUserAgent = ClientFunction(() => navigator.userAgent.toString());
+
+const hang = ClientFunction(() => {
+    const now = Date.now();
+
+    while (Date.now() < now + 5000) {
+        // hang for 5s
+    }
+});
+
+test('Should restart browser when it does not respond', async t => {
+    const userAgent = await getUserAgent();
+
+    counter[userAgent] = counter[userAgent] || 0;
+
+    counter[userAgent]++;
+
+    if (counter[userAgent] < 3)
+        await hang();
+
+    await t.expect(counter[userAgent]).eql(3);
+});