From e8f4d57a432b4b52e2b6d5ecbcfdaca406debfa9 Mon Sep 17 00:00:00 2001 From: Alex Mos Date: Tue, 20 Oct 2015 14:46:58 +0300 Subject: [PATCH] Enable screenshot functionality (closes #104) --- package.json | 3 +- src/client/core/transport.js | 13 +- src/client/runner/api/actions.js | 4 +- src/client/runner/iframe-runner.js | 5 +- src/client/runner/runner-base.js | 8 +- src/client/runner/runner.js | 170 ++++++------ src/client/runner/step-iterator.js | 220 +++++++++------- src/client/test-run/index.js.mustache | 8 +- src/reporters/base.js | 31 ++- src/reporters/errors/decorators/plain-text.js | 4 + src/reporters/errors/decorators/tty.js | 4 + src/reporters/errors/templates.js | 66 ++++- src/reporters/json.js | 4 +- src/reporters/list.js | 5 +- src/reporters/spec.js | 5 +- src/reporters/xunit.js | 5 +- src/runner/index.js | 10 +- src/runner/test-run/command.js | 1 - src/runner/test-run/index.js | 48 ++-- src/runner/test-run/screenshot-capturer.js | 45 ++++ .../fixtures/runner/take-screenshots-test.js | 248 ------------------ test/report-design-viewer/errors.js | 4 +- test/report-design-viewer/index.js | 31 ++- test/server/data/expected-reports/json | 20 +- test/server/data/expected-reports/list | 5 +- test/server/data/expected-reports/minimal | 2 + test/server/data/expected-reports/spec | 4 +- test/server/data/expected-reports/xunit | 4 +- test/server/reporters-test.js | 58 ++-- 29 files changed, 502 insertions(+), 533 deletions(-) create mode 100644 src/runner/test-run/screenshot-capturer.js delete mode 100644 test/client/fixtures/runner/take-screenshots-test.js diff --git a/package.json b/package.json index 043ae7a74d9..440597ca74f 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,8 @@ "testcafe-browser-natives": "^0.10.0", "testcafe-hammerhead": "^0.2.1", "uglify-js": "1.2.6", - "useragent": "^2.1.7" + "useragent": "^2.1.7", + "uuid": "^2.0.1" }, "devDependencies": { "babel": "^5.8.23", diff --git a/src/client/core/transport.js b/src/client/core/transport.js index 665373333d8..98b659ec3dd 100644 --- a/src/client/core/transport.js +++ b/src/client/core/transport.js @@ -18,21 +18,14 @@ export function fail (err, callback) { err: err }; - transport.asyncServiceMsg(testFailMsg, function () { - callback(); - }); - - //HACK: this helps stop current JS context execution - window.onerror = function () { - }; - throw 'STOP'; + transport.asyncServiceMsg(testFailMsg, callback); } -export function assertionFailed (err) { +export function assertionFailed (err, callback) { var assertionFailedMsg = { cmd: COMMAND.assertionFailed, err: err }; - transport.asyncServiceMsg(assertionFailedMsg); + transport.asyncServiceMsg(assertionFailedMsg, callback); } diff --git a/src/client/runner/api/actions.js b/src/client/runner/api/actions.js index d9a73514921..6665da83be8 100644 --- a/src/client/runner/api/actions.js +++ b/src/client/runner/api/actions.js @@ -698,11 +698,11 @@ export function upload (what, path) { ); } -export function screenshot () { +export function screenshot (filePath) { stepIterator.asyncAction(function (iteratorCallback) { stepIterator.takeScreenshot(function () { iteratorCallback(); - }, false); + }, filePath); }); } diff --git a/src/client/runner/iframe-runner.js b/src/client/runner/iframe-runner.js index 43a356c5bd2..b56d1cb493e 100644 --- a/src/client/runner/iframe-runner.js +++ b/src/client/runner/iframe-runner.js @@ -73,8 +73,6 @@ IFrameRunner.prototype._onAssertionFailed = function (e) { err: e }; - this.stepIterator.state.needScreeshot = true; - messageSandbox.sendServiceMsg(msg, window.top); if (SETTINGS.get().PLAYBACK) @@ -111,7 +109,8 @@ IFrameRunner.prototype._onGetStepsSharedData = function (e) { IFrameRunner.prototype._onTakeScreenshot = function (e) { var msg = { cmd: RunnerBase.IFRAME_TAKE_SCREENSHOT_REQUEST_CMD, - isFailedStep: e.isFailedStep + stepName: e.stepName, + filePath: e.filePath }; messageSandbox.sendServiceMsg(msg, window.top); diff --git a/src/client/runner/runner-base.js b/src/client/runner/runner-base.js index 95cf94b3b72..a2e38db225a 100644 --- a/src/client/runner/runner-base.js +++ b/src/client/runner/runner-base.js @@ -105,6 +105,9 @@ var RunnerBase = function () { //NOTE: start test execution only when all content is loaded or if loading //timeout is reached (whichever comes first). runner._prepareStepsExecuting(function () { + if (runner.stopped) + return; + delete runner.act._onJSError; delete runner.act._start; @@ -211,7 +214,7 @@ RunnerBase.prototype._initIFrameBehavior = function () { runner.executingStepInIFrameWindow = null; message.err.stepNum = runner.stepIterator.state.step - 1; - runner._onAssertionFailed(message.err, true); + runner._onAssertionFailed(message.err); break; case RunnerBase.IFRAME_GET_SHARED_DATA_REQUEST_CMD: @@ -254,7 +257,8 @@ RunnerBase.prototype._initIFrameBehavior = function () { case RunnerBase.IFRAME_TAKE_SCREENSHOT_REQUEST_CMD: runner._onTakeScreenshot({ - isFailedStep: message.isFailedStep, + stepName: message.stepName, + filePath: message.filePath, callback: function () { msg = { cmd: RunnerBase.IFRAME_TAKE_SCREENSHOT_RESPONSE_CMD diff --git a/src/client/runner/runner.js b/src/client/runner/runner.js index 568203a3fd8..cbebc1d899c 100644 --- a/src/client/runner/runner.js +++ b/src/client/runner/runner.js @@ -1,3 +1,4 @@ +import { Promise } from 'es6-promise'; import hammerhead from './deps/hammerhead'; import testCafeCore from './deps/testcafe-core'; import RunnerBase from './runner-base'; @@ -14,7 +15,6 @@ var serviceUtils = testCafeCore.serviceUtils; const WAITING_FOR_SERVICE_MESSAGES_COMPLETED_DELAY = 1000; - var Runner = function (startedCallback) { var runner = this; @@ -80,44 +80,80 @@ Runner.prototype._onActionRun = function () { transport.asyncServiceMsg(msg); }; -Runner.prototype._onError = function (err) { - var runner = this; +Runner.prototype._beforeScreenshot = function () { + this.stepIterator.suspend(); + this.eventEmitter.emit(RunnerBase.SCREENSHOT_CREATING_STARTED_EVENT, {}); + this.savedDocumentTitle = document.title; - if (this.stopped) - return; + var assignedTitle = `[ ${window.location.toString()} ]`; - //NOTE: we should stop stepIterator to prevent playback after an error is occurred - this.stepIterator.stop(); + // NOTE: we should keep page url in the + // document.title while screenshot creating run + this.checkTitleIntervalId = window.setInterval(() => { + if (document.title !== assignedTitle) { + this.savedDocumentTitle = document.title; + document.title = assignedTitle; + } + }, 50); - RunnerBase.prototype._onError.call(this, err); + document.title = assignedTitle; + + return new Promise(resolve => window.setTimeout(resolve), 500); +}; + +Runner.prototype._afterScreenshot = function () { + window.clearInterval(this.checkTitleIntervalId); + document.title = this.savedDocumentTitle; + this.checkTitleIntervalId = null; + this.savedDocumentTitle = null; + + this.eventEmitter.emit(RunnerBase.SCREENSHOT_CREATING_FINISHED_EVENT, {}); + this.stepIterator.resume(); + + return new Promise(resolve => window.setTimeout(resolve), 100); +}; + +Runner.prototype._onTestFail = function (err, isAssertion) { + // NOTE: we should not create several screenshots for a step + err.screenshotRequired = SETTINGS.get().TAKE_SCREENSHOTS && SETTINGS.get().TAKE_SCREENSHOT_ON_FAILS && + this.stepIterator.state.curStepErrors.length < 2; + err.pageUrl = document.location.toString(); + + return new Promise(resolve => { + if (err.screenshotRequired) + return this._beforeScreenshot().then(resolve); + + resolve(); + }) + .then(() => new Promise(resolve => { + if (isAssertion) + transport.assertionFailed(err, resolve); + else + transport.fail(err, resolve); + })) + .then(() => new Promise(resolve => { + if (err.screenshotRequired) + return this._afterScreenshot().then(resolve); + + resolve(); + })); +}; - if (!SETTINGS.get().TAKE_SCREENSHOT_ON_FAILS) { - this.stopped = true; - transport.fail(err, Runner.checkStatus); +Runner.prototype._onError = function (err) { + if (this.stopped) return; - } - var setErrorMsg = { - cmd: COMMAND.setTestError, - err: err - }; + this.stopped = true; + this.stepIterator.stop(); - transport.asyncServiceMsg(setErrorMsg); + RunnerBase.prototype._onError.call(this, err); - this._onTakeScreenshot({ - isFailedStep: true, - //TODO: - //withoutStepName: !(ERRORS.hasErrorStepName(err) && ERRORS.hasErrorStepName(err)), - callback: function () { - runner.stopped = true; - transport.fail(err, Runner.checkStatus); - } - }); + this._onTestFail(err) + .then(Runner.checkStatus); }; -Runner.prototype._onAssertionFailed = function (e, inIFrame) { - this.stepIterator.state.needScreeshot = !inIFrame; - transport.assertionFailed(e.err); +Runner.prototype._onAssertionFailed = function (e) { + this._onTestFail(e.err, true); }; Runner.prototype._onSetStepsSharedData = function (e) { @@ -138,62 +174,28 @@ Runner.prototype._onGetStepsSharedData = function (e) { }; Runner.prototype._onTakeScreenshot = function (e) { - var savedTitle = document.title, - windowMark = '[tc-' + Date.now() + ']', - browserName = null, - callback = e && e.callback ? e.callback : function () { - }, - runner = this; - - runner.eventEmitter.emit(RunnerBase.SCREENSHOT_CREATING_STARTED_EVENT, {}); - - - if (browserUtils.isMSEdge) - browserName = 'MSEDGE'; - else if (browserUtils.isSafari) - browserName = 'SAFARI'; - else if (browserUtils.isOpera || browserUtils.isOperaWithWebKit) - browserName = 'OPERA'; - else if (browserUtils.isWebKit) - browserName = 'CHROME'; - else if (browserUtils.isMozilla) - browserName = 'FIREFOX'; - else if (browserUtils.isIE) - browserName = 'IE'; - - var msg = { - cmd: 'CMD_TAKE_SCREENSHOT', //TODO: fix - windowMark: windowMark, - browserName: browserName, - isFailedStep: e.isFailedStep, - withoutStepName: e.withoutStepName, - url: window.location.toString() - }; - - var assignedTitle = savedTitle + windowMark, - checkTitleIntervalId = window.setInterval(function () { - if (document.title !== assignedTitle) { - savedTitle = document.title; - document.title = assignedTitle; - } - }, 50); - - document.title = assignedTitle; - - //NOTE: we should set timeouts to changing of document title - //in any case we are waiting response from server - window.setTimeout(function () { - transport.asyncServiceMsg(msg, function () { - window.clearInterval(checkTitleIntervalId); - checkTitleIntervalId = null; - document.title = savedTitle; - runner.eventEmitter.emit(RunnerBase.SCREENSHOT_CREATING_FINISHED_EVENT, {}); - - window.setTimeout(function () { - callback(); - }, 100); + if (!SETTINGS.get().TAKE_SCREENSHOTS) + return typeof e.callback === 'function' ? e.callback() : null; + + this + ._beforeScreenshot() + .then(() => { + return new Promise(resolve => { + var msg = { + cmd: COMMAND.takeScreenshot, + pageUrl: document.location.toString(), + stepName: e.stepName, + filePath: e.filePath + }; + + transport.asyncServiceMsg(msg, resolve); + }); + }) + .then(this._afterScreenshot.bind(this)) + .then(() => { + if (typeof e.callback === 'function') + e.callback(); }); - }, 500); }; Runner.prototype._onDialogsInfoChanged = function (info) { diff --git a/src/client/runner/step-iterator.js b/src/client/runner/step-iterator.js index 76bd4ad1578..50e71cf092b 100644 --- a/src/client/runner/step-iterator.js +++ b/src/client/runner/step-iterator.js @@ -19,6 +19,12 @@ const STEP_DELAY = 500; const PROLONGED_STEP_DELAY = 3000; const SHORT_PROLONGED_STEP_DELAY = 30; +const SUSPEND_ACTIONS = { + runStep: 'runStep', + asyncAction: 'asyncAction', + asyncActionSeries: 'asyncActionSeries' +}; + //Iterator var StepIterator = function (pingIFrame) { @@ -34,8 +40,11 @@ var StepIterator = function (pingIFrame) { stepsSharedData: {}, lastSyncedSharedDataJSON: null, stopped: false, + suspended: false, + suspendedAction: null, + suspendedArgs: [], waitedIFrame: null, - needScreeshot: false + curStepErrors: [] }; this.pingIFrame = pingIFrame; @@ -81,7 +90,13 @@ StepIterator.prototype._checkSharedDataSerializable = function () { }; StepIterator.prototype._runStep = function () { - this.state.stopped = false; + if (this.state.suspended) { + this.state.suspendedAction = SUSPEND_ACTIONS.runStep; + return; + } + + this.state.stopped = false; + this.state.curStepErrors = []; var iterator = this; @@ -158,14 +173,7 @@ StepIterator.prototype._runStep = function () { if (!iterator._checkSharedDataSerializable()) return; - if (SETTINGS.get().TAKE_SCREENSHOT_ON_FAILS && iterator.state.needScreeshot) { - iterator.takeScreenshot(function () { - iterator.state.needScreeshot = false; - runCallback(); - }, true); - } - else - runCallback(); + runCallback(); } }); } @@ -313,103 +321,93 @@ StepIterator.prototype._checkIFrame = function (element, callback) { }; StepIterator.prototype.asyncAction = function (action) { + if (this.state.suspended) { + this.state.suspendedAction = SUSPEND_ACTIONS.asyncAction; + this.state.suspendedArgs = arguments; + return; + } + var iterator = this; this.state.inAsyncAction = true; - var actionRun = function () { - iterator._syncSharedDataWithServer(function () { - actionBarrier.waitActionSideEffectsCompletion(action, function () { - iterator._completeAsyncAction.apply(iterator, arguments); - }); + iterator._syncSharedDataWithServer(function () { + actionBarrier.waitActionSideEffectsCompletion(action, function () { + iterator._completeAsyncAction.apply(iterator, arguments); }); - }; - - if (SETTINGS.get().TAKE_SCREENSHOT_ON_FAILS && this.state.needScreeshot) { - this.takeScreenshot(function () { - iterator.state.needScreeshot = false; - actionRun(); - }, true); - } - else - actionRun(); + }); }; StepIterator.prototype.asyncActionSeries = function (items, runArgumentsIterator, action) { + if (this.state.suspended) { + this.state.suspendedAction = SUSPEND_ACTIONS.asyncActionSeries; + this.state.suspendedArgs = arguments; + return; + } + var iterator = this; - var actionsRun = function () { - var seriesActionsRun = function (elements, callback) { - async.forEachSeries( - elements, - function (element, asyncCallback) { - //NOTE: since v.14.1.5 it's recommended to run actions with the inIFrame function. But we should support old-style iframes - //using, so, it'll be resolved here. - iterator._checkIFrame(element, function (iframe) { - if (!iframe) { - actionBarrier.waitActionSideEffectsCompletion(function (barrierCallback) { - action(element, barrierCallback); - }, asyncCallback); - } - else { - var iFrameStartXhrBarrier = iframe.contentWindow[xhrBarrier.GLOBAL_START_XHR_BARRIER], - iFrameWaitXhrBarrier = iframe.contentWindow[xhrBarrier.GLOBAL_WAIT_XHR_BARRIER]; - - actionBarrier.waitActionSideEffectsCompletion(function (barrierCallback) { - var iFrameBeforeUnloadRaised = false; - - iterator.iFrameActionCallback = function () { - iterator.iFrameActionCallback = null; - iterator.waitedIFrame = null; - barrierCallback(); - }; - - iterator.waitedIFrame = iframe; - - iFrameStartXhrBarrier(function () { - if (!iFrameBeforeUnloadRaised) - iterator.iFrameActionCallback(); - }); - - function onBeforeUnload () { - nativeMethods.windowRemoveEventListener.call(iframe.contentWindow, 'beforeunload', onBeforeUnload); - iFrameBeforeUnloadRaised = true; - } - - nativeMethods.windowAddEventListener.call(iframe.contentWindow, 'beforeunload', onBeforeUnload, true); - - action(element, function () { - iFrameWaitXhrBarrier(); - }, iframe); - }, asyncCallback); - } - }); - }, - function () { - if (iterator.state.stopped) - return; + iterator.state.inAsyncAction = true; - callback(); + var seriesActionsRun = function (elements, callback) { + async.forEachSeries( + elements, + function (element, asyncCallback) { + //NOTE: since v.14.1.5 it's recommended to run actions with the inIFrame function. But we should support old-style iframes + //using, so, it'll be resolved here. + iterator._checkIFrame(element, function (iframe) { + if (!iframe) { + actionBarrier.waitActionSideEffectsCompletion(function (barrierCallback) { + action(element, barrierCallback); + }, asyncCallback); + } + else { + var iFrameStartXhrBarrier = iframe.contentWindow[xhrBarrier.GLOBAL_START_XHR_BARRIER], + iFrameWaitXhrBarrier = iframe.contentWindow[xhrBarrier.GLOBAL_WAIT_XHR_BARRIER]; + + actionBarrier.waitActionSideEffectsCompletion(function (barrierCallback) { + var iFrameBeforeUnloadRaised = false; + + iterator.iFrameActionCallback = function () { + iterator.iFrameActionCallback = null; + iterator.waitedIFrame = null; + barrierCallback(); + }; + + iterator.waitedIFrame = iframe; + + iFrameStartXhrBarrier(function () { + if (!iFrameBeforeUnloadRaised) + iterator.iFrameActionCallback(); + }); + + function onBeforeUnload () { + nativeMethods.windowRemoveEventListener.call(iframe.contentWindow, 'beforeunload', onBeforeUnload); + iFrameBeforeUnloadRaised = true; + } + + nativeMethods.windowAddEventListener.call(iframe.contentWindow, 'beforeunload', onBeforeUnload, true); + + action(element, function () { + iFrameWaitXhrBarrier(); + }, iframe); + }, asyncCallback); + } }); - }; + }, + function () { + if (iterator.state.stopped) + return; - iterator._syncSharedDataWithServer(function () { - runArgumentsIterator(items, seriesActionsRun, function () { - iterator._completeAsyncAction.apply(iterator, arguments); + callback(); }); - }); }; - iterator.state.inAsyncAction = true; - - if (SETTINGS.get().TAKE_SCREENSHOT_ON_FAILS && this.state.needScreeshot) { - this.takeScreenshot(function () { - iterator.state.needScreeshot = false; - actionsRun(); - }, true); - } - else - actionsRun(); + iterator._syncSharedDataWithServer(function () { + runArgumentsIterator(items, seriesActionsRun, function () { + iterator._completeAsyncAction.apply(iterator, arguments); + }); + }); }; StepIterator.prototype._init = function () { @@ -467,6 +465,8 @@ StepIterator.prototype.onError = function (err) { if (this.state.stopped) return; + this.state.curStepErrors.push(err); + this.eventEmitter.emit(StepIterator.ERROR_EVENT, $.extend({ stepNum: this.state.step - 1 }, err)); @@ -476,6 +476,8 @@ StepIterator.prototype.onAssertionFailed = function (err) { if (this.state.stopped) return; + this.state.curStepErrors.push(err); + this.eventEmitter.emit(StepIterator.ASSERTION_FAILED_EVENT, { err: err, stepNum: this.state.step - 1, @@ -493,6 +495,9 @@ StepIterator.prototype.runLast = function () { }; StepIterator.prototype.getCurrentStep = function () { + if (!this.initialized) + return ''; + return this.state.stepNames ? this.state.stepNames[this.state.step - 1] : SETTINGS.get().CURRENT_TEST_STEP_NAME; }; @@ -508,6 +513,32 @@ StepIterator.prototype.onActionRun = function () { this.eventEmitter.emit(StepIterator.ACTION_RUN_EVENT, {}); }; +StepIterator.prototype.suspend = function () { + // NOTE: in fact we can suspend the iterator before + // the next step or an async action running + this.state.suspended = true; +}; + +StepIterator.prototype.resume = function () { + if (!this.state.suspended || this.state.stopped) + return; + + this.state.suspended = false; + + if (this.state.suspendedAction === SUSPEND_ACTIONS.runStep) + this._runStep(); + + if (this.state.suspendedAction === SUSPEND_ACTIONS.asyncAction) + this.asyncAction.apply(this, this.state.suspendedArgs); + + if (this.state.suspendedAction === SUSPEND_ACTIONS.asyncActionSeries) + this.asyncActionSeries.apply(this, this.state.suspendedArgs); + + this.state.suspendedAction = null; + this.state.suspendedArgs = []; +}; + + //Global __waitFor() StepIterator.prototype.setGlobalWaitFor = function (event, timeout) { this.globalWaitForEvent = event; @@ -540,10 +571,11 @@ StepIterator.prototype.__waitFor = function (callback) { }); }; -StepIterator.prototype.takeScreenshot = function (callback, isFailedStep) { +StepIterator.prototype.takeScreenshot = function (callback, filePath) { this.eventEmitter.emit(StepIterator.TAKE_SCREENSHOT_EVENT, { - isFailedStep: isFailedStep, - callback: callback + stepName: this.getCurrentStep(), + filePath: filePath || '', + callback: callback }); }; diff --git a/src/client/test-run/index.js.mustache b/src/client/test-run/index.js.mustache index 0abcd00575d..164a1064922 100644 --- a/src/client/test-run/index.js.mustache +++ b/src/client/test-run/index.js.mustache @@ -41,6 +41,7 @@ testCafeCore.SETTINGS.set({ CURRENT_TEST_STEP_NAME: nextStep ? stepNames[nextStep - 1] : 'Test initialization', BROWSER_STATUS_URL: '{{{browserStatusUrl}}}', + TAKE_SCREENSHOTS: {{{takeScreenshots}}}, TAKE_SCREENSHOT_ON_FAILS: {{{takeScreenshotOnFails}}}, SKIP_JS_ERRORS: {{{skipJsErrors}}}, ENABLE_SOURCE_INDEX: true, @@ -69,13 +70,6 @@ sandboxedJQuery.init(window, undefined); jQuerySelectorExtensions.init(); - var testError = {{{testError}}}; - - if (testError) { - transport.fail(testError, Runner.checkStatus); - return; - } - $ = jQuery = jQuerySelectorExtensions.create(sandboxedJQuery.jQuery); jQueryDataMethodProxy.setup($); diff --git a/src/reporters/base.js b/src/reporters/base.js index 4a7a14a5bf0..aacc225248a 100644 --- a/src/reporters/base.js +++ b/src/reporters/base.js @@ -41,13 +41,15 @@ export default class BaseReporter { static _createReportItem (test, runsPerTest) { return { - fixtureName: test.fixture.name, - fixturePath: test.fixture.path, - testName: test.name, - pendingRuns: runsPerTest, - errs: [], - unstable: false, - startTime: null + fixtureName: test.fixture.name, + fixturePath: test.fixture.path, + testName: test.name, + screenshotPath: test.screenshotPath, + hasActionScreenshots: false, + pendingRuns: runsPerTest, + errs: [], + unstable: false, + startTime: null }; } @@ -77,15 +79,16 @@ export default class BaseReporter { // This happens because the next test can't be completed until the // previous one frees all browser connections. // Therefore, tests always get completed sequentially. - var reportItem = this.reportQueue.shift(); - var durationMs = new Date() - reportItem.startTime; + var reportItem = this.reportQueue.shift(); + var durationMs = new Date() - reportItem.startTime; + var screenshotPath = reportItem.hasActionScreenshots ? reportItem.screenshotPath : null; if (!reportItem.errs.length) this.passed++; BaseReporter._errorSorter(reportItem.errs); - this._reportTestDone(reportItem.testName, reportItem.errs, durationMs, reportItem.unstable); + this._reportTestDone(reportItem.testName, reportItem.errs, durationMs, reportItem.unstable, screenshotPath); // NOTE: here we assume that tests are sorted by fixture. // Therefore, if the next report item has a different @@ -120,8 +123,10 @@ export default class BaseReporter { testRun.errs.forEach(err => err.userAgent = userAgent); reportItem.pendingRuns--; - reportItem.errs = reportItem.errs.concat(testRun.errs); - reportItem.unstable = reportItem.unstable || testRun.unstable; + reportItem.errs = reportItem.errs.concat(testRun.errs); + reportItem.unstable = reportItem.unstable || testRun.unstable; + reportItem.hasActionScreenshots = reportItem.hasActionScreenshots || testRun.hasActionScreenshots; + reportItem.screenshotPath = testRun.test.screenshotPath; if (!reportItem.pendingRuns) this._shiftReportQueue(); @@ -182,7 +187,7 @@ export default class BaseReporter { throw new Error('Not implemented'); } - _reportTestDone (name, errs, durationMs, unstable) { + _reportTestDone (name, errs, durationMs, unstable, screenshotPath) { throw new Error('Not implemented'); } diff --git a/src/reporters/errors/decorators/plain-text.js b/src/reporters/errors/decorators/plain-text.js index f68248b5634..438799d71e5 100644 --- a/src/reporters/errors/decorators/plain-text.js +++ b/src/reporters/errors/decorators/plain-text.js @@ -9,6 +9,10 @@ export default { 'span user-agent': str => str, + 'div screenshot-info': str => str, + + 'a screenshot-path': str => str, + 'code': str => str, 'code step-source': str => indentString(str, ' ', CODE_ALIGN_SPACES), diff --git a/src/reporters/errors/decorators/tty.js b/src/reporters/errors/decorators/tty.js index d745c0038d0..b7879ba4cf4 100644 --- a/src/reporters/errors/decorators/tty.js +++ b/src/reporters/errors/decorators/tty.js @@ -10,6 +10,10 @@ export default { 'span user-agent': str => chalk.gray(str), + 'div screenshot-info': str => str, + + 'a screenshot-path': str => chalk.underline(str), + 'code': str => chalk.yellow(str), 'code step-source': str => chalk.magenta(indentString(str, ' ', CODE_ALIGN_SPACES)), diff --git a/src/reporters/errors/templates.js b/src/reporters/errors/templates.js index 86e15ac9c24..111de4d2226 100644 --- a/src/reporters/errors/templates.js +++ b/src/reporters/errors/templates.js @@ -42,6 +42,12 @@ function getDiffHeader (err) { return ''; } +function getScreenshotInfoStr (err) { + return err.screenshotPath ? + `
Screenshot: ${err.screenshotPath}
` : + ''; +} + export default { [TYPE.okAssertion]: err => dedent` ${getAssertionMsgPrefix(err)} failed at step ${err.stepName}: @@ -50,6 +56,8 @@ export default { Expected: not null, not undefined, not false, not NaN and not '' Actual: ${escapeNewLines(err.actual)} + + ${getScreenshotInfoStr(err)} `, [TYPE.notOkAssertion]: err => dedent` @@ -59,6 +67,8 @@ export default { Expected: null, undefined, false, NaN or '' Actual: ${escapeNewLines(err.actual)} + + ${getScreenshotInfoStr(err)} `, [TYPE.eqAssertion]: (err, maxStringLength) => { @@ -75,6 +85,8 @@ export default { Expected: ${escapeNewLines(diff.expected)} Actual: ${escapeNewLines(diff.actual)} ${diffMarkerStr} + + ${getScreenshotInfoStr(err)} `; }, @@ -85,37 +97,51 @@ export default { Expected: not ${escapeNewLines(err.actual)} Actual: ${escapeNewLines(err.actual)} + + ${getScreenshotInfoStr(err)} `, [TYPE.iframeLoadingTimeout]: err => dedent` ${getMsgPrefix(err, CATEGORY.timeout)}IFrame loading timed out. + + ${getScreenshotInfoStr(err)} `, [TYPE.inIFrameTargetLoadingTimeout]: err => dedent` ${getMsgPrefix(err, CATEGORY.timeout)}Error at step ${err.stepName}: IFrame target loading timed out. + + ${getScreenshotInfoStr(err)} `, [TYPE.uncaughtJSError]: err => { if (err.pageUrl) { return dedent` ${getMsgPrefix(err, CATEGORY.unhandledException)}Uncaught JavaScript error ${err.scriptErr} on page ${err.pageUrl} + + ${getScreenshotInfoStr(err)} `; } return dedent` ${getMsgPrefix(err, CATEGORY.unhandledException)}Uncaught JavaScript error ${err.scriptErr} on page. + + ${getScreenshotInfoStr(err)} `; }, [TYPE.uncaughtJSErrorInTestCodeStep]: err => dedent` ${getMsgPrefix(err, CATEGORY.unhandledException)}Error at step ${err.stepName}: Uncaught JavaScript error in test code - ${err.scriptErr}. + + ${getScreenshotInfoStr(err)} `, [TYPE.storeDomNodeOrJqueryObject]: err => dedent` ${getMsgPrefix(err, CATEGORY.unhandledException)}Error at step ${err.stepName}: It is not allowed to share the DOM element, jQuery object or a function between test steps via "this" object. + + ${getScreenshotInfoStr(err)} `, [TYPE.emptyFirstArgument]: err => dedent` @@ -125,6 +151,8 @@ export default { A target element of the ${err.action} action has not been found in the DOM tree. If this element should be created after animation or a time-consuming operation is finished, use the waitFor action (available for use in code) to pause test execution until this element appears. + + ${getScreenshotInfoStr(err)} `, [TYPE.invisibleActionElement]: err => dedent` @@ -134,6 +162,8 @@ export default { A target element ${err.element} of the ${err.action} action is not visible. If this element should appear when you are hovering over another element, make sure that you properly recorded the hover action. + + ${getScreenshotInfoStr(err)} `, [TYPE.incorrectDraggingSecondArgument]: err => dedent` @@ -142,6 +172,8 @@ export default { ${getStepCode(err.relatedSourceCode)} drag action drop target is incorrect. + + ${getScreenshotInfoStr(err)} `, [TYPE.incorrectPressActionArgument]: err => dedent` @@ -150,6 +182,8 @@ export default { ${getStepCode(err.relatedSourceCode)} press action parameter contains incorrect key code. + + ${getScreenshotInfoStr(err)} `, [TYPE.emptyTypeActionArgument]: err => dedent` @@ -158,16 +192,22 @@ export default { ${getStepCode(err.relatedSourceCode)} The type action's parameter text is empty. + + ${getScreenshotInfoStr(err)} `, [TYPE.unexpectedDialog]: err => dedent` ${getMsgPrefix(err, CATEGORY.nativeDialogError)}Error at step ${err.stepName}: Unexpected system ${err.dialog} dialog ${err.message} appeared. + + ${getScreenshotInfoStr(err)} `, [TYPE.expectedDialogDoesntAppear]: err => dedent` ${getMsgPrefix(err, CATEGORY.nativeDialogError)}Error at step ${err.stepName}: The expected system ${err.dialog} dialog did not appear. + + ${getScreenshotInfoStr(err)} `, [TYPE.incorrectSelectActionArguments]: err => dedent` @@ -176,6 +216,8 @@ export default { ${getStepCode(err.relatedSourceCode)} select action's parameters contain an incorrect value. + + ${getScreenshotInfoStr(err)} `, [TYPE.incorrectWaitActionMillisecondsArgument]: err => dedent` @@ -184,6 +226,8 @@ export default { ${getStepCode(err.relatedSourceCode)} wait action's "milliseconds" parameter should be a positive number. + + ${getScreenshotInfoStr(err)} `, [TYPE.incorrectWaitForActionEventArgument]: err => dedent` @@ -192,6 +236,8 @@ export default { ${getStepCode(err.relatedSourceCode)} waitFor action's first parameter should be a function, a CSS selector or an array of CSS selectors. + + ${getScreenshotInfoStr(err)} `, [TYPE.incorrectWaitForActionTimeoutArgument]: err => dedent` @@ -200,6 +246,8 @@ export default { ${getStepCode(err.relatedSourceCode)} waitFor action's "timeout" parameter should be a positive number. + + ${getScreenshotInfoStr(err)} `, [TYPE.waitForActionTimeoutExceeded]: err => dedent` @@ -208,6 +256,8 @@ export default { ${getStepCode(err.relatedSourceCode)} waitFor action's timeout exceeded. + + ${getScreenshotInfoStr(err)} `, [TYPE.emptyIFrameArgument]: err => dedent` @@ -216,6 +266,8 @@ export default { ${getStepCode(err.relatedSourceCode)} The selector within the inIFrame function returns an empty value. + + ${getScreenshotInfoStr(err)} `, [TYPE.iframeArgumentIsNotIFrame]: err => dedent` @@ -224,6 +276,8 @@ export default { ${getStepCode(err.relatedSourceCode)} The selector within the inIFrame function doesn’t return an iframe element. + + ${getScreenshotInfoStr(err)} `, [TYPE.multipleIFrameArgument]: err => dedent` @@ -232,6 +286,8 @@ export default { ${getStepCode(err.relatedSourceCode)} The selector within the inIFrame function returns more than one iframe element. + + ${getScreenshotInfoStr(err)} `, [TYPE.incorrectIFrameArgument]: err => dedent` @@ -240,6 +296,8 @@ export default { ${getStepCode(err.relatedSourceCode)} The inIFrame function contains an invalid argument. + + ${getScreenshotInfoStr(err)} `, [TYPE.uploadCanNotFindFileToUpload]: err => { @@ -251,7 +309,9 @@ export default { Cannot find the following file(s) to upload: `; - return msg + err.filePaths.map(path => `\n ${path}`).join(','); + return msg + + err.filePaths.map(path => `\n ${path}`).join(',') + + (err.screenshotPath ? `\n\n${getScreenshotInfoStr(err)}` : ''); }, [TYPE.uploadElementIsNotFileInput]: err => dedent` @@ -260,6 +320,8 @@ export default { ${getStepCode(err.relatedSourceCode)} upload action argument does not contain a file input element. + + ${getScreenshotInfoStr(err)} `, [TYPE.uploadInvalidFilePathArgument]: err => dedent` @@ -268,6 +330,8 @@ export default { ${getStepCode(err.relatedSourceCode)} upload action's "path" parameter should be a string or an array of strings. + + ${getScreenshotInfoStr(err)} `, [TYPE.pageNotLoaded]: err => dedent` diff --git a/src/reporters/json.js b/src/reporters/json.js index 0ef8b6d2f70..7f72c1d718f 100644 --- a/src/reporters/json.js +++ b/src/reporters/json.js @@ -27,10 +27,10 @@ export default class JSONReporter extends BaseReporter { this.report.fixtures.push(this.currentFixture); } - _reportTestDone (name, errs, durationMs, unstable) { + _reportTestDone (name, errs, durationMs, unstable, screenshotPath) { errs = errs.map(err => this._formatError(err)); - this.currentFixture.tests.push({ name, errs, durationMs, unstable }); + this.currentFixture.tests.push({ name, errs, durationMs, unstable, screenshotPath }); } _reportTaskDone (passed, total, endTime) { diff --git a/src/reporters/list.js b/src/reporters/list.js index 23409d4fcf9..d2c99b3d0f5 100644 --- a/src/reporters/list.js +++ b/src/reporters/list.js @@ -11,7 +11,7 @@ export default class ListReporter extends SpecReporter { this.currentFixtureName = name; } - _reportTestDone (name, errs, durationMs, unstable) { + _reportTestDone (name, errs, durationMs, unstable, screenshotPath) { var hasErr = !!errs.length; var nameStyle = hasErr ? this.style.red : this.style.gray; var symbol = hasErr ? this.style.red(this.symbols.err) : this.style.green(this.symbols.ok); @@ -25,6 +25,9 @@ export default class ListReporter extends SpecReporter { if (unstable) title += this.style.yellow(' (unstable)'); + if (screenshotPath) + title += ` (screenshots: ${this.style.underline(screenshotPath)})`; + this._write(title); if (hasErr) { diff --git a/src/reporters/spec.js b/src/reporters/spec.js index 1919eeb178d..0c8eb38f2c1 100644 --- a/src/reporters/spec.js +++ b/src/reporters/spec.js @@ -36,7 +36,7 @@ export default class SpecReporter extends BaseReporter { ._newline(); } - _reportTestDone (name, errs, durationMs, unstable) { + _reportTestDone (name, errs, durationMs, unstable, screenshotPath) { var hasErr = !!errs.length; var nameStyle = hasErr ? this.style.red : this.style.gray; var symbol = hasErr ? this.style.red(this.symbols.err) : this.style.green(this.symbols.ok); @@ -47,6 +47,9 @@ export default class SpecReporter extends BaseReporter { if (unstable) title += this.style.yellow(' (unstable)'); + if (screenshotPath) + title += ` (screenshots: ${this.style.underline(screenshotPath)})`; + this._write(title); if (hasErr) { diff --git a/src/reporters/xunit.js b/src/reporters/xunit.js index 489338d87a9..b0e749bcab9 100644 --- a/src/reporters/xunit.js +++ b/src/reporters/xunit.js @@ -25,12 +25,15 @@ export default class XUnitReporter extends BaseReporter { this.currentFixtureName = escapeHtml(name); } - _reportTestDone (name, errs, durationMs, unstable) { + _reportTestDone (name, errs, durationMs, unstable, screenshotPath) { var hasErr = !!errs.length; if (unstable) name += ' (unstable)'; + if (screenshotPath) + name += ` (screenshots: ${screenshotPath})`; + name = escapeHtml(name); var openTag = ` test.screenshotPath = join(this.opts.screenshotPath, uuid.v4().substr(0, 8))); + return await this._runTask(Reporter, browserConnections, tests); } } diff --git a/src/runner/test-run/command.js b/src/runner/test-run/command.js index 2a796aeee85..cc3f2a4f21e 100644 --- a/src/runner/test-run/command.js +++ b/src/runner/test-run/command.js @@ -11,7 +11,6 @@ export default { getStepsSharedData: 'get-steps-shared-data', setNextStep: 'set-next-step', setActionTargetWaiting: 'set-action-target-waiting', - setTestError: 'set-test-error', getAndUncheckFileDownloadingFlag: 'get-and-uncheck-file-downloading-flag', uncheckFileDownloadingFlag: 'uncheck-file-downloading-flag', nativeDialogsInfoSet: 'native-dialogs-info-set', diff --git a/src/runner/test-run/index.js b/src/runner/test-run/index.js index 7b376d13470..2a26e578464 100644 --- a/src/runner/test-run/index.js +++ b/src/runner/test-run/index.js @@ -4,6 +4,7 @@ import Mustache from 'mustache'; import { Session } from 'testcafe-hammerhead'; import COMMAND from './command'; import ERROR_TYPE from '../../reporters/errors/type'; +import ScreenshotCapturer from './screenshot-capturer'; // Const @@ -13,7 +14,9 @@ const IFRAME_TEST_RUN_TEMPLATE = read('../../client/test-run/iframe.js.mustache' export default class TestRun extends Session { constructor (test, browserConnection, opts) { - super(path.dirname(test.fixture.path)); + var uploadsRoot = path.dirname(test.fixture.path); + + super(uploadsRoot); this.running = false; this.unstable = false; @@ -26,7 +29,6 @@ export default class TestRun extends Session { // TODO remove it then we move shared data to session storage this.errs = []; - this.testError = null; this.restartCount = 0; this.nextStep = 0; this.actionTargetWaiting = 0; @@ -34,14 +36,27 @@ export default class TestRun extends Session { this.nativeDialogsInfoTimeStamp = 0; this.stepsSharedData = {}; + this.takeScreenshots = !!this.test.screenshotPath; + this.hasActionScreenshots = false; + + this.screenshotCapturer = this.takeScreenshots ? + new ScreenshotCapturer(this.test.fixture.path, this.test.screenshotPath, + this.browserConnection.userAgent) : + null; + + this.injectable.scripts.push('/testcafe-core.js'); this.injectable.scripts.push('/testcafe-ui.js'); this.injectable.scripts.push('/testcafe-runner.js'); this.injectable.styles.push('/testcafe-ui-styles.css'); } - async _loadUploads () { - //TODO fix it after UploadStorage rewrite + static _escapeUserAgent (userAgent) { + return userAgent + .toString() + .split('/') + .map(str => str.trim().replace(/\s/g, '_')) + .join('_'); } _getPayloadScript () { @@ -62,9 +77,9 @@ export default class TestRun extends Session { testSteps: this.test.stepData.js, sharedJs: sharedJs, nextStep: nextStep, - testError: this.testError ? JSON.stringify(this.testError) : 'null', browserHeartbeatUrl: this.browserConnection.heartbeatUrl, browserStatusUrl: this.browserConnection.statusUrl, + takeScreenshots: this.takeScreenshots, takeScreenshotOnFails: this.opts.takeScreenshotOnFails, skipJsErrors: this.opts.skipJsErrors, nativeDialogsInfo: JSON.stringify(this.nativeDialogsInfo), @@ -83,17 +98,20 @@ export default class TestRun extends Session { }); } - _addError (err) { + async _addError (err) { if (err.__sourceIndex !== void 0 && err.__sourceIndex !== null) { err.relatedSourceCode = this.test.sourceIndex[err.__sourceIndex]; delete err.__sourceIndex; } + if (err.screenshotRequired) + err.screenshotPath = await this.screenshotCapturer.capture(err.pageUrl, err.stepName, true); + this.errs.push(err); } - _fatalError (err) { - this._addError(err); + async _fatalError (err) { + await this._addError(err); this.emit('done'); } @@ -119,11 +137,11 @@ export default class TestRun extends Session { var ServiceMessages = TestRun.prototype; ServiceMessages[COMMAND.fatalError] = function (msg) { - this._fatalError(msg.err); + return this._fatalError(msg.err); }; ServiceMessages[COMMAND.assertionFailed] = function (msg) { - this._addError(msg.err); + return this._addError(msg.err); }; ServiceMessages[COMMAND.done] = function () { @@ -146,10 +164,6 @@ ServiceMessages[COMMAND.setActionTargetWaiting] = function (msg) { this.actionTargetWaiting = msg.value; }; -ServiceMessages[COMMAND.setTestError] = function (msg) { - this.testError = msg.err; -}; - ServiceMessages[COMMAND.getAndUncheckFileDownloadingFlag] = function () { var isFileDownloading = this.isFileDownloading; @@ -171,6 +185,8 @@ ServiceMessages[COMMAND.nativeDialogsInfoSet] = function (msg) { } }; -ServiceMessages[COMMAND.takeScreenshot] = function () { - //TODO: +ServiceMessages[COMMAND.takeScreenshot] = async function (msg) { + this.hasActionScreenshots = true; + + return await this.screenshotCapturer.capture(msg.pageUrl, msg.stepName, false, msg.filePath); }; diff --git a/src/runner/test-run/screenshot-capturer.js b/src/runner/test-run/screenshot-capturer.js new file mode 100644 index 00000000000..346befa250c --- /dev/null +++ b/src/runner/test-run/screenshot-capturer.js @@ -0,0 +1,45 @@ +import path from 'path'; +import promisify from 'es6-promisify'; +import mkdirp from 'mkdirp'; +import { screenshot } from 'testcafe-browser-natives'; + +var ensureDir = promisify(mkdirp); + + +export default class ScreenshotCapturer { + constructor (testFixturePath, baseScreenshotPath, userAgent) { + this.testFixtureDirPath = path.dirname(testFixturePath); + this.screenshotPath = path.join(baseScreenshotPath, ScreenshotCapturer._escapeUserAgent(userAgent)); + } + + static _escapeUserAgent (userAgent) { + return userAgent + .toString() + .split('/') + .map(str => str.trim().replace(/\s/g, '_')) + .join('_'); + } + + async capture (pageUrl, stepName, isError, customFilePath) { + await ensureDir(this.screenshotPath); + + var filePath = null; + var fileName = `${stepName && stepName.replace(/\s/g, '_') || 'Page_Load'}.png`; + + if (customFilePath) + filePath = path.join(this.testFixtureDirPath, customFilePath); + else if (isError) + filePath = path.join(this.screenshotPath, 'errors', fileName); + else + filePath = path.join(this.screenshotPath, fileName); + + //NOTE: the test should not fail if we can't make a screenshot for some reason + try { + await screenshot(pageUrl, filePath); + return filePath; + } + catch (err) { + return null; + } + } +} diff --git a/test/client/fixtures/runner/take-screenshots-test.js b/test/client/fixtures/runner/take-screenshots-test.js deleted file mode 100644 index 138bd2444e7..00000000000 --- a/test/client/fixtures/runner/take-screenshots-test.js +++ /dev/null @@ -1,248 +0,0 @@ -var testCafeCore = window.getTestCafeModule('testCafeCore'); -var transport = testCafeCore.get('./transport'); -var ERROR_TYPE = testCafeCore.ERROR_TYPE; -var SETTINGS = testCafeCore.get('./settings').get(); - -var testCafeRunner = window.getTestCafeModule('testCafeRunner'); -var Runner = testCafeRunner.get('./runner'); -var actionsAPI = testCafeRunner.get('./api/actions'); -var actionBarrier = testCafeRunner.get('./action-barrier/action-barrier'); - - -var runner = null, - lastError = null, - lastIsFailedStep = false, - screenShotRequestCount = false, - expectedError = null, - expectedScreenshotCount = 0; - -transport.batchUpdate = function (callback) { - callback(); -}; - -transport.fail = function (err) { - ok(err.code === expectedError); - ok(screenShotRequestCount === expectedScreenshotCount); - - runner._destroyIFrameBehavior(); - $('iframe').remove(); - start(); -}; -transport.asyncServiceMsg = function (msg, callback) { - if (msg.cmd === 'CMD_TAKE_SCREENSHOT') { //TODO: fix - screenShotRequestCount++; - ok(msg.isFailedStep); - } - - if (callback) - callback(); -}; -transport.assertionFailed = function () { -}; - -actionBarrier.waitPageInitialization = function (callback) { - callback(); -}; -$.fn.load = function (callback) { - callback(); -}; - -Runner.checkStatus = function () { -}; - -QUnit.testStart(function () { - runner = new Runner(); - screenShotRequestCount = 0; - expectedError = null; - expectedScreenshotCount = 0; - lastIsFailedStep = false; -}); - -asyncTest('Uncaught error in test script', function () { - var errorText = 'Test error', - stepNames = ['1.Step name'], - testSteps = [function () { - throw errorText; - }]; - - SETTINGS.TAKE_SCREENSHOT_ON_FAILS = true; - expectedError = ERROR_TYPE.uncaughtJSErrorInTestCodeStep; - expectedScreenshotCount = 1; - - runner.act._start(stepNames, testSteps, 0); -}); - -asyncTest('Invisible element', function () { - var stepNames = ['1.Step name'], - testSteps = [function () { - actionsAPI.click('body1'); - }]; - - SETTINGS.TAKE_SCREENSHOT_ON_FAILS = true; - expectedError = ERROR_TYPE.emptyFirstArgument; - expectedScreenshotCount = 1; - - runner.act._start(stepNames, testSteps, 0); -}); - -asyncTest('Failed assertion in step with action', function () { - var stepNames = ['1.Step name'], - eq = runner.eq, - ok = runner.ok, - testSteps = [function () { - ok(0); - eq(0, 1); - actionsAPI.wait('body1'); - }]; - - SETTINGS.TAKE_SCREENSHOT_ON_FAILS = true; - expectedError = ERROR_TYPE.incorrectWaitActionMillisecondsArgument; - expectedScreenshotCount = 1; - - runner.act._start(stepNames, testSteps, 0); -}); - -asyncTest('Failed assertion in step without action', function () { - var stepNames = ['1.Step name'], - eq = runner.eq, - ok = runner.ok, - testSteps = [ - function () { - ok(0); - eq(0, 1); - }, - function () { - actionsAPI.wait('#thowError'); - } - ]; - - SETTINGS.TAKE_SCREENSHOT_ON_FAILS = true; - expectedError = ERROR_TYPE.incorrectWaitActionMillisecondsArgument; - expectedScreenshotCount = 2; - - runner.act._start(stepNames, testSteps, 0); -}); - -asyncTest('Failed assertion and error: without "Take scr" flag', function () { - var stepNames = ['1.Step name'], - eq = runner.eq, - ok = runner.ok, - testSteps = [ - function () { - ok(0); - eq(0, 1); - }, - function () { - actionsAPI.wait('#thowError'); - } - ]; - - SETTINGS.TAKE_SCREENSHOT_ON_FAILS = false; - expectedError = ERROR_TYPE.incorrectWaitActionMillisecondsArgument; - expectedScreenshotCount = 0; - - runner.act._start(stepNames, testSteps, 0); -}); - -module('in IFrame'); - -asyncTest('Uncaught error in test script', function () { - var $iframe = $('