diff --git a/package.json b/package.json index f6eb61d11f2..d6af8811361 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,8 @@ "testcafe-browser-natives": "^0.10.0", "testcafe-hammerhead": "^0.2.1", "uglify-js": "1.2.6", - "useragent": "^2.1.7" + "useragent": "^2.1.7", + "uuid": "^2.0.1" }, "devDependencies": { "babel": "^5.8.23", diff --git a/src/client/core/transport.js b/src/client/core/transport.js index 665373333d8..98770c209fc 100644 --- a/src/client/core/transport.js +++ b/src/client/core/transport.js @@ -1,7 +1,10 @@ import hammerhead from './deps/hammerhead'; import COMMAND from '../../runner/test-run/command'; -var transport = hammerhead.transport; +var transport = hammerhead.transport; +var beforeUnloadRaised = false; + +hammerhead.on(hammerhead.EVENTS.beforeUnload, () => beforeUnloadRaised = true); //Exports @@ -12,27 +15,24 @@ export var waitForServiceMessagesCompleted = transport.waitForServiceMessagesCom export var batchUpdate = transport.batchUpdate.bind(transport); export var queuedAsyncServiceMsg = transport.queuedAsyncServiceMsg.bind(transport); -export function fail (err, callback) { +export function fatalError (err, callback) { + // NOTE: we should not stop the test run if an error occured during page unloading because we + // would destroy the session in this case and wouldn't be able to get the next page in the browser. + // We should set the deferred error to the task to have the test fail after the page reloading. var testFailMsg = { - cmd: COMMAND.fatalError, - err: err + cmd: COMMAND.fatalError, + err: err, + deferred: beforeUnloadRaised }; - transport.asyncServiceMsg(testFailMsg, function () { - callback(); - }); - - //HACK: this helps stop current JS context execution - window.onerror = function () { - }; - throw 'STOP'; + transport.asyncServiceMsg(testFailMsg, callback); } -export function assertionFailed (err) { +export function assertionFailed (err, callback) { var assertionFailedMsg = { cmd: COMMAND.assertionFailed, err: err }; - transport.asyncServiceMsg(assertionFailedMsg); + transport.asyncServiceMsg(assertionFailedMsg, callback); } diff --git a/src/client/runner/api/actions.js b/src/client/runner/api/actions.js index d5b83ed2c93..bf520181186 100644 --- a/src/client/runner/api/actions.js +++ b/src/client/runner/api/actions.js @@ -698,11 +698,11 @@ export function upload (what, path) { ); } -export function screenshot () { +export function screenshot (filePath) { stepIterator.asyncAction(function (iteratorCallback) { stepIterator.takeScreenshot(function () { iteratorCallback(); - }, false); + }, filePath); }); } diff --git a/src/client/runner/iframe-runner.js b/src/client/runner/iframe-runner.js index ae774f2ea44..a55314fa01e 100644 --- a/src/client/runner/iframe-runner.js +++ b/src/client/runner/iframe-runner.js @@ -59,8 +59,7 @@ IFrameRunner.prototype._onActionRun = function () { messageSandbox.sendServiceMsg({ cmd: RunnerBase.IFRAME_ACTION_RUN_CMD }, window.top); }; -IFrameRunner.prototype._onError = function (err) { - +IFrameRunner.prototype._onFatalError = function (err) { if (!SETTINGS.get().PLAYBACK || err.dialog) this.stepIterator.stop(); @@ -78,8 +77,6 @@ IFrameRunner.prototype._onAssertionFailed = function (e) { err: e }; - this.stepIterator.state.needScreeshot = true; - messageSandbox.sendServiceMsg(msg, window.top); if (SETTINGS.get().PLAYBACK) @@ -115,8 +112,9 @@ IFrameRunner.prototype._onGetStepsSharedData = function (e) { IFrameRunner.prototype._onTakeScreenshot = function (e) { var msg = { - cmd: RunnerBase.IFRAME_TAKE_SCREENSHOT_REQUEST_CMD, - isFailedStep: e.isFailedStep + cmd: RunnerBase.IFRAME_TAKE_SCREENSHOT_REQUEST_CMD, + stepName: e.stepName, + filePath: e.filePath }; messageSandbox.sendServiceMsg(msg, window.top); diff --git a/src/client/runner/runner-base.js b/src/client/runner/runner-base.js index f2279a96c65..b8d49414fc0 100644 --- a/src/client/runner/runner-base.js +++ b/src/client/runner/runner-base.js @@ -81,21 +81,22 @@ var RunnerBase = function () { if (err.inIFrame && !SETTINGS.get().PLAYBACK) runner.stepIterator.stop(); else if (!SETTINGS.get().SKIP_JS_ERRORS || SETTINGS.get().RECORDING) { - runner._onError({ + runner._onFatalError({ type: ERROR_TYPE.uncaughtJSError, scriptErr: err.msg, pageError: true, - pageUrl: err.pageUrl + pageUrl: err.pageUrl, + stepName: runner.stepIterator.getCurrentStep() }); } }); runner.stepIterator.on(StepIterator.ERROR_EVENT, function (e) { - runner._onError(e); + runner._onFatalError(e); }); runner.act._onJSError = function (err) { - runner._onError({ + runner._onFatalError({ type: ERROR_TYPE.uncaughtJSError, scriptErr: (err && err.message) || err }); @@ -105,6 +106,9 @@ var RunnerBase = function () { //NOTE: start test execution only when all content is loaded or if loading //timeout is reached (whichever comes first). runner._prepareStepsExecuting(function () { + if (runner.stopped) + return; + delete runner.act._onJSError; delete runner.act._start; @@ -203,7 +207,7 @@ RunnerBase.prototype._initIFrameBehavior = function () { message.err.stepName = runner.stepIterator.getCurrentStep(); } runner._clearIFrameExistenceWatcherInterval(); - runner._onError(message.err); + runner._onFatalError(message.err); break; case RunnerBase.IFRAME_FAILED_ASSERTION_CMD: @@ -211,7 +215,7 @@ RunnerBase.prototype._initIFrameBehavior = function () { runner.executingStepInIFrameWindow = null; message.err.stepNum = runner.stepIterator.state.step - 1; - runner._onAssertionFailed(message.err, true); + runner._onAssertionFailed(message.err); break; case RunnerBase.IFRAME_GET_SHARED_DATA_REQUEST_CMD: @@ -256,8 +260,9 @@ RunnerBase.prototype._initIFrameBehavior = function () { case RunnerBase.IFRAME_TAKE_SCREENSHOT_REQUEST_CMD: runner._onTakeScreenshot({ - isFailedStep: message.isFailedStep, - callback: function () { + stepName: message.stepName, + filePath: message.filePath, + callback: function () { msg = { cmd: RunnerBase.IFRAME_TAKE_SCREENSHOT_RESPONSE_CMD }; @@ -375,9 +380,9 @@ RunnerBase.prototype._runInIFrame = function (iframe, stepName, step, stepNum) { pingIFrame(iframe, function (err) { if (err) { - runner._onError({ + runner._onFatalError({ type: ERROR_TYPE.inIFrameTargetLoadingTimeout, - stepName: SETTINGS.get().CURRENT_TEST_STEP_NAME + stepName: runner.stepIterator.getCurrentStep() }); } else { @@ -389,9 +394,9 @@ RunnerBase.prototype._runInIFrame = function (iframe, stepName, step, stepNum) { RunnerBase.prototype._ensureIFrame = function (arg) { if (!arg) { - this._onError({ + this._onFatalError({ type: ERROR_TYPE.emptyIFrameArgument, - stepName: SETTINGS.get().CURRENT_TEST_STEP_NAME + stepName: this.stepIterator.getCurrentStep() }); return null; } @@ -400,9 +405,9 @@ RunnerBase.prototype._ensureIFrame = function (arg) { if (arg.tagName && arg.tagName.toLowerCase() === 'iframe') return arg; else { - this._onError({ + this._onFatalError({ type: ERROR_TYPE.iframeArgumentIsNotIFrame, - stepName: SETTINGS.get().CURRENT_TEST_STEP_NAME + stepName: this.stepIterator.getCurrentStep() }); return null; } @@ -413,16 +418,16 @@ RunnerBase.prototype._ensureIFrame = function (arg) { if (serviceUtils.isJQueryObj(arg)) { if (arg.length === 0) { - this._onError({ + this._onFatalError({ type: ERROR_TYPE.emptyIFrameArgument, - stepName: SETTINGS.get().CURRENT_TEST_STEP_NAME + stepName: this.stepIterator.getCurrentStep() }); return null; } else if (arg.length > 1) { - this._onError({ + this._onFatalError({ type: ERROR_TYPE.multipleIFrameArgument, - stepName: SETTINGS.get().CURRENT_TEST_STEP_NAME + stepName: this.stepIterator.getCurrentStep() }); return null; } @@ -433,10 +438,11 @@ RunnerBase.prototype._ensureIFrame = function (arg) { if (typeof arg === 'function') return this._ensureIFrame(arg()); - this._onError({ + this._onFatalError({ type: ERROR_TYPE.incorrectIFrameArgument, - stepName: SETTINGS.get().CURRENT_TEST_STEP_NAME + stepName: this.stepIterator.getCurrentStep() }); + return null; }; @@ -468,7 +474,7 @@ RunnerBase.prototype._initApi = function () { iFrame = runner._ensureIFrame(iFrameGetter()); if (iFrame) - runner._runInIFrame(iFrame, SETTINGS.get().CURRENT_TEST_STEP_NAME, step, stepNum); + runner._runInIFrame(iFrame, runner.stepIterator.getCurrentStep(), step, stepNum); }; }; }; @@ -526,7 +532,7 @@ RunnerBase.prototype._onActionRun = function () { this.eventEmitter.emit(this.ACTION_RUN_EVENT, {}); }; -RunnerBase.prototype._onError = function (err) { +RunnerBase.prototype._onFatalError = function (err) { this.eventEmitter.emit(this.TEST_FAILED_EVENT, { stepNum: this.stepIterator.state.step - 1, err: err diff --git a/src/client/runner/runner.js b/src/client/runner/runner.js index 568203a3fd8..6e1b22b78dc 100644 --- a/src/client/runner/runner.js +++ b/src/client/runner/runner.js @@ -1,24 +1,26 @@ +import { Promise } from 'es6-promise'; import hammerhead from './deps/hammerhead'; import testCafeCore from './deps/testcafe-core'; import RunnerBase from './runner-base'; import * as browser from '../browser'; -var browserUtils = hammerhead.utils.browser; var SETTINGS = testCafeCore.SETTINGS; var COMMAND = testCafeCore.COMMAND; -var ERROR_TYPE = testCafeCore.ERROR_TYPE; var transport = testCafeCore.transport; var serviceUtils = testCafeCore.serviceUtils; const WAITING_FOR_SERVICE_MESSAGES_COMPLETED_DELAY = 1000; +const APPLY_DOCUMENT_TITLE_TIMEOUT = 500; +const RESTORE_DOCUMENT_TITLE_TIMEOUT = 100; +const CHECK_TITLE_INTERVAL = 50; - -var Runner = function (startedCallback) { - var runner = this; - +var Runner = function (startedCallback, err) { RunnerBase.apply(this, [startedCallback]); + + if (err) + this._onFatalError(err); }; serviceUtils.inherit(Runner, RunnerBase); @@ -80,44 +82,84 @@ Runner.prototype._onActionRun = function () { transport.asyncServiceMsg(msg); }; -Runner.prototype._onError = function (err) { - var runner = this; +Runner.prototype._beforeScreenshot = function () { + this.stepIterator.suspend(); + this.eventEmitter.emit(RunnerBase.SCREENSHOT_CREATING_STARTED_EVENT, {}); + this.savedDocumentTitle = document.title; - if (this.stopped) - return; + var assignedTitle = `[ ${window.location.toString()} ]`; - //NOTE: we should stop stepIterator to prevent playback after an error is occurred - this.stepIterator.stop(); + // NOTE: we should keep the page url in document.title + // while the screenshot is being created + this.checkTitleIntervalId = window.setInterval(() => { + if (document.title !== assignedTitle) { + this.savedDocumentTitle = document.title; + document.title = assignedTitle; + } + }, CHECK_TITLE_INTERVAL); - RunnerBase.prototype._onError.call(this, err); + document.title = assignedTitle; - if (!SETTINGS.get().TAKE_SCREENSHOT_ON_FAILS) { - this.stopped = true; - transport.fail(err, Runner.checkStatus); - return; - } + return new Promise(resolve => window.setTimeout(resolve), APPLY_DOCUMENT_TITLE_TIMEOUT); +}; - var setErrorMsg = { - cmd: COMMAND.setTestError, - err: err - }; +Runner.prototype._afterScreenshot = function () { + window.clearInterval(this.checkTitleIntervalId); - transport.asyncServiceMsg(setErrorMsg); + document.title = this.savedDocumentTitle; + this.checkTitleIntervalId = null; + this.savedDocumentTitle = null; - this._onTakeScreenshot({ - isFailedStep: true, - //TODO: - //withoutStepName: !(ERRORS.hasErrorStepName(err) && ERRORS.hasErrorStepName(err)), - callback: function () { - runner.stopped = true; - transport.fail(err, Runner.checkStatus); - } + this.eventEmitter.emit(RunnerBase.SCREENSHOT_CREATING_FINISHED_EVENT, {}); + this.stepIterator.resume(); + + return new Promise(resolve => window.setTimeout(resolve), RESTORE_DOCUMENT_TITLE_TIMEOUT); +}; + +Runner.prototype._reportErrorToServer = function (err, isAssertion) { + return new Promise(resolve => { + if (isAssertion) + transport.assertionFailed(err, resolve); + else + transport.fatalError(err, resolve); }); }; -Runner.prototype._onAssertionFailed = function (e, inIFrame) { - this.stepIterator.state.needScreeshot = !inIFrame; - transport.assertionFailed(e.err); +Runner.prototype._onTestError = function (err, isAssertion) { + // NOTE: we should not create multiple screenshots for a step. Create a screenshot if + // it's the first error at this step or it's an error that occurs on page initialization. + err.pageUrl = document.location.toString(); + err.screenshotRequired = SETTINGS.get().TAKE_SCREENSHOTS && SETTINGS.get().TAKE_SCREENSHOTS_ON_FAILS && + this.stepIterator.state.curStepErrors.length < 2; + + var errorProcessingChain = Promise.resolve(); + + if (err.screenshotRequired) + errorProcessingChain = errorProcessingChain.then(() => this._beforeScreenshot()); + + errorProcessingChain = errorProcessingChain.then(() => this._reportErrorToServer(err, isAssertion)); + + if (err.screenshotRequired) + errorProcessingChain = errorProcessingChain.then(() => this._afterScreenshot()); + + return errorProcessingChain; +}; + +Runner.prototype._onFatalError = function (err) { + if (this.stopped) + return; + + this.stopped = true; + this.stepIterator.stop(); + + RunnerBase.prototype._onFatalError.call(this, err); + + this._onTestError(err) + .then(Runner.checkStatus); +}; + +Runner.prototype._onAssertionFailed = function (e) { + this._onTestError(e.err, true); }; Runner.prototype._onSetStepsSharedData = function (e) { @@ -138,62 +180,28 @@ Runner.prototype._onGetStepsSharedData = function (e) { }; Runner.prototype._onTakeScreenshot = function (e) { - var savedTitle = document.title, - windowMark = '[tc-' + Date.now() + ']', - browserName = null, - callback = e && e.callback ? e.callback : function () { - }, - runner = this; - - runner.eventEmitter.emit(RunnerBase.SCREENSHOT_CREATING_STARTED_EVENT, {}); - - - if (browserUtils.isMSEdge) - browserName = 'MSEDGE'; - else if (browserUtils.isSafari) - browserName = 'SAFARI'; - else if (browserUtils.isOpera || browserUtils.isOperaWithWebKit) - browserName = 'OPERA'; - else if (browserUtils.isWebKit) - browserName = 'CHROME'; - else if (browserUtils.isMozilla) - browserName = 'FIREFOX'; - else if (browserUtils.isIE) - browserName = 'IE'; - - var msg = { - cmd: 'CMD_TAKE_SCREENSHOT', //TODO: fix - windowMark: windowMark, - browserName: browserName, - isFailedStep: e.isFailedStep, - withoutStepName: e.withoutStepName, - url: window.location.toString() - }; - - var assignedTitle = savedTitle + windowMark, - checkTitleIntervalId = window.setInterval(function () { - if (document.title !== assignedTitle) { - savedTitle = document.title; - document.title = assignedTitle; - } - }, 50); - - document.title = assignedTitle; - - //NOTE: we should set timeouts to changing of document title - //in any case we are waiting response from server - window.setTimeout(function () { - transport.asyncServiceMsg(msg, function () { - window.clearInterval(checkTitleIntervalId); - checkTitleIntervalId = null; - document.title = savedTitle; - runner.eventEmitter.emit(RunnerBase.SCREENSHOT_CREATING_FINISHED_EVENT, {}); - - window.setTimeout(function () { - callback(); - }, 100); + if (!SETTINGS.get().TAKE_SCREENSHOTS) + return typeof e.callback === 'function' ? e.callback() : null; + + this + ._beforeScreenshot() + .then(() => { + return new Promise(resolve => { + var msg = { + cmd: COMMAND.takeScreenshot, + pageUrl: document.location.toString(), + stepName: e.stepName, + customPath: e.filePath + }; + + transport.asyncServiceMsg(msg, resolve); + }); + }) + .then(() => this._afterScreenshot()) + .then(() => { + if (typeof e.callback === 'function') + e.callback(); }); - }, 500); }; Runner.prototype._onDialogsInfoChanged = function (info) { diff --git a/src/client/runner/step-iterator.js b/src/client/runner/step-iterator.js index cb4fb0bfc50..53035064c5b 100644 --- a/src/client/runner/step-iterator.js +++ b/src/client/runner/step-iterator.js @@ -19,6 +19,12 @@ const STEP_DELAY = 500; const PROLONGED_STEP_DELAY = 3000; const SHORT_PROLONGED_STEP_DELAY = 30; +const SUSPEND_ACTIONS = { + runStep: 'runStep', + asyncAction: 'asyncAction', + asyncActionSeries: 'asyncActionSeries' +}; + //Iterator var StepIterator = function (pingIFrame) { @@ -34,8 +40,11 @@ var StepIterator = function (pingIFrame) { stepsSharedData: {}, lastSyncedSharedDataJSON: null, stopped: false, + suspended: false, + suspendedAction: null, + suspendedArgs: [], waitedIFrame: null, - needScreeshot: false + curStepErrors: [] }; this.pingIFrame = pingIFrame; @@ -81,7 +90,13 @@ StepIterator.prototype._checkSharedDataSerializable = function () { }; StepIterator.prototype._runStep = function () { - this.state.stopped = false; + if (this.state.suspended) { + this.state.suspendedAction = SUSPEND_ACTIONS.runStep; + return; + } + + this.state.stopped = false; + this.state.curStepErrors = []; var iterator = this; @@ -158,14 +173,7 @@ StepIterator.prototype._runStep = function () { if (!iterator._checkSharedDataSerializable()) return; - if (SETTINGS.get().TAKE_SCREENSHOT_ON_FAILS && iterator.state.needScreeshot) { - iterator.takeScreenshot(function () { - iterator.state.needScreeshot = false; - runCallback(); - }, true); - } - else - runCallback(); + runCallback(); } }); } @@ -313,103 +321,93 @@ StepIterator.prototype._checkIFrame = function (element, callback) { }; StepIterator.prototype.asyncAction = function (action) { + if (this.state.suspended) { + this.state.suspendedAction = SUSPEND_ACTIONS.asyncAction; + this.state.suspendedArgs = arguments; + return; + } + var iterator = this; this.state.inAsyncAction = true; - var actionRun = function () { - iterator._syncSharedDataWithServer(function () { - actionBarrier.waitActionSideEffectsCompletion(action, function () { - iterator._completeAsyncAction.apply(iterator, arguments); - }); + iterator._syncSharedDataWithServer(function () { + actionBarrier.waitActionSideEffectsCompletion(action, function () { + iterator._completeAsyncAction.apply(iterator, arguments); }); - }; - - if (SETTINGS.get().TAKE_SCREENSHOT_ON_FAILS && this.state.needScreeshot) { - this.takeScreenshot(function () { - iterator.state.needScreeshot = false; - actionRun(); - }, true); - } - else - actionRun(); + }); }; StepIterator.prototype.asyncActionSeries = function (items, runArgumentsIterator, action) { + if (this.state.suspended) { + this.state.suspendedAction = SUSPEND_ACTIONS.asyncActionSeries; + this.state.suspendedArgs = arguments; + return; + } + var iterator = this; - var actionsRun = function () { - var seriesActionsRun = function (elements, callback) { - async.forEachSeries( - elements, - function (element, asyncCallback) { - //NOTE: since v.14.1.5 it's recommended to run actions with the inIFrame function. But we should support old-style iframes - //using, so, it'll be resolved here. - iterator._checkIFrame(element, function (iframe) { - if (!iframe) { - actionBarrier.waitActionSideEffectsCompletion(function (barrierCallback) { - action(element, barrierCallback); - }, asyncCallback); - } - else { - var iFrameStartXhrBarrier = iframe.contentWindow[xhrBarrier.GLOBAL_START_XHR_BARRIER], - iFrameWaitXhrBarrier = iframe.contentWindow[xhrBarrier.GLOBAL_WAIT_XHR_BARRIER]; - - actionBarrier.waitActionSideEffectsCompletion(function (barrierCallback) { - var iFrameBeforeUnloadRaised = false; - - iterator.iFrameActionCallback = function () { - iterator.iFrameActionCallback = null; - iterator.waitedIFrame = null; - barrierCallback(); - }; - - iterator.waitedIFrame = iframe; - - iFrameStartXhrBarrier(function () { - if (!iFrameBeforeUnloadRaised) - iterator.iFrameActionCallback(); - }); - - function onBeforeUnload () { - nativeMethods.windowRemoveEventListener.call(iframe.contentWindow, 'beforeunload', onBeforeUnload); - iFrameBeforeUnloadRaised = true; - } - - nativeMethods.windowAddEventListener.call(iframe.contentWindow, 'beforeunload', onBeforeUnload, true); - - action(element, function () { - iFrameWaitXhrBarrier(); - }, iframe); - }, asyncCallback); - } - }); - }, - function () { - if (iterator.state.stopped) - return; + iterator.state.inAsyncAction = true; - callback(); + var seriesActionsRun = function (elements, callback) { + async.forEachSeries( + elements, + function (element, asyncCallback) { + // NOTE: since v.14.1.5 it's recommended to run actions using the inIFrame function. + // But we should support old-style iframes, so it'll be resolved here. + iterator._checkIFrame(element, function (iframe) { + if (!iframe) { + actionBarrier.waitActionSideEffectsCompletion(function (barrierCallback) { + action(element, barrierCallback); + }, asyncCallback); + } + else { + var iFrameStartXhrBarrier = iframe.contentWindow[xhrBarrier.GLOBAL_START_XHR_BARRIER], + iFrameWaitXhrBarrier = iframe.contentWindow[xhrBarrier.GLOBAL_WAIT_XHR_BARRIER]; + + actionBarrier.waitActionSideEffectsCompletion(function (barrierCallback) { + var iFrameBeforeUnloadRaised = false; + + iterator.iFrameActionCallback = function () { + iterator.iFrameActionCallback = null; + iterator.waitedIFrame = null; + barrierCallback(); + }; + + iterator.waitedIFrame = iframe; + + iFrameStartXhrBarrier(function () { + if (!iFrameBeforeUnloadRaised) + iterator.iFrameActionCallback(); + }); + + function onBeforeUnload () { + nativeMethods.windowRemoveEventListener.call(iframe.contentWindow, 'beforeunload', onBeforeUnload); + iFrameBeforeUnloadRaised = true; + } + + nativeMethods.windowAddEventListener.call(iframe.contentWindow, 'beforeunload', onBeforeUnload, true); + + action(element, function () { + iFrameWaitXhrBarrier(); + }, iframe); + }, asyncCallback); + } }); - }; + }, + function () { + if (iterator.state.stopped) + return; - iterator._syncSharedDataWithServer(function () { - runArgumentsIterator(items, seriesActionsRun, function () { - iterator._completeAsyncAction.apply(iterator, arguments); + callback(); }); - }); }; - iterator.state.inAsyncAction = true; - - if (SETTINGS.get().TAKE_SCREENSHOT_ON_FAILS && this.state.needScreeshot) { - this.takeScreenshot(function () { - iterator.state.needScreeshot = false; - actionsRun(); - }, true); - } - else - actionsRun(); + iterator._syncSharedDataWithServer(function () { + runArgumentsIterator(items, seriesActionsRun, function () { + iterator._completeAsyncAction.apply(iterator, arguments); + }); + }); }; StepIterator.prototype._init = function () { @@ -467,6 +465,8 @@ StepIterator.prototype.onError = function (err) { if (this.state.stopped) return; + this.state.curStepErrors.push(err); + this.eventEmitter.emit(StepIterator.ERROR_EVENT, $.extend({ stepNum: this.state.step - 1 }, err)); @@ -476,6 +476,8 @@ StepIterator.prototype.onAssertionFailed = function (err) { if (this.state.stopped) return; + this.state.curStepErrors.push(err); + this.eventEmitter.emit(StepIterator.ASSERTION_FAILED_EVENT, { err: err, stepNum: this.state.step - 1, @@ -508,6 +510,32 @@ StepIterator.prototype.onActionRun = function () { this.eventEmitter.emit(StepIterator.ACTION_RUN_EVENT, {}); }; +StepIterator.prototype.suspend = function () { + // NOTE: in fact we can suspend the iterator before + // the next step or an async action run + this.state.suspended = true; +}; + +StepIterator.prototype.resume = function () { + if (!this.state.suspended || this.state.stopped) + return; + + this.state.suspended = false; + + if (this.state.suspendedAction === SUSPEND_ACTIONS.runStep) + this._runStep(); + + if (this.state.suspendedAction === SUSPEND_ACTIONS.asyncAction) + this.asyncAction.apply(this, this.state.suspendedArgs); + + if (this.state.suspendedAction === SUSPEND_ACTIONS.asyncActionSeries) + this.asyncActionSeries.apply(this, this.state.suspendedArgs); + + this.state.suspendedAction = null; + this.state.suspendedArgs = []; +}; + + //Global __waitFor() StepIterator.prototype.setGlobalWaitFor = function (event, timeout) { this.globalWaitForEvent = event; @@ -540,10 +568,11 @@ StepIterator.prototype.__waitFor = function (callback) { }); }; -StepIterator.prototype.takeScreenshot = function (callback, isFailedStep) { +StepIterator.prototype.takeScreenshot = function (callback, filePath) { this.eventEmitter.emit(StepIterator.TAKE_SCREENSHOT_EVENT, { - isFailedStep: isFailedStep, - callback: callback + stepName: this.getCurrentStep(), + filePath: filePath || '', + callback: callback }); }; diff --git a/src/client/test-run/iframe.js.mustache b/src/client/test-run/iframe.js.mustache index c4f1bdf65b8..2c56ba4d91d 100644 --- a/src/client/test-run/iframe.js.mustache +++ b/src/client/test-run/iframe.js.mustache @@ -70,7 +70,7 @@ testCafeCore.SETTINGS.set({ - TAKE_SCREENSHOT_ON_FAILS: {{{takeScreenshotOnFails}}}, + TAKE_SCREENSHOTS_ON_FAILS: {{{takeScreenshotsOnFails}}}, SKIP_JS_ERRORS: {{{skipJsErrors}}}, ENABLE_SOURCE_INDEX: true, NATIVE_DIALOGS_INFO: {{{nativeDialogsInfo}}} diff --git a/src/client/test-run/index.js.mustache b/src/client/test-run/index.js.mustache index 9d49fa2c0d1..4d058e70edc 100644 --- a/src/client/test-run/index.js.mustache +++ b/src/client/test-run/index.js.mustache @@ -39,9 +39,10 @@ var stepNames = {{{stepNames}}}; testCafeCore.SETTINGS.set({ - CURRENT_TEST_STEP_NAME: nextStep ? stepNames[nextStep - 1] : 'Test initialization', + CURRENT_TEST_STEP_NAME: nextStep ? stepNames[nextStep - 1] : 'Page Load', BROWSER_STATUS_URL: '{{{browserStatusUrl}}}', - TAKE_SCREENSHOT_ON_FAILS: {{{takeScreenshotOnFails}}}, + TAKE_SCREENSHOTS: {{{takeScreenshots}}}, + TAKE_SCREENSHOTS_ON_FAILS: {{{takeScreenshotsOnFails}}}, SKIP_JS_ERRORS: {{{skipJsErrors}}}, ENABLE_SOURCE_INDEX: true, NATIVE_DIALOGS_INFO: {{{nativeDialogsInfo}}} @@ -70,20 +71,17 @@ sandboxedJQuery.init(window, undefined); jQuerySelectorExtensions.init(); - var testError = {{{testError}}}; - - if (testError) { - transport.fail(testError, Runner.checkStatus); - return; - } - $ = jQuery = jQuerySelectorExtensions.create(sandboxedJQuery.jQuery); jQueryDataMethodProxy.setup($); //NOTE: initialize API - var runner = new Runner(); + var deferredError = {{{deferredError}}}; + var runner = new Runner(null, deferredError); var progressPanel = null; + if (deferredError) + return; + runner.on(RunnerBase.SCREENSHOT_CREATING_STARTED_EVENT, function () { shadowUI.setBlind(true); }); diff --git a/src/reporters/base.js b/src/reporters/base.js index 33d15066512..0408982efb9 100644 --- a/src/reporters/base.js +++ b/src/reporters/base.js @@ -44,13 +44,14 @@ export default class BaseReporter { static _createReportItem (test, runsPerTest) { return { - fixtureName: test.fixture.name, - fixturePath: test.fixture.path, - testName: test.name, - pendingRuns: runsPerTest, - errs: [], - unstable: false, - startTime: null + fixtureName: test.fixture.name, + fixturePath: test.fixture.path, + testName: test.name, + screenshotPath: null, + pendingRuns: runsPerTest, + errs: [], + unstable: false, + startTime: null }; } @@ -88,7 +89,7 @@ export default class BaseReporter { BaseReporter._errorSorter(reportItem.errs); - this._reportTestDone(reportItem.testName, reportItem.errs, durationMs, reportItem.unstable); + this._reportTestDone(reportItem.testName, reportItem.errs, durationMs, reportItem.unstable, reportItem.screenshotPath); // NOTE: here we assume that tests are sorted by fixture. // Therefore, if the next report item has a different @@ -126,8 +127,12 @@ export default class BaseReporter { reportItem.errs = reportItem.errs.concat(testRun.errs); reportItem.unstable = reportItem.unstable || testRun.unstable; - if (!reportItem.pendingRuns) + if (!reportItem.pendingRuns) { + if (task.screenshots.hasCapturedFor(testRun.test)) + reportItem.screenshotPath = task.screenshots.getPathFor(testRun.test); + this._shiftReportQueue(); + } }); task.once('done', () => { @@ -185,7 +190,7 @@ export default class BaseReporter { throw new Error('Not implemented'); } - _reportTestDone (name, errs, durationMs, unstable) { + _reportTestDone (name, errs, durationMs, unstable, screenshotPath) { throw new Error('Not implemented'); } diff --git a/src/reporters/errors/decorators/colored-text.js b/src/reporters/errors/decorators/colored-text.js index d745c0038d0..b7879ba4cf4 100644 --- a/src/reporters/errors/decorators/colored-text.js +++ b/src/reporters/errors/decorators/colored-text.js @@ -10,6 +10,10 @@ export default { 'span user-agent': str => chalk.gray(str), + 'div screenshot-info': str => str, + + 'a screenshot-path': str => chalk.underline(str), + 'code': str => chalk.yellow(str), 'code step-source': str => chalk.magenta(indentString(str, ' ', CODE_ALIGN_SPACES)), diff --git a/src/reporters/errors/decorators/plain-text.js b/src/reporters/errors/decorators/plain-text.js index f68248b5634..438799d71e5 100644 --- a/src/reporters/errors/decorators/plain-text.js +++ b/src/reporters/errors/decorators/plain-text.js @@ -9,6 +9,10 @@ export default { 'span user-agent': str => str, + 'div screenshot-info': str => str, + + 'a screenshot-path': str => str, + 'code': str => str, 'code step-source': str => indentString(str, ' ', CODE_ALIGN_SPACES), diff --git a/src/reporters/errors/templates.js b/src/reporters/errors/templates.js index 86e15ac9c24..fe423f28df8 100644 --- a/src/reporters/errors/templates.js +++ b/src/reporters/errors/templates.js @@ -42,6 +42,13 @@ function getDiffHeader (err) { return ''; } +function getScreenshotInfoStr (err) { + if (err.screenshotPath) + return `
Screenshot: ${err.screenshotPath}
`; + + return ''; +} + export default { [TYPE.okAssertion]: err => dedent` ${getAssertionMsgPrefix(err)} failed at step ${err.stepName}: @@ -50,6 +57,8 @@ export default { Expected: not null, not undefined, not false, not NaN and not '' Actual: ${escapeNewLines(err.actual)} + + ${getScreenshotInfoStr(err)} `, [TYPE.notOkAssertion]: err => dedent` @@ -59,6 +68,8 @@ export default { Expected: null, undefined, false, NaN or '' Actual: ${escapeNewLines(err.actual)} + + ${getScreenshotInfoStr(err)} `, [TYPE.eqAssertion]: (err, maxStringLength) => { @@ -75,6 +86,8 @@ export default { Expected: ${escapeNewLines(diff.expected)} Actual: ${escapeNewLines(diff.actual)} ${diffMarkerStr} + + ${getScreenshotInfoStr(err)} `; }, @@ -85,37 +98,51 @@ export default { Expected: not ${escapeNewLines(err.actual)} Actual: ${escapeNewLines(err.actual)} + + ${getScreenshotInfoStr(err)} `, [TYPE.iframeLoadingTimeout]: err => dedent` ${getMsgPrefix(err, CATEGORY.timeout)}IFrame loading timed out. + + ${getScreenshotInfoStr(err)} `, [TYPE.inIFrameTargetLoadingTimeout]: err => dedent` ${getMsgPrefix(err, CATEGORY.timeout)}Error at step ${err.stepName}: IFrame target loading timed out. + + ${getScreenshotInfoStr(err)} `, [TYPE.uncaughtJSError]: err => { if (err.pageUrl) { return dedent` ${getMsgPrefix(err, CATEGORY.unhandledException)}Uncaught JavaScript error ${err.scriptErr} on page ${err.pageUrl} + + ${getScreenshotInfoStr(err)} `; } return dedent` ${getMsgPrefix(err, CATEGORY.unhandledException)}Uncaught JavaScript error ${err.scriptErr} on page. + + ${getScreenshotInfoStr(err)} `; }, [TYPE.uncaughtJSErrorInTestCodeStep]: err => dedent` ${getMsgPrefix(err, CATEGORY.unhandledException)}Error at step ${err.stepName}: Uncaught JavaScript error in test code - ${err.scriptErr}. + + ${getScreenshotInfoStr(err)} `, [TYPE.storeDomNodeOrJqueryObject]: err => dedent` ${getMsgPrefix(err, CATEGORY.unhandledException)}Error at step ${err.stepName}: It is not allowed to share the DOM element, jQuery object or a function between test steps via "this" object. + + ${getScreenshotInfoStr(err)} `, [TYPE.emptyFirstArgument]: err => dedent` @@ -125,6 +152,8 @@ export default { A target element of the ${err.action} action has not been found in the DOM tree. If this element should be created after animation or a time-consuming operation is finished, use the waitFor action (available for use in code) to pause test execution until this element appears. + + ${getScreenshotInfoStr(err)} `, [TYPE.invisibleActionElement]: err => dedent` @@ -134,6 +163,8 @@ export default { A target element ${err.element} of the ${err.action} action is not visible. If this element should appear when you are hovering over another element, make sure that you properly recorded the hover action. + + ${getScreenshotInfoStr(err)} `, [TYPE.incorrectDraggingSecondArgument]: err => dedent` @@ -142,6 +173,8 @@ export default { ${getStepCode(err.relatedSourceCode)} drag action drop target is incorrect. + + ${getScreenshotInfoStr(err)} `, [TYPE.incorrectPressActionArgument]: err => dedent` @@ -150,6 +183,8 @@ export default { ${getStepCode(err.relatedSourceCode)} press action parameter contains incorrect key code. + + ${getScreenshotInfoStr(err)} `, [TYPE.emptyTypeActionArgument]: err => dedent` @@ -158,16 +193,22 @@ export default { ${getStepCode(err.relatedSourceCode)} The type action's parameter text is empty. + + ${getScreenshotInfoStr(err)} `, [TYPE.unexpectedDialog]: err => dedent` ${getMsgPrefix(err, CATEGORY.nativeDialogError)}Error at step ${err.stepName}: Unexpected system ${err.dialog} dialog ${err.message} appeared. + + ${getScreenshotInfoStr(err)} `, [TYPE.expectedDialogDoesntAppear]: err => dedent` ${getMsgPrefix(err, CATEGORY.nativeDialogError)}Error at step ${err.stepName}: The expected system ${err.dialog} dialog did not appear. + + ${getScreenshotInfoStr(err)} `, [TYPE.incorrectSelectActionArguments]: err => dedent` @@ -176,6 +217,8 @@ export default { ${getStepCode(err.relatedSourceCode)} select action's parameters contain an incorrect value. + + ${getScreenshotInfoStr(err)} `, [TYPE.incorrectWaitActionMillisecondsArgument]: err => dedent` @@ -184,6 +227,8 @@ export default { ${getStepCode(err.relatedSourceCode)} wait action's "milliseconds" parameter should be a positive number. + + ${getScreenshotInfoStr(err)} `, [TYPE.incorrectWaitForActionEventArgument]: err => dedent` @@ -192,6 +237,8 @@ export default { ${getStepCode(err.relatedSourceCode)} waitFor action's first parameter should be a function, a CSS selector or an array of CSS selectors. + + ${getScreenshotInfoStr(err)} `, [TYPE.incorrectWaitForActionTimeoutArgument]: err => dedent` @@ -200,6 +247,8 @@ export default { ${getStepCode(err.relatedSourceCode)} waitFor action's "timeout" parameter should be a positive number. + + ${getScreenshotInfoStr(err)} `, [TYPE.waitForActionTimeoutExceeded]: err => dedent` @@ -208,6 +257,8 @@ export default { ${getStepCode(err.relatedSourceCode)} waitFor action's timeout exceeded. + + ${getScreenshotInfoStr(err)} `, [TYPE.emptyIFrameArgument]: err => dedent` @@ -216,6 +267,8 @@ export default { ${getStepCode(err.relatedSourceCode)} The selector within the inIFrame function returns an empty value. + + ${getScreenshotInfoStr(err)} `, [TYPE.iframeArgumentIsNotIFrame]: err => dedent` @@ -224,6 +277,8 @@ export default { ${getStepCode(err.relatedSourceCode)} The selector within the inIFrame function doesn’t return an iframe element. + + ${getScreenshotInfoStr(err)} `, [TYPE.multipleIFrameArgument]: err => dedent` @@ -232,6 +287,8 @@ export default { ${getStepCode(err.relatedSourceCode)} The selector within the inIFrame function returns more than one iframe element. + + ${getScreenshotInfoStr(err)} `, [TYPE.incorrectIFrameArgument]: err => dedent` @@ -240,6 +297,8 @@ export default { ${getStepCode(err.relatedSourceCode)} The inIFrame function contains an invalid argument. + + ${getScreenshotInfoStr(err)} `, [TYPE.uploadCanNotFindFileToUpload]: err => { @@ -251,7 +310,9 @@ export default { Cannot find the following file(s) to upload: `; - return msg + err.filePaths.map(path => `\n ${path}`).join(','); + return msg + + err.filePaths.map(path => `\n ${path}`).join(',') + + (err.screenshotPath ? `\n\n${getScreenshotInfoStr(err)}` : ''); }, [TYPE.uploadElementIsNotFileInput]: err => dedent` @@ -260,6 +321,8 @@ export default { ${getStepCode(err.relatedSourceCode)} upload action argument does not contain a file input element. + + ${getScreenshotInfoStr(err)} `, [TYPE.uploadInvalidFilePathArgument]: err => dedent` @@ -268,6 +331,8 @@ export default { ${getStepCode(err.relatedSourceCode)} upload action's "path" parameter should be a string or an array of strings. + + ${getScreenshotInfoStr(err)} `, [TYPE.pageNotLoaded]: err => dedent` diff --git a/src/reporters/json.js b/src/reporters/json.js index 0ef8b6d2f70..7f72c1d718f 100644 --- a/src/reporters/json.js +++ b/src/reporters/json.js @@ -27,10 +27,10 @@ export default class JSONReporter extends BaseReporter { this.report.fixtures.push(this.currentFixture); } - _reportTestDone (name, errs, durationMs, unstable) { + _reportTestDone (name, errs, durationMs, unstable, screenshotPath) { errs = errs.map(err => this._formatError(err)); - this.currentFixture.tests.push({ name, errs, durationMs, unstable }); + this.currentFixture.tests.push({ name, errs, durationMs, unstable, screenshotPath }); } _reportTaskDone (passed, total, endTime) { diff --git a/src/reporters/list.js b/src/reporters/list.js index d01acf77612..1ff7f81c144 100644 --- a/src/reporters/list.js +++ b/src/reporters/list.js @@ -17,7 +17,7 @@ export default class ListReporter extends SpecReporter { this.currentFixtureName = name; } - _reportTestDone (name, errs, durationMs, unstable) { + _reportTestDone (name, errs, durationMs, unstable, screenshotPath) { var hasErr = !!errs.length; var nameStyle = hasErr ? this.chalk.red : this.chalk.gray; var symbol = hasErr ? this.chalk.red(this.symbols.err) : this.chalk.green(this.symbols.ok); @@ -31,6 +31,9 @@ export default class ListReporter extends SpecReporter { if (unstable) title += this.chalk.yellow(' (unstable)'); + if (screenshotPath) + title += ` (screenshots: ${this.chalk.underline(screenshotPath)})`; + this._write(title); if (hasErr) { diff --git a/src/reporters/spec.js b/src/reporters/spec.js index b191f1af4a9..4866e887458 100644 --- a/src/reporters/spec.js +++ b/src/reporters/spec.js @@ -38,7 +38,7 @@ export default class SpecReporter extends BaseReporter { ._newline(); } - _reportTestDone (name, errs, durationMs, unstable) { + _reportTestDone (name, errs, durationMs, unstable, screenshotPath) { var hasErr = !!errs.length; var nameStyle = hasErr ? this.chalk.red : this.chalk.gray; var symbol = hasErr ? this.chalk.red(this.symbols.err) : this.chalk.green(this.symbols.ok); @@ -49,6 +49,9 @@ export default class SpecReporter extends BaseReporter { if (unstable) title += this.chalk.yellow(' (unstable)'); + if (screenshotPath) + title += ` (screenshots: ${this.chalk.underline(screenshotPath)})`; + this._write(title); if (hasErr) { diff --git a/src/reporters/xunit.js b/src/reporters/xunit.js index 489338d87a9..b0e749bcab9 100644 --- a/src/reporters/xunit.js +++ b/src/reporters/xunit.js @@ -25,12 +25,15 @@ export default class XUnitReporter extends BaseReporter { this.currentFixtureName = escapeHtml(name); } - _reportTestDone (name, errs, durationMs, unstable) { + _reportTestDone (name, errs, durationMs, unstable, screenshotPath) { var hasErr = !!errs.length; if (unstable) name += ' (unstable)'; + if (screenshotPath) + name += ` (screenshots: ${screenshotPath})`; + name = escapeHtml(name); var openTag = ` this._createTestRun(test)); + this.testRunQueue = tests.map(test => this._createTestRun(test, screenshots)); } _shouldStartQuarantine (testRun) { @@ -80,11 +80,12 @@ export default class BrowserJob extends EventEmitter { this.emit('done'); } - _createTestRun (test) { - var testRun = new TestRun(test, this.browserConnection, this.opts); - var done = this.opts.quarantineMode ? - () => this._testRunDoneInQuarantineMode(testRun) : - () => this._testRunDone(testRun); + _createTestRun (test, screenshots) { + var screenshotCapturer = screenshots.createCapturerFor(test, this.browserConnection.userAgent); + var testRun = new TestRun(test, this.browserConnection, screenshotCapturer, this.opts); + var done = this.opts.quarantineMode ? + () => this._testRunDoneInQuarantineMode(testRun) : + () => this._testRunDone(testRun); testRun.once('done', done); diff --git a/src/runner/index.js b/src/runner/index.js index 66c5b87b19f..0698182a4e5 100644 --- a/src/runner/index.js +++ b/src/runner/index.js @@ -15,12 +15,12 @@ export default class Runner extends EventEmitter { this.bootstrapper = new Bootstrapper(browserConnectionGateway); this.opts = { - screenshotPath: null, - takeScreenshotOnFails: false, - skipJsErrors: false, - quarantineMode: false, - reportOutStream: void 0, - errorDecorator: void 0 + screenshotPath: null, + takeScreenshotsOnFails: false, + skipJsErrors: false, + quarantineMode: false, + reportOutStream: void 0, + errorDecorator: void 0 }; } @@ -108,8 +108,8 @@ export default class Runner extends EventEmitter { } screenshots (path, takeOnFails = false) { - this.opts.takeScreenshotOnFails = takeOnFails; - this.bootstrapper.screenshotPath = path; + this.opts.takeScreenshotsOnFails = takeOnFails; + this.opts.screenshotPath = path; return this; } diff --git a/src/runner/screenshots/capturer.js b/src/runner/screenshots/capturer.js new file mode 100644 index 00000000000..9e0b675c8d8 --- /dev/null +++ b/src/runner/screenshots/capturer.js @@ -0,0 +1,48 @@ +import { join as joinPath, dirname } from 'path'; +import promisify from 'es6-promisify'; +import mkdirp from 'mkdirp'; +import { screenshot as takeScreenshot } from 'testcafe-browser-natives'; + +var ensureDir = promisify(mkdirp); + + +export default class Capturer { + constructor (screenshotPath, testDirPath, testEntry) { + this.enabled = !!screenshotPath; + this.path = screenshotPath; + this.testDirPath = testDirPath; + this.testEntry = testEntry; + } + + static _getFileName (stepName) { + return `${stepName && stepName.replace(/\s|\\|\/|"|\*|\?|<|>|\|/g, '_') || 'Page_Load'}.png`; + } + + async _takeScreenshot (url, filePath) { + await ensureDir(dirname(filePath)); + await takeScreenshot(url, filePath); + + this.testEntry.hasScreenshots = true; + + return filePath; + } + + async captureAction ({ pageUrl, stepName, customPath }) { + var fileName = Capturer._getFileName(stepName); + var filePath = customPath ? + joinPath(this.testDirPath, customPath, fileName) : + joinPath(this.path, fileName); + + return await this._takeScreenshot(pageUrl, filePath); + } + + async captureError ({ pageUrl, stepName, screenshotRequired }) { + if (!screenshotRequired) + return null; + + var filePath = joinPath(this.path, 'errors', Capturer._getFileName(stepName)); + + return await this._takeScreenshot(pageUrl, filePath); + } +} + diff --git a/src/runner/screenshots/index.js b/src/runner/screenshots/index.js new file mode 100644 index 00000000000..a07dc624072 --- /dev/null +++ b/src/runner/screenshots/index.js @@ -0,0 +1,55 @@ +import { join as joinPath, dirname } from 'path'; +import uuid from 'uuid'; +import find from 'array-find'; +import Capturer from './capturer'; + +export default class Screenshots { + constructor (path) { + this.enabled = !!path; + this.path = path; + this.testEntries = []; + } + + static _escapeUserAgent (userAgent) { + return userAgent + .toString() + .split('/') + .map(str => str.trim().replace(/\s/g, '_')) + .join('_'); + } + + _addTestEntry (test) { + var testEntry = { + test: test, + path: this.path ? joinPath(this.path, uuid.v4().substr(0, 8)) : '', + hasScreenshots: false + }; + + this.testEntries.push(testEntry); + + return testEntry; + } + + _getTestEntry (test) { + return find(this.testEntries, entry => entry.test === test); + } + + hasCapturedFor (test) { + return this._getTestEntry(test).hasScreenshots; + } + + getPathFor (test) { + return this._getTestEntry(test).path; + } + + createCapturerFor (test, userAgent) { + var testEntry = this._getTestEntry(test); + + if (!testEntry) + testEntry = this._addTestEntry(test); + + var screenshotPath = this.enabled ? joinPath(testEntry.path, Screenshots._escapeUserAgent(userAgent)) : null; + + return new Capturer(screenshotPath, dirname(test.fixture.path), testEntry); + } +} diff --git a/src/runner/task.js b/src/runner/task.js index d32f6bd22d4..8e3805e8c7e 100644 --- a/src/runner/task.js +++ b/src/runner/task.js @@ -1,5 +1,6 @@ import { EventEmitter } from 'events'; import BrowserJob from './browser-job'; +import Screenshots from './screenshots'; import remove from '../utils/array-remove'; @@ -10,8 +11,9 @@ export default class Task extends EventEmitter { this.running = false; this.browserConnections = browserConnections; this.tests = tests; + this.screenshots = new Screenshots(opts.screenshotPath); - this.pendingBrowserJobs = this._createBrowserJobs(tests, proxy, opts); + this.pendingBrowserJobs = this._createBrowserJobs(tests, proxy, this.screenshots, opts); } _assignBrowserJobEventHandlers (job) { @@ -34,9 +36,9 @@ export default class Task extends EventEmitter { }); } - _createBrowserJobs (tests, proxy, opts) { + _createBrowserJobs (tests, proxy, screenshots, opts) { return this.browserConnections.map(bc => { - var job = new BrowserJob(tests, bc, proxy, opts); + var job = new BrowserJob(tests, bc, proxy, screenshots, opts); this._assignBrowserJobEventHandlers(job); bc.addJob(job); diff --git a/src/runner/test-run/command.js b/src/runner/test-run/command.js index 2a796aeee85..cc3f2a4f21e 100644 --- a/src/runner/test-run/command.js +++ b/src/runner/test-run/command.js @@ -11,7 +11,6 @@ export default { getStepsSharedData: 'get-steps-shared-data', setNextStep: 'set-next-step', setActionTargetWaiting: 'set-action-target-waiting', - setTestError: 'set-test-error', getAndUncheckFileDownloadingFlag: 'get-and-uncheck-file-downloading-flag', uncheckFileDownloadingFlag: 'uncheck-file-downloading-flag', nativeDialogsInfoSet: 'native-dialogs-info-set', diff --git a/src/runner/test-run/index.js b/src/runner/test-run/index.js index c6bd2fb886f..a46aada7424 100644 --- a/src/runner/test-run/index.js +++ b/src/runner/test-run/index.js @@ -12,8 +12,10 @@ const IFRAME_TEST_RUN_TEMPLATE = read('../../client/test-run/iframe.js.mustache' export default class TestRun extends Session { - constructor (test, browserConnection, opts) { - super(path.dirname(test.fixture.path)); + constructor (test, browserConnection, screenshotCapturer, opts) { + var uploadsRoot = path.dirname(test.fixture.path); + + super(uploadsRoot); this.running = false; this.unstable = false; @@ -26,13 +28,14 @@ export default class TestRun extends Session { // TODO remove it then we move shared data to session storage this.errs = []; - this.testError = null; + this.deferredError = null; this.restartCount = 0; this.nextStep = 0; this.actionTargetWaiting = 0; this.nativeDialogsInfo = null; this.nativeDialogsInfoTimeStamp = 0; this.stepsSharedData = {}; + this.screenshotCapturer = screenshotCapturer; this.injectable.scripts.push('/testcafe-core.js'); this.injectable.scripts.push('/testcafe-ui.js'); @@ -40,10 +43,6 @@ export default class TestRun extends Session { this.injectable.styles.push('/testcafe-ui-styles.css'); } - async _loadUploads () { - //TODO fix it after UploadStorage rewrite - } - _getPayloadScript () { var sharedJs = this.test.fixture.getSharedJs(); @@ -58,17 +57,18 @@ export default class TestRun extends Session { } return Mustache.render(TEST_RUN_TEMPLATE, { - stepNames: JSON.stringify(this.test.stepData.names), - testSteps: this.test.stepData.js, - sharedJs: sharedJs, - nextStep: nextStep, - testError: this.testError ? JSON.stringify(this.testError) : 'null', - browserHeartbeatUrl: this.browserConnection.heartbeatUrl, - browserStatusUrl: this.browserConnection.statusUrl, - takeScreenshotOnFails: this.opts.takeScreenshotOnFails, - skipJsErrors: this.opts.skipJsErrors, - nativeDialogsInfo: JSON.stringify(this.nativeDialogsInfo), - iFrameTestRunScript: JSON.stringify(this._getIFramePayloadScript()) + stepNames: JSON.stringify(this.test.stepData.names), + testSteps: this.test.stepData.js, + sharedJs: sharedJs, + nextStep: nextStep, + deferredError: this.deferredError ? JSON.stringify(this.deferredError) : 'null', + browserHeartbeatUrl: this.browserConnection.heartbeatUrl, + browserStatusUrl: this.browserConnection.statusUrl, + takeScreenshots: this.screenshotCapturer.enabled, + takeScreenshotsOnFails: this.opts.takeScreenshotsOnFails, + skipJsErrors: this.opts.skipJsErrors, + nativeDialogsInfo: JSON.stringify(this.nativeDialogsInfo), + iFrameTestRunScript: JSON.stringify(this._getIFramePayloadScript()) }); } @@ -76,24 +76,39 @@ export default class TestRun extends Session { var sharedJs = this.test.fixture.getSharedJs(); return Mustache.render(IFRAME_TEST_RUN_TEMPLATE, { - sharedJs: sharedJs, - takeScreenshotOnFails: this.opts.takeScreenshotOnFails, - skipJsErrors: this.opts.skipJsErrors, - nativeDialogsInfo: JSON.stringify(this.nativeDialogsInfo) + sharedJs: sharedJs, + takeScreenshotsOnFails: this.opts.takeScreenshotsOnFails, + skipJsErrors: this.opts.skipJsErrors, + nativeDialogsInfo: JSON.stringify(this.nativeDialogsInfo) }); } - _addError (err) { + async _addError (err) { if (err.__sourceIndex !== void 0 && err.__sourceIndex !== null) { err.relatedSourceCode = this.test.sourceIndex[err.__sourceIndex]; delete err.__sourceIndex; } + try { + err.screenshotPath = await this.screenshotCapturer.captureError(err); + } + catch (e) { + // NOTE: swallow the error silently if we can't take screenshots for some + // reason (e.g. we don't have permissions to write a screenshot file). + } + this.errs.push(err); } - _fatalError (err) { - this._addError(err); + async _fatalError (err, deferred) { + // TODO: move this logic to the client when localStorageSandbox will be implemented in the + // testcafe-hammerhead repo https://github.com/DevExpress/testcafe-hammerhead/issues/252 !!! + if (deferred) { + this.deferredError = err; + return; + } + + await this._addError(err); this.emit('done'); } @@ -119,11 +134,11 @@ export default class TestRun extends Session { var ServiceMessages = TestRun.prototype; ServiceMessages[COMMAND.fatalError] = function (msg) { - this._fatalError(msg.err); + return this._fatalError(msg.err, msg.deferred); }; ServiceMessages[COMMAND.assertionFailed] = function (msg) { - this._addError(msg.err); + return this._addError(msg.err); }; ServiceMessages[COMMAND.done] = function () { @@ -146,10 +161,6 @@ ServiceMessages[COMMAND.setActionTargetWaiting] = function (msg) { this.actionTargetWaiting = msg.value; }; -ServiceMessages[COMMAND.setTestError] = function (msg) { - this.testError = msg.err; -}; - ServiceMessages[COMMAND.getAndUncheckFileDownloadingFlag] = function () { var isFileDownloading = this.isFileDownloading; @@ -171,6 +182,14 @@ ServiceMessages[COMMAND.nativeDialogsInfoSet] = function (msg) { } }; -ServiceMessages[COMMAND.takeScreenshot] = function () { - //TODO: +ServiceMessages[COMMAND.takeScreenshot] = async function (msg) { + try { + return await this.screenshotCapturer.captureAction(msg); + } + catch (e) { + // NOTE: swallow the error silently if we can't take screenshots for some + // reason (e.g. we don't have permissions to write a screenshot file). + return null; + } + }; diff --git a/test/client/fixtures/runner/cross-domain/in-iframe-test.js b/test/client/fixtures/runner/cross-domain/in-iframe-test.js index 9f5e71e4b71..ad60d06f9cb 100644 --- a/test/client/fixtures/runner/cross-domain/in-iframe-test.js +++ b/test/client/fixtures/runner/cross-domain/in-iframe-test.js @@ -64,7 +64,7 @@ asyncTest('run steps in iframe', function () { } }; - testRunner._onError = function () { + testRunner._onFatalError = function () { errorRaised = true; }; @@ -104,7 +104,7 @@ asyncTest('element error', function () { } ]; - testRunner._onError = function () { + testRunner._onFatalError = function () { errorRaised = true; }; @@ -206,7 +206,7 @@ asyncTest('shared data', function () { assertionFailed = true; }; - testRunner._onErrorRaised = function () { + testRunner._onFatalError = function () { errorRaised = true; }; @@ -353,7 +353,7 @@ asyncTest('waiting for postback', function () { assertionFailed = true; }; - testRunner._onError = function () { + testRunner._onFatalError = function () { errorRaised = true; }; diff --git a/test/client/fixtures/runner/cross-domain/run-test.js b/test/client/fixtures/runner/cross-domain/run-test.js index 3460f941cd0..4ae3ec4e868 100644 --- a/test/client/fixtures/runner/cross-domain/run-test.js +++ b/test/client/fixtures/runner/cross-domain/run-test.js @@ -48,7 +48,7 @@ asyncTest('run test', function () { storedIFrameStepExecuted.call(runner); }; - runner._onError = function () { + runner._onFatalError = function () { errorRaised = true; }; diff --git a/test/client/fixtures/runner/detection-element-after-events-simulation-test.js b/test/client/fixtures/runner/detection-element-after-events-simulation-test.js index d3b7895b41b..32e1b63ff53 100644 --- a/test/client/fixtures/runner/detection-element-after-events-simulation-test.js +++ b/test/client/fixtures/runner/detection-element-after-events-simulation-test.js @@ -47,7 +47,7 @@ $(document).ready(function () { runArgumentsIterator(items, seriesActionsRun, asyncActionCallback); }; - transport.fail = function (err) { + transport.fatalError = function (err) { currentErrorType = err.type; if (err.element) currentErrorElement = err.element; @@ -97,7 +97,7 @@ $(document).ready(function () { callbackFunction(); }; actions(); - var timeoutId = setTimeout(function () { + var timeoutId = setTimeout(function () { callbackFunction = function () { }; ok(false, 'Timeout is exceeded'); diff --git a/test/client/fixtures/runner/element-availability-waiting-test.js b/test/client/fixtures/runner/element-availability-waiting-test.js index fed259be242..8a6fab64650 100644 --- a/test/client/fixtures/runner/element-availability-waiting-test.js +++ b/test/client/fixtures/runner/element-availability-waiting-test.js @@ -18,7 +18,7 @@ actionsAPI.init(stepIterator); var errorRaised = false; -transport.fail = function (err) { +transport.fatalError = function (err) { if (err) { errorRaised = true; ok(!errorRaised, 'error raised'); diff --git a/test/client/fixtures/runner/runner-base-test.js b/test/client/fixtures/runner/runner-base-test.js index 3b0d99f93c8..3b32b62b517 100644 --- a/test/client/fixtures/runner/runner-base-test.js +++ b/test/client/fixtures/runner/runner-base-test.js @@ -37,11 +37,11 @@ $.fn.load = function (callback) { var lastError = null; -RunnerBase.prototype._onError = function (err) { +RunnerBase.prototype._onFatalError = function (err) { lastError = err; }; -Runner.prototype._onError = function (err) { +Runner.prototype._onFatalError = function (err) { lastError = err; }; @@ -114,8 +114,8 @@ function wrapIFrameArgument (arg) { } test('DOM element', function () { - var arg = null, - testRunner = new RunnerBase(); + var arg = null, + testRunner = new RunnerBase(); testRunner._initApi(); testRunner._runInIFrame = function (iFrame) { @@ -129,8 +129,8 @@ test('DOM element', function () { }); test('jQuery object', function () { - var arg = null, - testRunner = new RunnerBase(); + var arg = null, + testRunner = new RunnerBase(); testRunner._initApi(); testRunner._runInIFrame = function (iFrame) { @@ -144,8 +144,8 @@ test('jQuery object', function () { }); test('string selector', function () { - var arg = null, - testRunner = new RunnerBase(); + var arg = null, + testRunner = new RunnerBase(); testRunner._initApi(); testRunner._runInIFrame = function (iFrame) { @@ -159,8 +159,8 @@ test('string selector', function () { }); test('function', function () { - var arg = null, - testRunner = new RunnerBase(); + var arg = null, + testRunner = new RunnerBase(); testRunner._initApi(); testRunner._runInIFrame = function (iFrame) { @@ -183,7 +183,7 @@ test('empty argument error', function () { testRunner.inIFrame(wrapIFrameArgument(null), 0)(); equal(lastError.type, ERROR_TYPE.emptyIFrameArgument); - lastError = null; + lastError = null; testRunner.inIFrame(wrapIFrameArgument('#notExistingIFrame'), 0)(); equal(lastError.type, ERROR_TYPE.emptyIFrameArgument); @@ -218,9 +218,9 @@ test('incorrect argument error', function () { testRunner.inIFrame(wrapIFrameArgument(['#iframe']), 0)(); equal(lastError.type, ERROR_TYPE.incorrectIFrameArgument); - lastError = null; + lastError = null; testRunner.inIFrame(wrapIFrameArgument({ iFrame: $('#iframe') }), 0)(); equal(lastError.type, ERROR_TYPE.incorrectIFrameArgument); - lastError = null; + lastError = null; }); diff --git a/test/client/fixtures/runner/runner-test.js b/test/client/fixtures/runner/runner-test.js index bf3f726116e..2686876a7f2 100644 --- a/test/client/fixtures/runner/runner-test.js +++ b/test/client/fixtures/runner/runner-test.js @@ -42,7 +42,7 @@ asyncTest('T204773 - TestCafe - The assertion in last step with inIFrame wrapper var testRunner = new Runner(); - testRunner._onAssertionFailed({ err: 'err' }); + testRunner._onAssertionFailed({ err: { message: 'err' } }); testRunner._onTestComplete({ callback: function () { @@ -53,44 +53,53 @@ asyncTest('T204773 - TestCafe - The assertion in last step with inIFrame wrapper }); -test('Test iterator should not call Transport.fail twice (without screenshots)', function () { - var savedTakeScreenshotOnFails = HH_SETTINGS.TAKE_SCREENSHOT_ON_FAILS, - savedTransportFail = transport.fail; +asyncTest('Test iterator should not call Transport.fail twice (without screenshots)', function () { + var savedTakeScreenshotOnFails = HH_SETTINGS.TAKE_SCREENSHOTS_ON_FAILS, + savedTransportFatalError = transport.fatalError; var transportFailCount = 0; - transport.fail = function () { + transport.fatalError = function () { transportFailCount++; }; - HH_SETTINGS.TAKE_SCREENSHOT_ON_FAILS = false; + + HH_SETTINGS.TAKE_SCREENSHOTS_ON_FAILS = false; var testRunner = new Runner(); - testRunner._onError(); - testRunner._onError(); + testRunner._onFatalError({ message: 'err1' }); + testRunner._onFatalError({ message: 'err2' }); + + window.setTimeout(function () { + equal(transportFailCount, 1); - equal(transportFailCount, 1); + HH_SETTINGS.TAKE_SCREENSHOTS_ON_FAILS = savedTakeScreenshotOnFails; + transport.fatalError = savedTransportFatalError; - HH_SETTINGS.TAKE_SCREENSHOT_ON_FAILS = savedTakeScreenshotOnFails; - transport.fail = savedTransportFail; + start(); + }, 100); }); -test('Test iterator should not call Transport.fail twice (with screenshots)', function () { - var savedTakeScreenshotOnFails = HH_SETTINGS.TAKE_SCREENSHOT_ON_FAILS, - savedTransportFail = transport.fail; +asyncTest('Test iterator should not call Transport.fail twice (with screenshots)', function () { + var savedTakeScreenshotOnFails = HH_SETTINGS.TAKE_SCREENSHOTS_ON_FAILS, + savedTransportFatalError = transport.fatalError; var transportFailCount = 0; - transport.fail = function () { + transport.fatalError = function () { transportFailCount++; }; - HH_SETTINGS.TAKE_SCREENSHOT_ON_FAILS = true; + HH_SETTINGS.TAKE_SCREENSHOTS_ON_FAILS = true; var testRunner = new Runner(); - testRunner._onError(); - testRunner._onError(); + testRunner._onFatalError({ message: 'err1' }); + testRunner._onFatalError({ message: 'err2' }); + + window.setTimeout(function () { + equal(transportFailCount, 1); - equal(transportFailCount, 1); + HH_SETTINGS.TAKE_SCREENSHOTS_ON_FAILS = savedTakeScreenshotOnFails; + transport.fatalError = savedTransportFatalError; - HH_SETTINGS.TAKE_SCREENSHOT_ON_FAILS = savedTakeScreenshotOnFails; - transport.fail = savedTransportFail; + start(); + }, 100); }); diff --git a/test/client/fixtures/runner/take-screenshots-test.js b/test/client/fixtures/runner/take-screenshots-test.js deleted file mode 100644 index 12c471e5010..00000000000 --- a/test/client/fixtures/runner/take-screenshots-test.js +++ /dev/null @@ -1,248 +0,0 @@ -var testCafeCore = window.getTestCafeModule('testCafeCore'); -var transport = testCafeCore.get('./transport'); -var ERROR_TYPE = testCafeCore.ERROR_TYPE; -var SETTINGS = testCafeCore.get('./settings').get(); - -var testCafeRunner = window.getTestCafeModule('testCafeRunner'); -var Runner = testCafeRunner.get('./runner'); -var actionsAPI = testCafeRunner.get('./api/actions'); -var actionBarrier = testCafeRunner.get('./action-barrier/action-barrier'); - - -var runner = null, - lastError = null, - lastIsFailedStep = false, - screenShotRequestCount = false, - expectedError = null, - expectedScreenshotCount = 0; - -transport.batchUpdate = function (callback) { - callback(); -}; - -transport.fail = function (err) { - ok(err.type === expectedError); - ok(screenShotRequestCount === expectedScreenshotCount); - - runner._destroyIFrameBehavior(); - $('iframe').remove(); - start(); -}; -transport.asyncServiceMsg = function (msg, callback) { - if (msg.cmd === 'CMD_TAKE_SCREENSHOT') { //TODO: fix - screenShotRequestCount++; - ok(msg.isFailedStep); - } - - if (callback) - callback(); -}; -transport.assertionFailed = function () { -}; - -actionBarrier.waitPageInitialization = function (callback) { - callback(); -}; -$.fn.load = function (callback) { - callback(); -}; - -Runner.checkStatus = function () { -}; - -QUnit.testStart(function () { - runner = new Runner(); - screenShotRequestCount = 0; - expectedError = null; - expectedScreenshotCount = 0; - lastIsFailedStep = false; -}); - -asyncTest('Uncaught error in test script', function () { - var errorText = 'Test error', - stepNames = ['1.Step name'], - testSteps = [function () { - throw errorText; - }]; - - SETTINGS.TAKE_SCREENSHOT_ON_FAILS = true; - expectedError = ERROR_TYPE.uncaughtJSErrorInTestCodeStep; - expectedScreenshotCount = 1; - - runner.act._start(stepNames, testSteps, 0); -}); - -asyncTest('Invisible element', function () { - var stepNames = ['1.Step name'], - testSteps = [function () { - actionsAPI.click('body1'); - }]; - - SETTINGS.TAKE_SCREENSHOT_ON_FAILS = true; - expectedError = ERROR_TYPE.emptyFirstArgument; - expectedScreenshotCount = 1; - - runner.act._start(stepNames, testSteps, 0); -}); - -asyncTest('Failed assertion in step with action', function () { - var stepNames = ['1.Step name'], - eq = runner.eq, - ok = runner.ok, - testSteps = [function () { - ok(0); - eq(0, 1); - actionsAPI.wait('body1'); - }]; - - SETTINGS.TAKE_SCREENSHOT_ON_FAILS = true; - expectedError = ERROR_TYPE.incorrectWaitActionMillisecondsArgument; - expectedScreenshotCount = 1; - - runner.act._start(stepNames, testSteps, 0); -}); - -asyncTest('Failed assertion in step without action', function () { - var stepNames = ['1.Step name'], - eq = runner.eq, - ok = runner.ok, - testSteps = [ - function () { - ok(0); - eq(0, 1); - }, - function () { - actionsAPI.wait('#thowError'); - } - ]; - - SETTINGS.TAKE_SCREENSHOT_ON_FAILS = true; - expectedError = ERROR_TYPE.incorrectWaitActionMillisecondsArgument; - expectedScreenshotCount = 2; - - runner.act._start(stepNames, testSteps, 0); -}); - -asyncTest('Failed assertion and error: without "Take scr" flag', function () { - var stepNames = ['1.Step name'], - eq = runner.eq, - ok = runner.ok, - testSteps = [ - function () { - ok(0); - eq(0, 1); - }, - function () { - actionsAPI.wait('#thowError'); - } - ]; - - SETTINGS.TAKE_SCREENSHOT_ON_FAILS = false; - expectedError = ERROR_TYPE.incorrectWaitActionMillisecondsArgument; - expectedScreenshotCount = 0; - - runner.act._start(stepNames, testSteps, 0); -}); - -module('in IFrame'); - -asyncTest('Uncaught error in test script', function () { - var $iframe = $('