From 8afefd412fdb32aa104976f9ff32877dd84b2f51 Mon Sep 17 00:00:00 2001 From: Alex Mos Date: Tue, 20 Oct 2015 14:46:58 +0300 Subject: [PATCH] Enable screenshot functionality (closes #104) --- src/client/runner/api/actions.js | 4 +- src/client/runner/iframe-runner.js | 4 +- src/client/runner/runner-base.js | 2 + src/client/runner/runner.js | 42 +++++++------------ src/client/runner/step-iterator.js | 8 +++- src/client/test-run/index.js.mustache | 1 + src/reporters/base.js | 15 ++++--- src/reporters/json.js | 4 +- src/reporters/list.js | 5 ++- src/reporters/spec.js | 5 ++- src/reporters/xunit.js | 5 ++- src/runner/index.js | 4 +- src/runner/test-run/index.js | 30 ++++++++++++- src/utils/fs.js | 34 +++++++++++++++ .../fixtures/runner/take-screenshots-test.js | 33 ++++++++++++--- test/report-design-viewer/index.js | 25 ++++++++--- test/server/data/expected-reports/json | 18 +++++--- test/server/data/expected-reports/list | 2 +- test/server/data/expected-reports/spec | 2 +- test/server/data/expected-reports/xunit | 2 +- test/server/reporters-test.js | 42 ++++++++++++------- 21 files changed, 204 insertions(+), 83 deletions(-) create mode 100644 src/utils/fs.js diff --git a/src/client/runner/api/actions.js b/src/client/runner/api/actions.js index d9a73514921..00c129eff4c 100644 --- a/src/client/runner/api/actions.js +++ b/src/client/runner/api/actions.js @@ -698,11 +698,11 @@ export function upload (what, path) { ); } -export function screenshot () { +export function screenshot (filePath) { stepIterator.asyncAction(function (iteratorCallback) { stepIterator.takeScreenshot(function () { iteratorCallback(); - }, false); + }, false, filePath); }); } diff --git a/src/client/runner/iframe-runner.js b/src/client/runner/iframe-runner.js index 43a356c5bd2..80385b45dc4 100644 --- a/src/client/runner/iframe-runner.js +++ b/src/client/runner/iframe-runner.js @@ -111,7 +111,9 @@ IFrameRunner.prototype._onGetStepsSharedData = function (e) { IFrameRunner.prototype._onTakeScreenshot = function (e) { var msg = { cmd: RunnerBase.IFRAME_TAKE_SCREENSHOT_REQUEST_CMD, - isFailedStep: e.isFailedStep + isFailedStep: e.isFailedStep, + stepName: e.stepName, + filePath: e.filePath }; messageSandbox.sendServiceMsg(msg, window.top); diff --git a/src/client/runner/runner-base.js b/src/client/runner/runner-base.js index 64898834e92..9589c2341f1 100644 --- a/src/client/runner/runner-base.js +++ b/src/client/runner/runner-base.js @@ -255,6 +255,8 @@ RunnerBase.prototype._initIFrameBehavior = function () { case RunnerBase.IFRAME_TAKE_SCREENSHOT_REQUEST_CMD: runner._onTakeScreenshot({ isFailedStep: message.isFailedStep, + stepName: message.stepName, + filePath: message.filePath, callback: function () { msg = { cmd: RunnerBase.IFRAME_TAKE_SCREENSHOT_RESPONSE_CMD diff --git a/src/client/runner/runner.js b/src/client/runner/runner.js index 568203a3fd8..cbe8e4e7679 100644 --- a/src/client/runner/runner.js +++ b/src/client/runner/runner.js @@ -106,8 +106,7 @@ Runner.prototype._onError = function (err) { this._onTakeScreenshot({ isFailedStep: true, - //TODO: - //withoutStepName: !(ERRORS.hasErrorStepName(err) && ERRORS.hasErrorStepName(err)), + stepName: this.stepIterator.getCurrentStep(), callback: function () { runner.stopped = true; transport.fail(err, Runner.checkStatus); @@ -138,39 +137,26 @@ Runner.prototype._onGetStepsSharedData = function (e) { }; Runner.prototype._onTakeScreenshot = function (e) { - var savedTitle = document.title, - windowMark = '[tc-' + Date.now() + ']', - browserName = null, - callback = e && e.callback ? e.callback : function () { + var savedTitle = document.title, + pageUrl = window.location.toString(), + callback = e && e.callback ? e.callback : function () { }, - runner = this; - - runner.eventEmitter.emit(RunnerBase.SCREENSHOT_CREATING_STARTED_EVENT, {}); + runner = this; + if (!SETTINGS.get().ENABLE_SCREENSHOTS) + return callback(); - if (browserUtils.isMSEdge) - browserName = 'MSEDGE'; - else if (browserUtils.isSafari) - browserName = 'SAFARI'; - else if (browserUtils.isOpera || browserUtils.isOperaWithWebKit) - browserName = 'OPERA'; - else if (browserUtils.isWebKit) - browserName = 'CHROME'; - else if (browserUtils.isMozilla) - browserName = 'FIREFOX'; - else if (browserUtils.isIE) - browserName = 'IE'; + runner.eventEmitter.emit(RunnerBase.SCREENSHOT_CREATING_STARTED_EVENT, {}); var msg = { - cmd: 'CMD_TAKE_SCREENSHOT', //TODO: fix - windowMark: windowMark, - browserName: browserName, - isFailedStep: e.isFailedStep, - withoutStepName: e.withoutStepName, - url: window.location.toString() + cmd: COMMAND.takeScreenshot, + pageUrl: pageUrl, + isFailedStep: e.isFailedStep, + stepName: e.stepName, + filePath: e.filePath }; - var assignedTitle = savedTitle + windowMark, + var assignedTitle = `[ ${pageUrl} ]`, checkTitleIntervalId = window.setInterval(function () { if (document.title !== assignedTitle) { savedTitle = document.title; diff --git a/src/client/runner/step-iterator.js b/src/client/runner/step-iterator.js index bf1568e676d..0c72008756f 100644 --- a/src/client/runner/step-iterator.js +++ b/src/client/runner/step-iterator.js @@ -498,6 +498,9 @@ StepIterator.prototype.runLast = function () { }; StepIterator.prototype.getCurrentStep = function () { + if (!this.initialized) + return ''; + return this.state.stepNames ? this.state.stepNames[this.state.step - 1] : SETTINGS.get().CURRENT_TEST_STEP_NAME; }; @@ -513,6 +516,7 @@ StepIterator.prototype.onActionRun = function () { this.eventEmitter.emit(StepIterator.ACTION_RUN_EVENT, {}); }; + //Global __waitFor() StepIterator.prototype.setGlobalWaitFor = function (event, timeout) { this.globalWaitForEvent = event; @@ -545,9 +549,11 @@ StepIterator.prototype.__waitFor = function (callback) { }); }; -StepIterator.prototype.takeScreenshot = function (callback, isFailedStep) { +StepIterator.prototype.takeScreenshot = function (callback, isFailedStep, filePath) { this.eventEmitter.emit(StepIterator.TAKE_SCREENSHOT_EVENT, { isFailedStep: isFailedStep, + stepName: this.getCurrentStep(), + filePath: filePath || '', callback: callback }); }; diff --git a/src/client/test-run/index.js.mustache b/src/client/test-run/index.js.mustache index 0abcd00575d..ec36c4c6a29 100644 --- a/src/client/test-run/index.js.mustache +++ b/src/client/test-run/index.js.mustache @@ -41,6 +41,7 @@ testCafeCore.SETTINGS.set({ CURRENT_TEST_STEP_NAME: nextStep ? stepNames[nextStep - 1] : 'Test initialization', BROWSER_STATUS_URL: '{{{browserStatusUrl}}}', + ENABLE_SCREENSHOTS: {{{enableScreenshots}}}, TAKE_SCREENSHOT_ON_FAILS: {{{takeScreenshotOnFails}}}, SKIP_JS_ERRORS: {{{skipJsErrors}}}, ENABLE_SOURCE_INDEX: true, diff --git a/src/reporters/base.js b/src/reporters/base.js index 4a7a14a5bf0..851fa7e3bac 100644 --- a/src/reporters/base.js +++ b/src/reporters/base.js @@ -77,15 +77,16 @@ export default class BaseReporter { // This happens because the next test can't be completed until the // previous one frees all browser connections. // Therefore, tests always get completed sequentially. - var reportItem = this.reportQueue.shift(); - var durationMs = new Date() - reportItem.startTime; + var reportItem = this.reportQueue.shift(); + var durationMs = new Date() - reportItem.startTime; + var screenshotPath = reportItem.screenshotCreated ? reportItem.screenshotPath : null; if (!reportItem.errs.length) this.passed++; BaseReporter._errorSorter(reportItem.errs); - this._reportTestDone(reportItem.testName, reportItem.errs, durationMs, reportItem.unstable); + this._reportTestDone(reportItem.testName, reportItem.errs, durationMs, reportItem.unstable, screenshotPath); // NOTE: here we assume that tests are sorted by fixture. // Therefore, if the next report item has a different @@ -120,8 +121,10 @@ export default class BaseReporter { testRun.errs.forEach(err => err.userAgent = userAgent); reportItem.pendingRuns--; - reportItem.errs = reportItem.errs.concat(testRun.errs); - reportItem.unstable = reportItem.unstable || testRun.unstable; + reportItem.errs = reportItem.errs.concat(testRun.errs); + reportItem.unstable = reportItem.unstable || testRun.unstable; + reportItem.screenshotCreated = reportItem.screenshotCreated || testRun.screenshotCreated; + reportItem.screenshotPath = reportItem.screenshotPath || testRun.test.screenshotPath; if (!reportItem.pendingRuns) this._shiftReportQueue(); @@ -182,7 +185,7 @@ export default class BaseReporter { throw new Error('Not implemented'); } - _reportTestDone (name, errs, durationMs, unstable) { + _reportTestDone (name, errs, durationMs, unstable, screenshotPath) { throw new Error('Not implemented'); } diff --git a/src/reporters/json.js b/src/reporters/json.js index 0ef8b6d2f70..7f72c1d718f 100644 --- a/src/reporters/json.js +++ b/src/reporters/json.js @@ -27,10 +27,10 @@ export default class JSONReporter extends BaseReporter { this.report.fixtures.push(this.currentFixture); } - _reportTestDone (name, errs, durationMs, unstable) { + _reportTestDone (name, errs, durationMs, unstable, screenshotPath) { errs = errs.map(err => this._formatError(err)); - this.currentFixture.tests.push({ name, errs, durationMs, unstable }); + this.currentFixture.tests.push({ name, errs, durationMs, unstable, screenshotPath }); } _reportTaskDone (passed, total, endTime) { diff --git a/src/reporters/list.js b/src/reporters/list.js index 23409d4fcf9..f12d773b004 100644 --- a/src/reporters/list.js +++ b/src/reporters/list.js @@ -11,7 +11,7 @@ export default class ListReporter extends SpecReporter { this.currentFixtureName = name; } - _reportTestDone (name, errs, durationMs, unstable) { + _reportTestDone (name, errs, durationMs, unstable, screenshotPath) { var hasErr = !!errs.length; var nameStyle = hasErr ? this.style.red : this.style.gray; var symbol = hasErr ? this.style.red(this.symbols.err) : this.style.green(this.symbols.ok); @@ -25,6 +25,9 @@ export default class ListReporter extends SpecReporter { if (unstable) title += this.style.yellow(' (unstable)'); + if (screenshotPath) + title += ` (screenshots: ${screenshotPath})`; + this._write(title); if (hasErr) { diff --git a/src/reporters/spec.js b/src/reporters/spec.js index 1919eeb178d..c5d4770df36 100644 --- a/src/reporters/spec.js +++ b/src/reporters/spec.js @@ -36,7 +36,7 @@ export default class SpecReporter extends BaseReporter { ._newline(); } - _reportTestDone (name, errs, durationMs, unstable) { + _reportTestDone (name, errs, durationMs, unstable, screenshotPath) { var hasErr = !!errs.length; var nameStyle = hasErr ? this.style.red : this.style.gray; var symbol = hasErr ? this.style.red(this.symbols.err) : this.style.green(this.symbols.ok); @@ -47,6 +47,9 @@ export default class SpecReporter extends BaseReporter { if (unstable) title += this.style.yellow(' (unstable)'); + if (screenshotPath) + title += ` (screenshots: ${screenshotPath})`; + this._write(title); if (hasErr) { diff --git a/src/reporters/xunit.js b/src/reporters/xunit.js index 489338d87a9..b0e749bcab9 100644 --- a/src/reporters/xunit.js +++ b/src/reporters/xunit.js @@ -25,12 +25,15 @@ export default class XUnitReporter extends BaseReporter { this.currentFixtureName = escapeHtml(name); } - _reportTestDone (name, errs, durationMs, unstable) { + _reportTestDone (name, errs, durationMs, unstable, screenshotPath) { var hasErr = !!errs.length; if (unstable) name += ' (unstable)'; + if (screenshotPath) + name += ` (screenshots: ${screenshotPath})`; + name = escapeHtml(name); var openTag = ` this.screenshotCreated = true); }; diff --git a/src/utils/fs.js b/src/utils/fs.js new file mode 100644 index 00000000000..56c5e1719cb --- /dev/null +++ b/src/utils/fs.js @@ -0,0 +1,34 @@ +import fs from 'fs'; +import path from 'path'; +import promisify from 'es6-promisify'; + + +var stat = promisify(fs.stat); +var mkdirPromisified = promisify(fs.mkdir); + + +export async function exists (dirPath) { + try { + await stat(dirPath); + } + catch (err) { + return false; + } + + return true; +} + +export async function mkdir (dirPath) { + var dirs = dirPath.split(path.sep); + var i = 0; + var curPath = ''; + + while (dirs[i]) { + curPath = curPath ? path.join(curPath, dirs[i]) : dirs[i]; + + if (!await exists(curPath)) + await mkdirPromisified(curPath); + + i++; + } +} diff --git a/test/client/fixtures/runner/take-screenshots-test.js b/test/client/fixtures/runner/take-screenshots-test.js index 138bd2444e7..da830d1a270 100644 --- a/test/client/fixtures/runner/take-screenshots-test.js +++ b/test/client/fixtures/runner/take-screenshots-test.js @@ -2,6 +2,7 @@ var testCafeCore = window.getTestCafeModule('testCafeCore'); var transport = testCafeCore.get('./transport'); var ERROR_TYPE = testCafeCore.ERROR_TYPE; var SETTINGS = testCafeCore.get('./settings').get(); +var COMMAND = testCafeCore.COMMAND; var testCafeRunner = window.getTestCafeModule('testCafeRunner'); var Runner = testCafeRunner.get('./runner'); @@ -14,13 +15,16 @@ var runner = null, lastIsFailedStep = false, screenShotRequestCount = false, expectedError = null, + expectedStepName = null, expectedScreenshotCount = 0; +SETTINGS.ENABLE_SCREENSHOTS = true; + transport.batchUpdate = function (callback) { callback(); }; -transport.fail = function (err) { +transport.fail = function (err) { ok(err.code === expectedError); ok(screenShotRequestCount === expectedScreenshotCount); @@ -28,22 +32,30 @@ transport.fail = function (err) { $('iframe').remove(); start(); }; + transport.asyncServiceMsg = function (msg, callback) { - if (msg.cmd === 'CMD_TAKE_SCREENSHOT') { //TODO: fix + if (msg.cmd === COMMAND.takeScreenshot) { screenShotRequestCount++; ok(msg.isFailedStep); + equal(msg.pageUrl, window.location.toString()); + ok(document.title.indexOf(msg.pageUrl) > -1); + + if (expectedStepName !== null) + equal(msg.stepName, expectedStepName); } if (callback) callback(); }; + transport.assertionFailed = function () { }; actionBarrier.waitPageInitialization = function (callback) { callback(); }; -$.fn.load = function (callback) { + +$.fn.load = function (callback) { callback(); }; @@ -56,6 +68,7 @@ QUnit.testStart(function () { expectedError = null; expectedScreenshotCount = 0; lastIsFailedStep = false; + expectedStepName = null; }); asyncTest('Uncaught error in test script', function () { @@ -68,6 +81,7 @@ asyncTest('Uncaught error in test script', function () { SETTINGS.TAKE_SCREENSHOT_ON_FAILS = true; expectedError = ERROR_TYPE.uncaughtJSErrorInTestCodeStep; expectedScreenshotCount = 1; + expectedStepName = '1.Step name'; runner.act._start(stepNames, testSteps, 0); }); @@ -81,6 +95,7 @@ asyncTest('Invisible element', function () { SETTINGS.TAKE_SCREENSHOT_ON_FAILS = true; expectedError = ERROR_TYPE.emptyFirstArgument; expectedScreenshotCount = 1; + expectedStepName = '1.Step name'; runner.act._start(stepNames, testSteps, 0); }); @@ -98,20 +113,23 @@ asyncTest('Failed assertion in step with action', function () { SETTINGS.TAKE_SCREENSHOT_ON_FAILS = true; expectedError = ERROR_TYPE.incorrectWaitActionMillisecondsArgument; expectedScreenshotCount = 1; + expectedStepName = '1.Step name'; runner.act._start(stepNames, testSteps, 0); }); asyncTest('Failed assertion in step without action', function () { - var stepNames = ['1.Step name'], + var stepNames = ['1.First step', '2.Second step'], eq = runner.eq, ok = runner.ok, testSteps = [ function () { + expectedStepName = '1.First step'; ok(0); eq(0, 1); }, function () { + expectedStepName = '2.Second step'; actionsAPI.wait('#thowError'); } ]; @@ -124,15 +142,17 @@ asyncTest('Failed assertion in step without action', function () { }); asyncTest('Failed assertion and error: without "Take scr" flag', function () { - var stepNames = ['1.Step name'], + var stepNames = ['1.First step', '2.Second step'], eq = runner.eq, ok = runner.ok, testSteps = [ function () { + expectedStepName = '1.First step'; ok(0); eq(0, 1); }, function () { + expectedStepName = '2.Second step'; actionsAPI.wait('#thowError'); } ]; @@ -140,6 +160,7 @@ asyncTest('Failed assertion and error: without "Take scr" flag', function () { SETTINGS.TAKE_SCREENSHOT_ON_FAILS = false; expectedError = ERROR_TYPE.incorrectWaitActionMillisecondsArgument; expectedScreenshotCount = 0; + expectedStepName = '1.Step name'; runner.act._start(stepNames, testSteps, 0); }); @@ -166,6 +187,7 @@ asyncTest('Uncaught error in test script', function () { SETTINGS.TAKE_SCREENSHOT_ON_FAILS = true; expectedError = ERROR_TYPE.uncaughtJSErrorInTestCodeStep; expectedScreenshotCount = 1; + runner._runInIFrame($iframe[0], steps[0].stepName, steps[0].step, steps[0].stepNum); }); }); @@ -215,6 +237,7 @@ asyncTest('Error in api iframe argument', function () { SETTINGS.TAKE_SCREENSHOT_ON_FAILS = true; expectedError = ERROR_TYPE.iframeArgumentIsNotIFrame; expectedScreenshotCount = 1; + expectedStepName = '1'; runner.act._start(stepNames, steps, 0); }); diff --git a/test/report-design-viewer/index.js b/test/report-design-viewer/index.js index 1264a39f816..734e53956d5 100644 --- a/test/report-design-viewer/index.js +++ b/test/report-design-viewer/index.js @@ -6,41 +6,53 @@ var errs = require('./errors'); var decoratorsPath = '../../lib/reporters/errors/decorators'; +var screenshotDir = '/Screenshots'; + var testMocks = [ { name: 'fixture1Test1', fixture: { name: 'fixture1', path: './fixture1.js' - } + }, + + screenshotPath: screenshotDir }, { name: 'fixture1Test2', fixture: { name: 'fixture1', path: './fixture1.js' - } + }, + + screenshotPath: screenshotDir }, { name: 'fixture2Test1', fixture: { name: 'fixture2', path: './fixture2.js' - } + }, + + screenshotPath: screenshotDir }, { name: 'fixture2Test2', fixture: { name: 'fixture2', path: './fixture2.js' - } + }, + + screenshotPath: screenshotDir }, { name: 'fixture3Test1', fixture: { name: 'fixture3', path: './fixture3.js' - } + }, + + screenshotPath: screenshotDir } ]; @@ -61,7 +73,8 @@ var testRunMocks = [ test: testMocks[0], unstable: false, browserConnection: browserConnectionMock, - errs: errs[0] + errs: errs[0], + screenshotCreated: true }, { test: testMocks[1], diff --git a/test/server/data/expected-reports/json b/test/server/data/expected-reports/json index 1fc18571554..97d23034904 100644 --- a/test/server/data/expected-reports/json +++ b/test/server/data/expected-reports/json @@ -16,7 +16,8 @@ "name": "fixture1test1", "errs": [], "durationMs": 74000, - "unstable": true + "unstable": true, + "screenshotPath": "/Screenshots" }, { "name": "fixture1test2", @@ -26,13 +27,15 @@ "Firefox\nAssertion failed at step \"Step\":\n\n ok(false)\n\nExpected: not null, not undefined, not false, not NaN and not ''\nActual: false" ], "durationMs": 74000, - "unstable": false + "unstable": false, + "screenshotPath": null }, { "name": "fixture1test3", "errs": [], "durationMs": 74000, - "unstable": false + "unstable": false, + "screenshotPath": null } ] }, @@ -44,13 +47,15 @@ "name": "fixture2test1", "errs": [], "durationMs": 74000, - "unstable": false + "unstable": false, + "screenshotPath": null }, { "name": "fixture2test2", "errs": [], "durationMs": 74000, - "unstable": false + "unstable": false, + "screenshotPath": null } ] }, @@ -64,7 +69,8 @@ "Firefox\nAssertion failed at step \"Step\":\n\n eq([\"12345678901\"], [\"00000000000\"])\n\nArrays differ at index 0:\n\nExpected: [0]: \"12345678901\"\nActual: [0]: \"00000000000\"\n ^" ], "durationMs": 74000, - "unstable": true + "unstable": true, + "screenshotPath": null } ] } diff --git a/test/server/data/expected-reports/list b/test/server/data/expected-reports/list index fd1c4e590e4..1758da96478 100644 --- a/test/server/data/expected-reports/list +++ b/test/server/data/expected-reports/list @@ -1,5 +1,5 @@ Running tests in: Chrome, Firefox - ✓ fixture1 - fixture1test1 (unstable) + ✓ fixture1 - fixture1test1 (unstable) (screenshots: /Screenshots) ✖ fixture1 - fixture1test2 1) Chrome diff --git a/test/server/data/expected-reports/spec b/test/server/data/expected-reports/spec index 5fb3299e7dd..af234852cfe 100644 --- a/test/server/data/expected-reports/spec +++ b/test/server/data/expected-reports/spec @@ -1,7 +1,7 @@ Running tests in: Chrome, Firefox fixture1 (./fixture1.js) - ✓ fixture1test1 (unstable) + ✓ fixture1test1 (unstable) (screenshots: /Screenshots) ✖ fixture1test2 1) Chrome diff --git a/test/server/data/expected-reports/xunit b/test/server/data/expected-reports/xunit index b4a6e92f648..948b5895788 100644 --- a/test/server/data/expected-reports/xunit +++ b/test/server/data/expected-reports/xunit @@ -1,6 +1,6 @@ - +