From 55a91ea137395891248db148df75dd6408c3b3a2 Mon Sep 17 00:00:00 2001 From: Hank Duan Date: Thu, 6 Nov 2014 15:28:09 -0800 Subject: [PATCH] feat(launcher): reorganize launcher + add option to store test results as JSON --- docs/referenceConf.js | 4 + lib/cli.js | 1 + lib/frameworks/README.md | 15 +- lib/frameworks/cucumber.js | 62 ++- lib/frameworks/jasmine.js | 25 +- lib/frameworks/mocha.js | 51 ++- lib/launcher.js | 458 +++++++---------------- lib/runner.js | 11 +- lib/{runFromLauncher.js => runnerCli.js} | 12 +- lib/taskLogger.js | 88 +++++ lib/taskRunner.js | 131 +++++++ lib/taskScheduler.js | 50 +-- 12 files changed, 517 insertions(+), 391 deletions(-) rename lib/{runFromLauncher.js => runnerCli.js} (83%) create mode 100644 lib/taskLogger.js create mode 100644 lib/taskRunner.js diff --git a/docs/referenceConf.js b/docs/referenceConf.js index a84588aa2..3af371275 100644 --- a/docs/referenceConf.js +++ b/docs/referenceConf.js @@ -212,6 +212,10 @@ exports.config = { } }, + // If set, protractor will save the test output in json format at this path. + // The path is relative to the location of this config. + resultJsonOutputFile: null, + // --------------------------------------------------------------------------- // ----- The test framework -------------------------------------------------- // --------------------------------------------------------------------------- diff --git a/lib/cli.js b/lib/cli.js index b9fe16361..107b78f62 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -50,6 +50,7 @@ var optimist = require('optimist'). describe('stackTrace', 'Print stack trace on error'). describe('params', 'Param object to be passed to the tests'). describe('framework', 'Test framework to use: jasmine, cucumber or mocha'). + describe('resultJsonOutputFile', 'Path to save JSON test result'). alias('browser', 'capabilities.browserName'). alias('name', 'capabilities.name'). alias('platform', 'capabilities.platform'). diff --git a/lib/frameworks/README.md b/lib/frameworks/README.md index 3fb0bdda1..5e5613e74 100644 --- a/lib/frameworks/README.md +++ b/lib/frameworks/README.md @@ -23,5 +23,16 @@ Requirements - `runner.getConfig().onComplete` must be called when tests are finished. - - The returned promise must be resolved when tests are finished and it should return a results object. - This object must have a `failedCount` property. + - The returned promise must be resolved when tests are finished and it should return a results object. This object must have a `failedCount` property and optionally a `specResults` + object of the following structure: + ``` + specResults = [{ + description: string, + assertions: [{ + passed: boolean, + errorMsg: string, + stackTrace: string + }], + duration: integer + }] + ``` diff --git a/lib/frameworks/cucumber.js b/lib/frameworks/cucumber.js index b5e572da8..8a6cfe3c9 100644 --- a/lib/frameworks/cucumber.js +++ b/lib/frameworks/cucumber.js @@ -58,17 +58,65 @@ exports.run = function(runner, specs) { } global.cucumber = Cucumber.Cli(execOptions); - return runner.runTestPreparer().then(function () { - return q.promise(function (resolve, reject) { - global.cucumber.run(function (succeeded) { + var testResult = []; + var failedCount = 0; + // Add a listener into cucumber so that protractor can find out which + // steps passed/failed + var addResultListener = function(formatter) { + var originalHandleStepResultEvent = formatter.handleStepResultEvent; + formatter.handleStepResultEvent = function(event, callback) { + + var stepResult = event.getPayloadItem('stepResult'); + if (stepResult.isSuccessful()) { + runner.emit('testPass'); + testResult.push({ + description: stepResult.getStep().getName(), + assertions: [{ + passed: true + }], + duration: stepResult.getDuration() + }); + } + else if (stepResult.isFailed()) { + runner.emit('testFail'); + ++failedCount; + var failureMessage = stepResult.getFailureException(); + testResult.push({ + description: stepResult.getStep().getName(), + assertions: [{ + passed: false, + errorMsg: failureMessage.message, + stackTrace: failureMessage.stack + }], + duration: stepResult.getDuration() + }); + } + + if (originalHandleStepResultEvent + && typeof(originalHandleStepResultEvent) === 'function') { + originalHandleStepResultEvent(event, callback); + } else { + callback(); + } + } + }; + + return runner.runTestPreparer().then(function() { + return q.promise(function(resolve, reject) { + var cucumberConf = Cucumber.Cli.Configuration(execOptions); + var runtime = Cucumber.Runtime(cucumberConf); + var formatter = cucumberConf.getFormatter(); + addResultListener(formatter); + runtime.attachListener(formatter); + runtime.start(function(succeeded) { try { if (runner.getConfig().onComplete) { runner.getConfig().onComplete(); } - var resolvedObj = { - failedCount: succeeded ? 0 : 1 - }; - resolve(resolvedObj); + resolve({ + failedCount: failedCount, + specResults: testResult + }); } catch (err) { reject(err); } diff --git a/lib/frameworks/jasmine.js b/lib/frameworks/jasmine.js index 4cae42b82..d9af2d9a2 100644 --- a/lib/frameworks/jasmine.js +++ b/lib/frameworks/jasmine.js @@ -13,6 +13,8 @@ exports.run = function(runner, specs) { require('jasminewd'); /* global jasmine */ + var testResult = []; + var RunnerReporter = function(emitter) { this.emitter = emitter; }; @@ -20,13 +22,29 @@ exports.run = function(runner, specs) { RunnerReporter.prototype.reportRunnerStarting = function() {}; RunnerReporter.prototype.reportRunnerResults = function() {}; RunnerReporter.prototype.reportSuiteResults = function() {}; - RunnerReporter.prototype.reportSpecStarting = function() {}; + RunnerReporter.prototype.reportSpecStarting = function() { + this.startTime = new Date(); + }; RunnerReporter.prototype.reportSpecResults = function(spec) { if (spec.results().passed()) { this.emitter.emit('testPass'); } else { this.emitter.emit('testFail'); } + + var entry = { + description: spec.results().description, + assertions: [], + duration: new Date().getTime() - this.startTime.getTime() + }; + spec.results().getItems().forEach(function(item) { + entry.assertions.push({ + passed: item.passed(), + errorMsg: item.passed() ? undefined : item.message, + stackTrace: item.passed() ? undefined : item.trace.stack + }); + }); + testResult.push(entry); }; RunnerReporter.prototype.log = function() {}; @@ -47,7 +65,10 @@ exports.run = function(runner, specs) { if (originalOnComplete) { originalOnComplete(jasmineRunner, log); } - resolve(jasmineRunner.results()); + resolve({ + failedCount: jasmineRunner.results().failedCount, + specResults: testResult + }); } catch(err) { reject(err); } diff --git a/lib/frameworks/mocha.js b/lib/frameworks/mocha.js index b7c5c443c..78b20f851 100644 --- a/lib/frameworks/mocha.js +++ b/lib/frameworks/mocha.js @@ -35,19 +35,11 @@ exports.run = function(runner, specs) { global.it.only = global.iit = originalOnly; global.it.skip = global.xit = mochaAdapters.xit; - }catch(err){ + } catch(err) { deferred.reject(err); } }); - mocha.suite.on('pass', function() { - runner.emit('testPass'); - }); - - mocha.suite.on('fail', function() { - runner.emit('testFail'); - }); - mocha.loadFiles(); runner.runTestPreparer().then(function() { @@ -56,21 +48,46 @@ exports.run = function(runner, specs) { mocha.addFile(file); }); - mocha.run(function(failures) { + var testResult = []; + + var mochaRunner = mocha.run(function(failures) { try { if (runner.getConfig().onComplete) { runner.getConfig().onComplete(); } - var resolvedObj = { - failedCount: failures - }; - - deferred.resolve(resolvedObj); - }catch(err){ + deferred.resolve({ + failedCount: failures, + specResults: testResult + }); + } catch(err) { deferred.reject(err); } }); - }).catch(function(reason){ + + mochaRunner.on('pass', function(test) { + runner.emit('testPass'); + testResult.push({ + description: test.title, + assertions: [{ + passed: true + }], + duration: test.duration + }); + }); + + mochaRunner.on('fail', function(test) { + runner.emit('testFail'); + testResult.push({ + description: test.title, + assertions: [{ + passed: false, + errorMsg: test.err.message, + stackTrace: test.err.stack + }], + duration: test.duration + }); + }); + }).catch(function(reason) { deferred.reject(reason); }); diff --git a/lib/launcher.js b/lib/launcher.js index ac044294e..7baf25bb2 100644 --- a/lib/launcher.js +++ b/lib/launcher.js @@ -4,13 +4,13 @@ */ 'use strict'; -var child = require('child_process'), - ConfigParser = require('./configParser'), +var ConfigParser = require('./configParser'), TaskScheduler = require('./taskScheduler'), helper = require('./util'), - q = require('q'); + q = require('q'), + TaskRunner = require('./taskRunner'); -var launcherPrefix = '[launcher]'; +var logPrefix = '[launcher]'; var RUNNERS_FAILED_EXIT_CODE = 100; /** @@ -19,12 +19,83 @@ var RUNNERS_FAILED_EXIT_CODE = 100; * @private */ var log_ = function() { - var args = [launcherPrefix].concat([].slice.call(arguments)); + var args = [logPrefix].concat([].slice.call(arguments)); console.log.apply(console, args); }; +/** + * Keeps track of a list of task results. Provides method to add a new + * result, aggregate the results into a summary, count failures, + * and save results into a JSON file. + */ +var taskResults_ = { + results_: [], + + add: function(result) { + this.results_.push(result); + }, + + totalSpecFailures: function() { + var specFailures = 0; + this.results_.forEach(function(result) { + specFailures += result.failedCount; + }); + return specFailures; + }, + + totalProcessFailures: function() { + var processFailures = 0; + this.results_.forEach(function(result) { + if (!result.failedCount && result.exitCode !== 0) { + processFailures += 1; + } + }); + return processFailures; + }, + + saveResults: function(filepath) { + var jsonOutput = []; + this.results_.forEach(function(result) { + jsonOutput = jsonOutput.concat(result.specResults); + }) + + var json = JSON.stringify(jsonOutput, null, ' '); + var fs = require('fs'); + fs.writeFileSync(filepath, json); + }, + + reportSummary: function() { + var specFailures = this.totalSpecFailures(); + var processFailures = this.totalProcessFailures(); + this.results_.forEach(function(result) { + var capabilities = result.capabilities; + var shortName = (capabilities.browserName) ? capabilities.browserName : ''; + shortName += (capabilities.version) ? capabilities.version : ''; + shortName += (' #' + result.taskId); + if (result.failedCount) { + log_(shortName + ' failed ' + result.failedCount + ' test(s)'); + } else if (result.exitCode !== 0) { + log_(shortName + ' failed with exit code: ' + result.exitCode); + } else { + log_(shortName + ' passed'); + } + }); + + if (specFailures && processFailures) { + log_('overall: ' + specFailures + ' failed spec(s) and ' + + processFailures + ' process(es) failed to complete'); + } else if (specFailures) { + log_('overall: ' + specFailures + ' failed spec(s)'); + } else if (processFailures) { + log_('overall: ' + processFailures + ' process(es) failed to complete'); + } + } +}; + /** * Initialize and run the tests. + * Exits with 1 on test failure, and RUNNERS_FAILED_EXIT_CODE on unexpected + * failures. * * @param {string=} configFile * @param {Object=} additionalConfig @@ -40,100 +111,15 @@ var init = function(configFile, additionalConfig) { var config = configParser.getConfig(); var scheduler = new TaskScheduler(config); - /** - * A fork of a runner for running a specified task. The RunnerFork will - * start a new process that calls on '/runFromLauncher.js' and report the - * result to a reporter. - * - * @constructor - * @param {object} task Task to run. - */ - var RunnerFork = function(task) { - this.capability = task.capability; - this.specs = task.specs; - this.process = child.fork( - __dirname + '/runFromLauncher.js', - process.argv.slice(2), { - cwd: process.cwd(), - silent: true - } - ); - this.reporter = reporter.addTaskReporter(task, this.process.pid); - }; - - /** - * Add handlers for the RunnerFork for events like stdout, stderr, testsDone, - * testPass, testFail, error, and exit. Optionally, you can pass in a - * callback function to be called when a test completes. - * - * @param {function()} testsDoneCallback Callback function for testsDone events. - */ - RunnerFork.prototype.addEventHandlers = function(testsDoneCallback) { - var self = this; - - // stdout pipe - this.process.stdout.on('data', function(chunk) { - self.reporter.logStdout(chunk); - }); - - // stderr pipe - this.process.stderr.on('data', function(chunk) { - self.reporter.logStderr(chunk); - }); - - this.process.on('message', function(m) { - switch (m.event) { - case 'testPass': - process.stdout.write('.'); - break; - case 'testFail': - process.stdout.write('F'); - break; - case 'testsDone': - self.reporter.testsDone(m.failedCount); - break; - } - }); - - this.process.on('error', function(err) { - self.reporter.flush(); - log_('Runner Process(' + self.process.pid + ') Error: ' + err); - self.reporter.exitCode = RUNNERS_FAILED_EXIT_CODE; - }); - - this.process.on('exit', function(code) { - self.reporter.flush(); - if (code) { - if (self.reporter.failedCount) { - log_('Test runner exited with ' + self.reporter.failedCount + - ' failed test(s)'); - } else { - log_('Runner process exited unexpectedly with error code: ' + code); - } - } - self.reporter.exitCode = code; - - if (typeof testsDoneCallback === 'function') { - testsDoneCallback(); - } - log_(scheduler.countActiveTasks() + - ' instance(s) of WebDriver still running'); - }); - }; - - /** - * Sends the run command. - */ - RunnerFork.prototype.run = function() { - this.process.send({ - command: 'run', - configFile: configFile, - additionalConfig: additionalConfig, - capability: this.capability, - specs: this.specs - }); - this.reporter.reportHeader_(); - }; + process.on('exit', function(code) { + if (code) { + log_('Process exited with error code ' + code); + } else if (scheduler.numTasksOutstanding() > 0) { + log_('BUG: launcher exited with ' + + scheduler.numTasksOutstanding() + ' tasks remaining'); + process.exit(RUNNERS_FAILED_EXIT_CODE); + } + }); var cleanUpAndExit = function(exitCode) { return helper.runFilenameOrFn_( @@ -151,245 +137,65 @@ var init = function(configFile, additionalConfig) { }; helper.runFilenameOrFn_(config.configDir, config.beforeLaunch).then(function() { - // Don't start new process if there is only 1 task. var totalTasks = scheduler.numTasksOutstanding(); - if (totalTasks === 1) { - var Runner = require('./runner'); - var task = scheduler.nextTask(); - config.capabilities = task.capability; - config.specs = task.specs; - - var runner = new Runner(config); - runner.run().then(function(exitCode) { - cleanUpAndExit(exitCode); - }).catch(function(err) { - log_('Error:', err.stack || err.message || err); - cleanUpAndExit(RUNNERS_FAILED_EXIT_CODE); - }); - } else { + var forkProcess = false; + if (totalTasks > 1) { // Start new processes only if there are >1 tasks. + forkProcess = true; if (config.debug) { throw new Error('Cannot run in debug mode with ' + 'multiCapabilities, count > 1, or sharding'); } - var deferred = q.defer(); - for (var i = 0; i < scheduler.maxConcurrentTasks(); ++i) { - var createNextRunnerFork = function() { - var task = scheduler.nextTask(); - if (task) { - var done = function() { - task.done(); - createNextRunnerFork(); - if (scheduler.numTasksOutstanding() === 0) { - deferred.fulfill(); - } - }; - var runnerFork = new RunnerFork(task); - runnerFork.addEventHandlers(done); - runnerFork.run(); - } - }; - createNextRunnerFork(); - } - log_('Running ' + scheduler.countActiveTasks() + ' instances of WebDriver'); - - deferred.promise.then(function() { - reporter.reportSummary(); - var exitCode = 0; - if (reporter.totalProcessFailures() > 0) { - exitCode = RUNNERS_FAILED_EXIT_CODE; - } else if (reporter.totalSpecFailures() > 0) { - exitCode = 1; - } - return cleanUpAndExit(exitCode); - }); - - process.on('exit', function(code) { - if (code) { - log_('Process exited with error code ' + code); - } else if (scheduler.numTasksOutstanding() > 0) { - code = RUNNERS_FAILED_EXIT_CODE; - log_('BUG: launcher exited with ' + - scheduler.numTasksOutstanding() + ' tasks remaining'); - } - process.exit(code); - }); } - }).done(); -}; - -//###### REPORTER #######// -/** - * Keeps track of a list of task reporters. Provides method to add a new - * reporter and to aggregate the reports into a summary. - */ -var reporter = { - taskReporters_: [], - addTaskReporter: function(task, pid) { - var taskReporter = new TaskReporter_(task, pid); - this.taskReporters_.push(taskReporter); - return taskReporter; - }, - - totalSpecFailures: function() { - var specFailures = 0; - this.taskReporters_.forEach(function(taskReporter) { - specFailures += taskReporter.failedCount; - }); - return specFailures; - }, - - totalProcessFailures: function() { - var processFailures = 0; - this.taskReporters_.forEach(function(taskReporter) { - if (!taskReporter.failedCount && taskReporter.exitCode !== 0) { - processFailures += 1; - } - }); - return processFailures; - }, - - reportSummary: function() { - var specFailures = this.totalSpecFailures(); - var processFailures = this.totalProcessFailures(); - this.taskReporters_.forEach(function(taskReporter) { - var capability = taskReporter.task.capability; - var shortName = (capability.browserName) ? capability.browserName : ''; - shortName += (capability.version) ? capability.version : ''; - shortName += (' #' + taskReporter.task.taskId); - if (taskReporter.failedCount) { - log_(shortName + ' failed ' + taskReporter.failedCount + ' test(s)'); - } else if (taskReporter.exitCode !== 0) { - log_(shortName + ' failed with exit code: ' + taskReporter.exitCode); - } else { - log_(shortName + ' passed'); - } - }); - if (this.taskReporters_.length > 1) { - if (specFailures && processFailures) { - log_('overall: ' + specFailures + ' failed spec(s) and ' + - processFailures + ' process(es) failed to complete'); - } else if (specFailures) { - log_('overall: ' + specFailures + ' failed spec(s)'); - } else if (processFailures) { - log_('overall: ' + processFailures + ' process(es) failed to complete'); + var deferred = q.defer(); // Resolved when all tasks are completed + var createNextTaskRunner = function() { + var task = scheduler.nextTask(); + if (task) { + var taskRunner = new TaskRunner(configFile, additionalConfig, task, forkProcess); + taskRunner.run().then(function(result) { + if (result.exitCode && !result.failedCount) { + log_('Runner process exited unexpectedly with error code: ' + result.exitCode); + } + taskResults_.add(result); + task.done(); + createNextTaskRunner(); + // If all tasks are finished + if (scheduler.numTasksOutstanding() === 0) { + deferred.fulfill(); + } + log_(scheduler.countActiveTasks() + + ' instance(s) of WebDriver still running'); + }).catch(function(err) { + log_('Error:', err.stack || err.message || err); + cleanUpAndExit(RUNNERS_FAILED_EXIT_CODE); + }); } + }; + // Start `scheduler.maxConcurrentTasks()` workers for handling tasks in + // the beginning. As a worker finishes a task, it will pick up the next task + // from the scheduler's queue until all tasks are gone. + for (var i = 0; i < scheduler.maxConcurrentTasks(); ++i) { + createNextTaskRunner(); } - } -}; - -/** - * A reporter for a specific task. - * - * @constructor - * @param {object} task Task that is being reported. - * @param {number} pid PID of process running the task. - */ -var TaskReporter_ = function(task, pid) { - this.task = task; - this.pid = pid; - this.failedCount = 0; - this.buffer = ''; - this.exitCode = -1; - this.insertTag = true; -}; + log_('Running ' + scheduler.countActiveTasks() + ' instances of WebDriver'); -/** - * Report the header for the current task including information such as - * PID, browser name/version, task Id, specs being run. - */ -TaskReporter_.prototype.reportHeader_ = function() { - var eol = require('os').EOL; - var output = 'PID: ' + this.pid + eol; - if (this.task.specs.length === 1) { - output += 'Specs: '+ this.task.specs.toString() + eol + eol; - } - this.log_(output); -}; - -/** - * Log the stdout. The reporter is responsible for reporting this data when - * appropriate. - * - * @param {string} stdout Stdout data to log - */ -TaskReporter_.prototype.logStdout = function(stdout) { - this.log_(stdout); -}; - -/** - * Log the stderr. The reporter is responsible for reporting this data when - * appropriate. - * - * @param {string} stderr Stderr data to log - */ -TaskReporter_.prototype.logStderr = function(stderr) { - this.log_(stderr); -}; - -/** - * Signal that the task is completed. This must be called at the end of a task. - * - * @param {number} failedCount Number of failures - */ -TaskReporter_.prototype.testsDone = function(failedCount) { - this.failedCount = failedCount; - this.flush(); -}; - -/** - * Flushes the buffer to stdout. - */ -TaskReporter_.prototype.flush = function() { - if (this.buffer) { - // Flush buffer if nonempty - var eol = require('os').EOL; - process.stdout.write(eol + '------------------------------------' + eol); - process.stdout.write(this.buffer); - process.stdout.write(eol); - this.buffer = ''; - } -}; - -/** - * Report the following data. The data will be saved to a buffer - * until it is flushed by the function testsDone. - * - * @private - * @param {string} data - */ -TaskReporter_.prototype.log_ = function(data) { - var tag = '['; - var capability = this.task.capability; - tag += (capability.browserName) ? - capability.browserName : ''; - tag += (capability.version) ? - (' ' + capability.version) : ''; - tag += (capability.platform) ? - (' ' + capability.platform) : ''; - tag += (' #' + this.task.taskId); - tag += '] '; + // By now all runners have completed. + deferred.promise.then(function() { + // Save results if desired + if (config.resultJsonOutputFile) { + taskResults_.saveResults(config.resultJsonOutputFile); + } - data = data.toString(); - for ( var i = 0; i < data.length; i++ ) { - if (this.insertTag) { - this.insertTag = false; - // This ensures that the '\x1B[0m' appears before the tag, so that - // data remains correct when color is not processed. - // See https://github.com/angular/protractor/pull/1216 - if (data[i] === '\x1B' && data.substring(i, i+4) === '\x1B[0m' ) { - this.buffer += ('\x1B[0m' + tag); - i += 3; - continue; + taskResults_.reportSummary(); + var exitCode = 0; + if (taskResults_.totalProcessFailures() > 0) { + exitCode = RUNNERS_FAILED_EXIT_CODE; + } else if (taskResults_.totalSpecFailures() > 0) { + exitCode = 1; } - - this.buffer += tag; - } - if (data[i] === '\n') { - this.insertTag = true; - } - this.buffer += data[i]; - } + return cleanUpAndExit(exitCode); + }).done(); + }).done(); }; exports.init = init; diff --git a/lib/runner.js b/lib/runner.js index d17c9fc76..e172c14ec 100644 --- a/lib/runner.js +++ b/lib/runner.js @@ -191,7 +191,7 @@ Runner.prototype.run = function() { var self = this, driver, specs, - testResult; + testPassed; specs = this.config_.specs; @@ -247,11 +247,11 @@ Runner.prototype.run = function() { return require(frameworkPath).run(self, specs); // 4) Teardown }).then(function(result) { - self.emit('testsDone', result.failedCount); - testResult = result; + self.emit('testsDone', result); + testPassed = result.failedCount === 0; if (self.driverprovider_.updateJob) { return self.driverprovider_.updateJob({ - 'passed': testResult.failedCount === 0 + 'passed': testPassed }).then(function() { return self.driverprovider_.teardownEnv(); }); @@ -260,8 +260,7 @@ Runner.prototype.run = function() { } // 5) Exit process }).then(function() { - var passed = testResult.failedCount === 0; - var exitCode = passed ? 0 : 1; + var exitCode = testPassed ? 0 : 1; return self.exit_(exitCode); }).fin(function() { return gracefulShutdown(driver); diff --git a/lib/runFromLauncher.js b/lib/runnerCli.js similarity index 83% rename from lib/runFromLauncher.js rename to lib/runnerCli.js index 9eff8dbbb..3dd64d2d6 100644 --- a/lib/runFromLauncher.js +++ b/lib/runnerCli.js @@ -9,8 +9,8 @@ var Runner = require('./runner'); process.on('message', function(m) { switch (m.command) { case 'run': - if (!m.capability) { - throw new Error('Run message missing capability'); + if (!m.capabilities) { + throw new Error('Run message missing capabilities'); } // Merge in config file options. var configParser = new ConfigParser(); @@ -22,8 +22,8 @@ process.on('message', function(m) { } var config = configParser.getConfig(); - // Grab capability to run from launcher. - config.capabilities = m.capability; + // Grab capabilities to run from launcher. + config.capabilities = m.capabilities; //Get specs to be executed by this runner config.specs = m.specs; @@ -42,10 +42,10 @@ process.on('message', function(m) { event: 'testFail' }); }); - runner.on('testsDone', function(failedCount) { + runner.on('testsDone', function(results) { process.send({ event: 'testsDone', - failedCount: failedCount + results: results }); }); diff --git a/lib/taskLogger.js b/lib/taskLogger.js new file mode 100644 index 000000000..0cf952711 --- /dev/null +++ b/lib/taskLogger.js @@ -0,0 +1,88 @@ +var EOL = require('os').EOL; + +/** + * Log output such that metadata are appended. + * Calling log(data) will not flush to console until you call flush() + * + * @constructor + * @param {object} task Task that is being reported. + * @param {number} pid PID of process running the task. + */ +var TaskLogger = function(task, pid) { + this.task = task; + this.pid = pid; + this.buffer = ''; + this.insertTag = true; + + this.logHeader_(); +}; + +/** + * Log the header for the current task including information such as + * PID, browser name/version, task Id, specs being run. + * + * @private + */ +TaskLogger.prototype.logHeader_ = function() { + var output = 'PID: ' + this.pid + EOL; + if (this.task.specs.length === 1) { + output += 'Specs: '+ this.task.specs.toString() + EOL + EOL; + } + this.log(output); +}; + + +/** + * Flushes the buffer to stdout. + */ +TaskLogger.prototype.flush = function() { + if (this.buffer) { + // Flush buffer if nonempty + process.stdout.write(EOL + '------------------------------------' + EOL); + process.stdout.write(this.buffer); + process.stdout.write(EOL); + this.buffer = ''; + } +}; + +/** + * Log the data in the argument such that metadata are appended. + * The data will be saved to a buffer until flush() is called. + * + * @param {string} data + */ +TaskLogger.prototype.log = function(data) { + var tag = '['; + var capabilities = this.task.capabilities; + tag += (capabilities.browserName) ? + capabilities.browserName : ''; + tag += (capabilities.version) ? + (' ' + capabilities.version) : ''; + tag += (capabilities.platform) ? + (' ' + capabilities.platform) : ''; + tag += (' #' + this.task.taskId); + tag += '] '; + + data = data.toString(); + for ( var i = 0; i < data.length; i++ ) { + if (this.insertTag) { + this.insertTag = false; + // This ensures that the '\x1B[0m' appears before the tag, so that + // data remains correct when color is not processed. + // See https://github.com/angular/protractor/pull/1216 + if (data[i] === '\x1B' && data.substring(i, i+4) === '\x1B[0m' ) { + this.buffer += ('\x1B[0m' + tag); + i += 3; + continue; + } + + this.buffer += tag; + } + if (data[i] === '\n') { + this.insertTag = true; + } + this.buffer += data[i]; + } +}; + +module.exports = TaskLogger; diff --git a/lib/taskRunner.js b/lib/taskRunner.js new file mode 100644 index 000000000..2b675aafa --- /dev/null +++ b/lib/taskRunner.js @@ -0,0 +1,131 @@ +var child = require('child_process'); +var q = require('q'); +var TaskLogger = require('./taskLogger.js'); +var EventEmitter = require('events').EventEmitter; +var util = require('util'); + + +/** + * A runner for running a specified task (capabilities + specs). + * The TaskRunner can either run the task from the current process (via + * './runner.js') or from a new process (via './runnerCli.js'). + * + * @constructor + * @param {string} configFile Path of test configuration. + * @param {object} additionalConfig Additional configuration. + * @param {object} task Task to run. + * @param {boolean} runInFork Whether to run test in a forked process. + * @constructor + */ +var TaskRunner = function(configFile, additionalConfig, task, runInFork) { + this.configFile = configFile; + this.additionalConfig = additionalConfig; + this.task = task; + this.runInFork = runInFork; +}; +util.inherits(TaskRunner, EventEmitter); + +/** + * Sends the run command. + * @return {q.Promise} A promise that will resolve when the task finishes + * running. The promise contains the following parameters representing the + * result of the run: + * taskId, specs, capabilities, failedCount, exitCode, specResults + */ +TaskRunner.prototype.run = function() { + var self = this; + + var runResults = { + taskId: this.task.taskId, + specs: this.task.specs, + capabilities: this.task.capabilities, + // The following are populated while running the test: + failedCount: 0, + exitCode: -1, + specResults: [] + } + + if (this.runInFork) { + var deferred = q.defer(); + + var childProcess = child.fork( + __dirname + '/runnerCli.js', + process.argv.slice(2), { + cwd: process.cwd(), + silent: true + } + ); + var taskLogger = new TaskLogger(this.task, childProcess.pid); + + // stdout pipe + childProcess.stdout.on('data', function(data) { + taskLogger.log(data); + }); + + // stderr pipe + childProcess.stderr.on('data', function(data) { + taskLogger.log(data); + }); + + childProcess.on('message', function(m) { + switch (m.event) { + case 'testPass': + process.stdout.write('.'); + break; + case 'testFail': + process.stdout.write('F'); + break; + case 'testsDone': + runResults.failedCount = m.results.failedCount; + runResults.specResults = m.results.specResults; + break; + } + }) + .on('error', function(err) { + taskLogger.flush(); + deferred.reject(err); + }) + .on('exit', function(code) { + taskLogger.flush(); + runResults.exitCode = code; + deferred.resolve(runResults); + }); + + childProcess.send({ + command: 'run', + configFile: this.configFile, + additionalConfig: this.additionalConfig, + capabilities: this.task.capabilities, + specs: this.task.specs + }); + + return deferred.promise; + } else { + var ConfigParser = require('./configParser'); + var configParser = new ConfigParser(); + if (this.configFile) { + configParser.addFileConfig(this.configFile); + } + if (this.additionalConfig) { + configParser.addConfig(this.additionalConfig); + } + var config = configParser.getConfig(); + config.capabilities = this.task.capabilities; + config.specs = this.task.specs; + + var Runner = require('./runner'); + var runner = new Runner(config); + + runner.on('testsDone', function(results) { + runResults.failedCount = results.failedCount; + runResults.specResults = results.specResults; + }) + + return runner.run().then(function(exitCode) { + runResults.exitCode = exitCode; + return runResults; + }); + } +}; + +module.exports = TaskRunner; diff --git a/lib/taskScheduler.js b/lib/taskScheduler.js index 205ab8273..656e74440 100644 --- a/lib/taskScheduler.js +++ b/lib/taskScheduler.js @@ -7,17 +7,17 @@ var ConfigParser = require('./configParser'); // A queue of specs for a particular capacity -var TaskQueue = function(capability, specLists) { - this.capability = capability; +var TaskQueue = function(capabilities, specLists) { + this.capabilities = capabilities; this.numRunningInstances = 0; - this.maxInstance = capability.maxInstances || 1; + this.maxInstance = capabilities.maxInstances || 1; this.specsIndex = 0; this.specLists = specLists; }; /** * A scheduler to keep track of specs that need running and their associated - * capability. It will suggest a task (combination of capability and spec) + * capabilities. It will suggest a task (combination of capabilities and spec) * to run while observing the following config rules: capabilities, * multiCapabilities, shardTestFiles, and maxInstance. * @@ -40,27 +40,27 @@ var TaskScheduler = function(config) { config.multiCapabilities = [config.capabilities]; } } else if (!config.multiCapabilities.length) { - // Default to chrome if no capability given + // Default to chrome if no capabilities given config.multiCapabilities = [{ browserName: 'chrome' }]; } var taskQueues = []; - config.multiCapabilities.forEach(function(capability) { - var capabilitySpecs = allSpecs; - if (capability.specs) { - var capabilitySpecificSpecs = ConfigParser.resolveFilePatterns( - capability.specs, false, config.configDir); - capabilitySpecs = capabilitySpecs.concat(capabilitySpecificSpecs); + config.multiCapabilities.forEach(function(capabilities) { + var capabilitiesSpecs = allSpecs; + if (capabilities.specs) { + var capabilitiesSpecificSpecs = ConfigParser.resolveFilePatterns( + capabilities.specs, false, config.configDir); + capabilitiesSpecs = capabilitiesSpecs.concat(capabilitiesSpecificSpecs); } - if (capability.exclude) { - var capabilitySpecExcludes = ConfigParser.resolveFilePatterns( - capability.exclude, true, config.configDir); - capabilitySpecs = ConfigParser.resolveFilePatterns( - capabilitySpecs).filter(function(path) { - return capabilitySpecExcludes.indexOf(path) < 0; + if (capabilities.exclude) { + var capabilitiesSpecExcludes = ConfigParser.resolveFilePatterns( + capabilities.exclude, true, config.configDir); + capabilitiesSpecs = ConfigParser.resolveFilePatterns( + capabilitiesSpecs).filter(function(path) { + return capabilitiesSpecExcludes.indexOf(path) < 0; }); } @@ -68,18 +68,18 @@ var TaskScheduler = function(config) { // If we shard, we return an array of one element arrays, each containing // the spec file. If we don't shard, we return an one element array // containing an array of all the spec files - if (capability.shardTestFiles) { - capabilitySpecs.forEach(function(spec) { + if (capabilities.shardTestFiles) { + capabilitiesSpecs.forEach(function(spec) { specLists.push([spec]); }); } else { - specLists.push(capabilitySpecs); + specLists.push(capabilitiesSpecs); } - capability.count = capability.count || 1; + capabilities.count = capabilities.count || 1; - for (var i = 0; i < capability.count; ++i) { - taskQueues.push(new TaskQueue(capability, specLists)); + for (var i = 0; i < capabilities.count; ++i) { + taskQueues.push(new TaskQueue(capabilities, specLists)); } }); this.taskQueues = taskQueues; @@ -90,7 +90,7 @@ var TaskScheduler = function(config) { /** * Get the next task that is allowed to run without going over maxInstance. * - * @return {{capability: Object, specs: Array., taskId: string, done: function()}} + * @return {{capabilities: Object, specs: Array., taskId: string, done: function()}} */ TaskScheduler.prototype.nextTask = function() { for (var i = 0; i < this.taskQueues.length; ++i) { @@ -108,7 +108,7 @@ TaskScheduler.prototype.nextTask = function() { ++queue.specsIndex; return { - capability: queue.capability, + capabilities: queue.capabilities, specs: specs, taskId: taskId, done: function() {