From 6ac5867dc4806c1cd4b8aacaa5662371e0f20e56 Mon Sep 17 00:00:00 2001 From: Hank Duan Date: Thu, 16 Oct 2014 11:01:02 -0700 Subject: [PATCH] feat(scripts): add framework to test protractor --- package.json | 2 +- scripts/test.js | 79 ++++-- scripts/test/test_util.js | 229 ++++++++++++++++++ .../afterLaunchChangesExitCodeConf.js | 25 ++ spec/errorTest/baseCase/error_spec.js | 6 + spec/errorTest/baseCase/mocha_failure_spec.js | 13 + spec/errorTest/baseCase/multi_failure_spec.js | 13 + .../baseCase/single_failure_spec1.js | 7 + .../baseCase/single_failure_spec2.js | 7 + spec/errorTest/baseCase/success_spec.js | 7 + spec/errorTest/baseCase/timeout_spec.js | 5 + spec/errorTest/mochaFailureConf.js | 21 ++ spec/errorTest/multiFailureConf.js | 22 ++ spec/errorTest/shardedFailureConf.js | 24 ++ spec/errorTest/singleFailureConf.js | 21 ++ spec/errorTest/timeoutConf.js | 21 ++ 16 files changed, 481 insertions(+), 21 deletions(-) create mode 100644 scripts/test/test_util.js create mode 100644 spec/errorTest/afterLaunchChangesExitCodeConf.js create mode 100644 spec/errorTest/baseCase/error_spec.js create mode 100644 spec/errorTest/baseCase/mocha_failure_spec.js create mode 100644 spec/errorTest/baseCase/multi_failure_spec.js create mode 100644 spec/errorTest/baseCase/single_failure_spec1.js create mode 100644 spec/errorTest/baseCase/single_failure_spec2.js create mode 100644 spec/errorTest/baseCase/success_spec.js create mode 100644 spec/errorTest/baseCase/timeout_spec.js create mode 100644 spec/errorTest/mochaFailureConf.js create mode 100644 spec/errorTest/multiFailureConf.js create mode 100644 spec/errorTest/shardedFailureConf.js create mode 100644 spec/errorTest/singleFailureConf.js create mode 100644 spec/errorTest/timeoutConf.js diff --git a/package.json b/package.json index 25adfa802..decccff16 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,7 @@ }, "main": "lib/protractor.js", "scripts": { - "pretest": "node_modules/.bin/jshint lib spec", + "pretest": "node_modules/.bin/jshint lib spec scripts", "test": "scripts/test.js", "start": "testapp/scripts/web-server.js" }, diff --git a/scripts/test.js b/scripts/test.js index 7e6d4cb5a..4c8d18c11 100755 --- a/scripts/test.js +++ b/scripts/test.js @@ -1,9 +1,9 @@ #!/usr/bin/env node +var Executor = require('./test/test_util').Executor; var glob = require('glob').sync; -var spawn = require('child_process').spawn; -var scripts = [ +var passing_tests = [ 'node lib/cli.js spec/basicConf.js', 'node lib/cli.js spec/multiConf.js', 'node lib/cli.js spec/altRootConf.js', @@ -22,29 +22,68 @@ var scripts = [ 'node lib/cli.js spec/suitesConf.js --suite okmany,okspec' ]; -scripts.push( +passing_tests.push( 'node node_modules/.bin/minijasminenode ' + glob('spec/unit/*.js').join(' ') + ' ' + glob('docgen/spec/*.js').join(' ')); -var failed = false; +var executor = new Executor(); -(function runTests(i) { - if (i < scripts.length) { - console.log('node ' + scripts[i]); - var args = scripts[i].split(/\s/); +passing_tests.forEach(function(passing_test) { + executor.addCommandlineTest(passing_test) + .assertExitCodeOnly(); +}); - var test = spawn(args[0], args.slice(1), {stdio: 'inherit'}); - test.on('error', function(err) { - throw err; +/************************* + *Below are failure tests* + *************************/ + +// assert stacktrace shows line of failure +executor.addCommandlineTest('node lib/cli.js spec/errorTest/singleFailureConf.js') + .expectExitCode(1) + .expectErrors({ + stackTrace: 'single_failure_spec1.js:5:32' }); - test.on('exit', function(code) { - if (code != 0) { - failed = true; - } - runTests(i + 1); + +// assert timeout works +executor.addCommandlineTest('node lib/cli.js spec/errorTest/timeoutConf.js') + .expectExitCode(1) + .expectErrors({ + message: 'timeout: timed out after 1 msec waiting for spec to complete' + }) + .expectTestDuration(0, 100); + +executor.addCommandlineTest('node lib/cli.js spec/errorTest/afterLaunchChangesExitCodeConf.js') + .expectExitCode(11) + .expectErrors({ + message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.' }); - } else { - process.exit(failed ? 1 : 0); - } -}(0)); + +executor.addCommandlineTest('node lib/cli.js spec/errorTest/multiFailureConf.js') + .expectExitCode(1) + .expectErrors([{ + message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.', + stacktrace: 'single_failure_spec1.js:5:32' + }, { + message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.', + stacktrace: 'single_failure_spec2.js:5:32' + }]); + +executor.addCommandlineTest('node lib/cli.js spec/errorTest/shardedFailureConf.js') + .expectExitCode(1) + .expectErrors([{ + message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.', + stacktrace: 'single_failure_spec1.js:5:32' + }, { + message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.', + stacktrace: 'single_failure_spec2.js:5:32' + }]); + +executor.addCommandlineTest('node lib/cli.js spec/errorTest/mochaFailureConf.js') + .expectExitCode(1) + .expectErrors([{ + message: 'expected \'My AngularJS App\' to equal \'INTENTIONALLY INCORRECT\'', + stacktrace: 'mocha_failure_spec.js:11:20' + }]); + +executor.execute(); diff --git a/scripts/test/test_util.js b/scripts/test/test_util.js new file mode 100644 index 000000000..3a1c99835 --- /dev/null +++ b/scripts/test/test_util.js @@ -0,0 +1,229 @@ +#!/usr/bin/env node + +var child_process = require('child_process'), + q = require('q'), + fs = require('fs'); + +// Remove warning message about too many listeners. +// By virtue of creating so many tests, this will happen. +process.setMaxListeners(0); + +var CommandlineTest = function(command) { + var self = this; + this.command_ = command; + this.expectedExitCode_ = 0; + this.stdioOnlyOnFailures_ = true; + this.expectedErrors_ = []; + this.assertExitCodeOnly_ = false; + + // If stdioOnlyOnFailures_ is true, do not stream stdio unless test failed. + // This is to prevent tests with expected failures from polluting the output. + this.alwaysEnableStdio = function() { + self.stdioOnlyOnFailures_ = false; + return self; + }; + + // Only assert the exit code and not failures. + // This must be true if the command you're running does not support + // the flag '--testResultOutput'. + this.assertExitCodeOnly = function() { + self.assertExitCodeOnly_ = true; + return self; + }; + + // Set the expected exit code for the test command. + this.expectExitCode = function(exitCode) { + self.expectedExitCode_ = exitCode; + return self; + }; + + // Set the expected total test duration. + this.expectTestDuration = function(min, max) { + self.expectedMinTestDuration_ = min; + self.expectedMaxTestDuration_ = max; + return self; + }; + + /** + * Add expected error(s) for the test command. + * Input is an object or list of objects of the following form: + * { + * message: string, // optional regex + * stackTrace: string, //optional regex + * } + */ + this.expectErrors = function(expectedErrors) { + if (expectedErrors instanceof Array) { + self.expectedErrors_ = self.expectedErrors_.concat(expectedErrors); + } else { + self.expectedErrors_.push(expectedErrors); + } + return self; + }; + + this.run = function() { + var self = this; + var start = new Date().getTime(); + var testOutputPath = 'test_output_' + start + '.tmp'; + var output = ''; + + var flushAndFail = function(errorMsg) { + process.stdout.write(output); + throw new Error(errorMsg); + }; + + // Clean up file + process.on('exit', function() { + try { + fs.unlinkSync(testOutputPath); + } catch (err) { + // don't do anything + } + }); + + return q.promise(function (resolve, reject) { + if (!self.assertExitCodeOnly_) { + self.command_ = self.command_ + ' --testResultOutput ' + testOutputPath; + } + var args = self.command_.split(/\s/); + + var test_process = child_process.spawn(args[0], args.slice(1)); + + test_process.stdout.on('data', function (data) { + if (self.stdioOnlyOnFailures_) { + output += data; + } else { + process.stdout.write(data.toString()); + } + }); + + test_process.stderr.on('data', function (data) { + if (self.stdioOnlyOnFailures_) { + output += data; + } else { + process.stdout.write(data.toString()); + } + }); + + test_process.on('error', function(err) { + reject(err); + }); + + test_process.on('exit', function(exitCode) { + resolve(exitCode); + }); + }).then(function(exitCode) { + if (self.expectedExitCode_ !== exitCode) { + flushAndFail('expecting exit code: ' + self.expectedExitCode_ + + ', actual: ' + exitCode); + } + + // Skip the rest if we are only verify exit code. + // Note: we're expecting a file populated by '--testResultOutput' after + // this point. + if (self.assertExitCodeOnly_) { + return; + } + + var raw_data = fs.readFileSync(testOutputPath); + fs.unlinkSync(testOutputPath); + var testOutput = JSON.parse(raw_data); + + var actualErrors = []; + var duration = 0; + testOutput.forEach(function(fileResult) { + duration += fileResult.duration; + fileResult.result.forEach(function(specResult) { + if (!specResult.passed) { + actualErrors.push(specResult); + } + }); + }); + + self.expectedErrors_.forEach(function(expectedError) { + var found = false; + var i = 0; + for ( ; i < actualErrors.length; ++i) { + var actualError = actualErrors[i]; + + // if expected message is defined and messages don't match + if (expectedError.message) { + if (!actualError.errorMsg || + !actualError.errorMsg.match(new RegExp(expectedError.message))) { + continue; + } + } + // if expected stacktrace is defined and stacktraces don't match + if (expectedError.stacktrace) { + if (!actualError.stacktrace || + !actualError.stacktrace.match(new RegExp(expectedError.stacktrace))) { + continue; + } + } + found = true; + break; + } + + if (!found) { + flushAndFail('did not fail with expected error with message: [' + + expectedError.message + '] and stacktrace: [' + + expectedError.stacktrace + ']'); + } else { + actualErrors.splice(i, 1); + } + }); + + if (actualErrors.length > 0) { + flushAndFail('failed with ' + actualErrors.length + ' unexpected failures'); + } + + if (self.expectedMinTestDuration_ + && duration < self.expectedMinTestDuration_) { + flushAndFail('expecting test min duration: ' + + self.expectedMinTestDuration_ + ', actual: ' + duration); + } + if (self.expectedMaxTestDuration_ + && duration > self.expectedMaxTestDuration_) { + flushAndFail('expecting test max duration: ' + + self.expectedMaxTestDuration_ + ', actual: ' + duration); + } + }); + }; +}; + +/** + * Util for running tests and testing functionalities including: + * exitCode, test durations, expected errors, and expected stacktrace + * Note, this will work with any commandline tests, but only if it supports + * the flag '--testResultOutput', unless only exitCode is being tested. + * For now, this means only protractor-jasmine. + */ +exports.Executor = function() { + var tests = []; + this.addCommandlineTest = function(command) { + var test = new CommandlineTest(command); + tests.push(test); + return test; + }; + + this.execute = function() { + var failed = false; + + (function runTests(i) { + if (i < tests.length) { + console.log('running: ' + tests[i].command_); + tests[i].run().then(function() { + console.log('>>> \033[1;32mpass\033[0m'); + }, function(err) { + failed = true; + console.log('>>> \033[1;31mfail: ' + err.toString() + '\033[0m'); + }).fin(function() { + runTests(i + 1); + }).done(); + } else { + console.log('Summary: ' + (failed ? 'fail' : 'pass')); + process.exit(failed ? 1 : 0); + } + }(0)); + }; +}; diff --git a/spec/errorTest/afterLaunchChangesExitCodeConf.js b/spec/errorTest/afterLaunchChangesExitCodeConf.js new file mode 100644 index 000000000..d641963f1 --- /dev/null +++ b/spec/errorTest/afterLaunchChangesExitCodeConf.js @@ -0,0 +1,25 @@ +var env = require('../environment.js'); + +// The main suite of Protractor tests to be run on CI servers. +exports.config = { + specs: [ + 'baseCase/single_failure_spec1.js' + ], + + multiCapabilities: [{ + 'browserName': 'chrome' + }], + + baseUrl: env.baseUrl, + + jasmineNodeOpts: { + isVerbose: true, + showTiming: true, + defaultTimeoutInterval: 90000 + }, + + afterLaunch: function(exitCode) { + return exitCode + 10; + }, + +}; diff --git a/spec/errorTest/baseCase/error_spec.js b/spec/errorTest/baseCase/error_spec.js new file mode 100644 index 000000000..0d8eebc27 --- /dev/null +++ b/spec/errorTest/baseCase/error_spec.js @@ -0,0 +1,6 @@ +describe('error spec', function() { + it('should throw an error', function() { + browser.get('index.html'); + var greeting = element(by.binding('INVALID')); + }); +}); diff --git a/spec/errorTest/baseCase/mocha_failure_spec.js b/spec/errorTest/baseCase/mocha_failure_spec.js new file mode 100644 index 000000000..b610e9e34 --- /dev/null +++ b/spec/errorTest/baseCase/mocha_failure_spec.js @@ -0,0 +1,13 @@ +// Use the external Chai As Promised to deal with resolving promises in +// expectations. +var chai = require('chai'); +var chaiAsPromised = require('chai-as-promised'); +chai.use(chaiAsPromised); +var expect = chai.expect; + +describe('protractor library', function() { + it('should wrap webdriver', function() { + browser.get('index.html'); + expect(browser.getTitle()).to.eventually.equal('INTENTIONALLY INCORRECT'); + }); +}); diff --git a/spec/errorTest/baseCase/multi_failure_spec.js b/spec/errorTest/baseCase/multi_failure_spec.js new file mode 100644 index 000000000..c32f3ae67 --- /dev/null +++ b/spec/errorTest/baseCase/multi_failure_spec.js @@ -0,0 +1,13 @@ +describe('multi failure spec', function() { + it('should fail expectation', function() { + browser.get('index.html'); + var greeting = element(by.binding('greeting')); + expect(greeting.getText()).toEqual('INTENTIONALLY INCORRECT'); + }); + + it('should fail expectation again', function() { + browser.get('index.html'); + var greeting = element(by.binding('greeting')); + expect(greeting.getText()).toEqual('INTENTIONALLY INCORRECT AGAIN'); + }); +}); diff --git a/spec/errorTest/baseCase/single_failure_spec1.js b/spec/errorTest/baseCase/single_failure_spec1.js new file mode 100644 index 000000000..c94068415 --- /dev/null +++ b/spec/errorTest/baseCase/single_failure_spec1.js @@ -0,0 +1,7 @@ +describe('single failure spec1', function() { + it('should fail expectation', function() { + browser.get('index.html'); + var greeting = element(by.binding('greeting')); + expect(greeting.getText()).toEqual('INTENTIONALLY INCORRECT'); + }); +}); diff --git a/spec/errorTest/baseCase/single_failure_spec2.js b/spec/errorTest/baseCase/single_failure_spec2.js new file mode 100644 index 000000000..c39404110 --- /dev/null +++ b/spec/errorTest/baseCase/single_failure_spec2.js @@ -0,0 +1,7 @@ +describe('single failure spec2', function() { + it('should fail expectation', function() { + browser.get('index.html'); + var greeting = element(by.binding('greeting')); + expect(greeting.getText()).toEqual('INTENTIONALLY INCORRECT'); + }); +}); diff --git a/spec/errorTest/baseCase/success_spec.js b/spec/errorTest/baseCase/success_spec.js new file mode 100644 index 000000000..e39caa3d6 --- /dev/null +++ b/spec/errorTest/baseCase/success_spec.js @@ -0,0 +1,7 @@ +describe('success spec', function() { + it('should pass', function() { + browser.get('index.html'); + var greeting = element(by.binding('greeting')); + expect(greeting.getText()).toEqual('Hiya'); + }); +}); diff --git a/spec/errorTest/baseCase/timeout_spec.js b/spec/errorTest/baseCase/timeout_spec.js new file mode 100644 index 000000000..e0740e793 --- /dev/null +++ b/spec/errorTest/baseCase/timeout_spec.js @@ -0,0 +1,5 @@ +describe('timeout spec', function() { + it('should timeout', function() { + browser.get('index.html#/form'); + }, 1); +}); diff --git a/spec/errorTest/mochaFailureConf.js b/spec/errorTest/mochaFailureConf.js new file mode 100644 index 000000000..286249d3f --- /dev/null +++ b/spec/errorTest/mochaFailureConf.js @@ -0,0 +1,21 @@ +var env = require('../environment.js'); + +// A small suite to make sure the mocha framework works. +exports.config = { + specs: [ + 'baseCase/mocha_failure_spec.js' + ], + + multiCapabilities: [{ + 'browserName': 'chrome' + }], + + baseUrl: env.baseUrl, + + mochaOpts: { + reporter: 'spec', + timeout: 4000 + }, + + framework: 'mocha', +}; diff --git a/spec/errorTest/multiFailureConf.js b/spec/errorTest/multiFailureConf.js new file mode 100644 index 000000000..67990f69d --- /dev/null +++ b/spec/errorTest/multiFailureConf.js @@ -0,0 +1,22 @@ +var env = require('../environment.js'); + +// The main suite of Protractor tests to be run on CI servers. +exports.config = { + specs: [ + 'baseCase/single_failure_spec1.js', + 'baseCase/single_failure_spec2.js' + ], + + multiCapabilities: [{ + 'browserName': 'chrome' + }], + + baseUrl: env.baseUrl, + + jasmineNodeOpts: { + isVerbose: true, + showTiming: true, + defaultTimeoutInterval: 90000 + }, + +}; diff --git a/spec/errorTest/shardedFailureConf.js b/spec/errorTest/shardedFailureConf.js new file mode 100644 index 000000000..40654d45a --- /dev/null +++ b/spec/errorTest/shardedFailureConf.js @@ -0,0 +1,24 @@ +var env = require('../environment.js'); + +// The main suite of Protractor tests to be run on CI servers. +exports.config = { + specs: [ + 'baseCase/single_failure_spec1.js', + 'baseCase/single_failure_spec2.js' + ], + + multiCapabilities: [{ + 'browserName': 'chrome', + maxInstances: 2, + shardTestFiles: true + }], + + baseUrl: env.baseUrl, + + jasmineNodeOpts: { + isVerbose: true, + showTiming: true, + defaultTimeoutInterval: 90000 + }, + +}; diff --git a/spec/errorTest/singleFailureConf.js b/spec/errorTest/singleFailureConf.js new file mode 100644 index 000000000..2dc857f0e --- /dev/null +++ b/spec/errorTest/singleFailureConf.js @@ -0,0 +1,21 @@ +var env = require('../environment.js'); + +// The main suite of Protractor tests to be run on CI servers. +exports.config = { + specs: [ + 'baseCase/single_failure_spec1.js' + ], + + multiCapabilities: [{ + 'browserName': 'chrome' + }], + + baseUrl: env.baseUrl, + + jasmineNodeOpts: { + isVerbose: true, + showTiming: true, + defaultTimeoutInterval: 90000 + }, + +}; diff --git a/spec/errorTest/timeoutConf.js b/spec/errorTest/timeoutConf.js new file mode 100644 index 000000000..9ecc35a96 --- /dev/null +++ b/spec/errorTest/timeoutConf.js @@ -0,0 +1,21 @@ +var env = require('../environment.js'); + +// The main suite of Protractor tests to be run on CI servers. +exports.config = { + specs: [ + 'baseCase/timeout_spec.js' + ], + + multiCapabilities: [{ + 'browserName': 'chrome' + }], + + baseUrl: env.baseUrl, + + jasmineNodeOpts: { + isVerbose: true, + showTiming: true, + defaultTimeoutInterval: 90000 + }, + +};