feat(scripts): add framework to test protractor

package.json

@@ -46,7 +46,7 @@
   },
   "main": "lib/protractor.js",
   "scripts": {
-    "pretest": "node_modules/.bin/jshint lib spec",
+    "pretest": "node_modules/.bin/jshint lib spec scripts",
     "test": "scripts/test.js",
     "start": "testapp/scripts/web-server.js"
   },

scripts/test.js

@@ -1,9 +1,9 @@
 #!/usr/bin/env node
 
+var Executor = require('./test/test_util').Executor;
 var glob = require('glob').sync;
-var spawn = require('child_process').spawn;
 
-var scripts = [
+var passing_tests = [
   'node lib/cli.js spec/basicConf.js',
   'node lib/cli.js spec/multiConf.js',
   'node lib/cli.js spec/altRootConf.js',
@@ -22,29 +22,68 @@ var scripts = [
   'node lib/cli.js spec/suitesConf.js --suite okmany,okspec'
 ];
 
-scripts.push(
+passing_tests.push(
     'node node_modules/.bin/minijasminenode ' +
     glob('spec/unit/*.js').join(' ') + ' ' +
     glob('docgen/spec/*.js').join(' '));
 
-var failed = false;
+var executor = new Executor();
 
-(function runTests(i) {
-  if (i < scripts.length) {
-    console.log('node ' + scripts[i]);
-    var args = scripts[i].split(/\s/);
+passing_tests.forEach(function(passing_test) {
+  executor.addCommandlineTest(passing_test)
+      .assertExitCodeOnly();
+});
 
-    var test = spawn(args[0], args.slice(1), {stdio: 'inherit'});
-    test.on('error', function(err) {
-      throw err;
+/*************************
+ *Below are failure tests*
+ *************************/
+
+// assert stacktrace shows line of failure
+executor.addCommandlineTest('node lib/cli.js spec/errorTest/singleFailureConf.js')
+    .expectExitCode(1)
+    .expectErrors({
+      stackTrace: 'single_failure_spec1.js:5:32'
     });
-    test.on('exit', function(code) {
-      if (code != 0) {
-        failed = true;
-      }
-      runTests(i + 1);
+
+// assert timeout works 
+executor.addCommandlineTest('node lib/cli.js spec/errorTest/timeoutConf.js')
+    .expectExitCode(1)
+    .expectErrors({
+      message: 'timeout: timed out after 1 msec waiting for spec to complete'
+    })
+    .expectTestDuration(0, 100);
+
+executor.addCommandlineTest('node lib/cli.js spec/errorTest/afterLaunchChangesExitCodeConf.js')
+    .expectExitCode(11)
+    .expectErrors({
+      message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.'
     });
-  } else {
-    process.exit(failed ? 1 : 0);
-  }
-}(0));
+
+executor.addCommandlineTest('node lib/cli.js spec/errorTest/multiFailureConf.js')
+    .expectExitCode(1)
+    .expectErrors([{
+      message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.',
+      stacktrace: 'single_failure_spec1.js:5:32'
+    }, {
+      message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.',
+      stacktrace: 'single_failure_spec2.js:5:32'
+    }]);
+
+executor.addCommandlineTest('node lib/cli.js spec/errorTest/shardedFailureConf.js')
+    .expectExitCode(1)
+    .expectErrors([{
+      message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.',
+      stacktrace: 'single_failure_spec1.js:5:32'
+    }, {
+      message: 'Expected \'Hiya\' to equal \'INTENTIONALLY INCORRECT\'.',
+      stacktrace: 'single_failure_spec2.js:5:32'
+    }]);
+
+executor.addCommandlineTest('node lib/cli.js spec/errorTest/mochaFailureConf.js')
+    .expectExitCode(1)
+    .expectErrors([{
+      message: 'expected \'My AngularJS App\' to equal \'INTENTIONALLY INCORRECT\'',
+      stacktrace: 'mocha_failure_spec.js:11:20'
+    }]);
+
+executor.execute();

scripts/test/test_util.js

@@ -0,0 +1,229 @@
+#!/usr/bin/env node
+
+var child_process = require('child_process'),
+    q = require('q'),
+    fs = require('fs');
+
+// Remove warning message about too many listeners.
+// By virtue of creating so many tests, this will happen.
+process.setMaxListeners(0);
+
+var CommandlineTest = function(command) {
+  var self = this; 
+  this.command_ = command;
+  this.expectedExitCode_ = 0;
+  this.stdioOnlyOnFailures_ = true;
+  this.expectedErrors_ = [];
+  this.assertExitCodeOnly_ = false; 
+
+  // If stdioOnlyOnFailures_ is true, do not stream stdio unless test failed.
+  // This is to prevent tests with expected failures from polluting the output.
+  this.alwaysEnableStdio = function() {
+    self.stdioOnlyOnFailures_ = false;
+    return self;
+  };
+
+  // Only assert the exit code and not failures.
+  // This must be true if the command you're running does not support
+  //   the flag '--testResultOutput'.
+  this.assertExitCodeOnly = function() {
+    self.assertExitCodeOnly_ = true;
+    return self;
+  };
+
+  // Set the expected exit code for the test command. 
+  this.expectExitCode = function(exitCode) {
+    self.expectedExitCode_ = exitCode;
+    return self;
+  };
+
+  // Set the expected total test duration.
+  this.expectTestDuration = function(min, max) {
+    self.expectedMinTestDuration_ = min;
+    self.expectedMaxTestDuration_ = max;
+    return self;
+  };
+
+  /**
+   * Add expected error(s) for the test command. 
+   * Input is an object or list of objects of the following form:
+   * {
+   *  message: string, // optional regex
+   *  stackTrace: string, //optional regex
+   * }
+   */
+  this.expectErrors = function(expectedErrors) {
+    if (expectedErrors instanceof Array) {
+      self.expectedErrors_ = self.expectedErrors_.concat(expectedErrors);
+    } else {
+      self.expectedErrors_.push(expectedErrors);
+    }
+    return self;
+  };
+
+  this.run = function() {
+    var self = this; 
+    var start = new Date().getTime();
+    var testOutputPath = 'test_output_' + start + '.tmp';
+    var output = '';
+
+    var flushAndFail = function(errorMsg) {
+      process.stdout.write(output);
+      throw new Error(errorMsg);
+    };
+
+    // Clean up file 
+    process.on('exit', function() {
+      try {
+        fs.unlinkSync(testOutputPath);
+      } catch (err) {
+        // don't do anything
+      }
+    });
+
+    return q.promise(function (resolve, reject) {
+      if (!self.assertExitCodeOnly_) {
+        self.command_ = self.command_ + ' --testResultOutput ' + testOutputPath;
+      }
+      var args = self.command_.split(/\s/);
+
+      var test_process = child_process.spawn(args[0], args.slice(1));
+
+      test_process.stdout.on('data', function (data) {
+        if (self.stdioOnlyOnFailures_) {
+          output += data;
+        } else {
+          process.stdout.write(data.toString());
+        }
+      });
+
+      test_process.stderr.on('data', function (data) {
+        if (self.stdioOnlyOnFailures_) {
+          output += data;
+        } else {
+          process.stdout.write(data.toString());
+        }
+      });
+
+      test_process.on('error', function(err) {
+        reject(err);
+      });
+
+      test_process.on('exit', function(exitCode) {
+        resolve(exitCode);
+      });
+    }).then(function(exitCode) {
+      if (self.expectedExitCode_ !== exitCode) {
+        flushAndFail('expecting exit code: ' + self.expectedExitCode_ +
+              ', actual: ' + exitCode);
+      }
+
+      // Skip the rest if we are only verify exit code.
+      // Note: we're expecting a file populated by '--testResultOutput' after
+      //   this point.
+      if (self.assertExitCodeOnly_) {
+        return;
+      }
+
+      var raw_data = fs.readFileSync(testOutputPath);
+      fs.unlinkSync(testOutputPath);
+      var testOutput = JSON.parse(raw_data);
+
+      var actualErrors = [];
+      var duration = 0;
+      testOutput.forEach(function(fileResult) {
+        duration += fileResult.duration;
+        fileResult.result.forEach(function(specResult) {
+          if (!specResult.passed) {
+            actualErrors.push(specResult);
+          }
+        });
+      });
+
+      self.expectedErrors_.forEach(function(expectedError) {
+        var found = false;
+        var i = 0;
+        for ( ; i < actualErrors.length; ++i) {
+          var actualError = actualErrors[i];
+
+          // if expected message is defined and messages don't match
+          if (expectedError.message) {
+            if (!actualError.errorMsg || 
+                !actualError.errorMsg.match(new RegExp(expectedError.message))) {
+                  continue;
+                }
+          }
+          // if expected stacktrace is defined and stacktraces don't match
+          if (expectedError.stacktrace) {
+            if (!actualError.stacktrace ||
+                !actualError.stacktrace.match(new RegExp(expectedError.stacktrace))) {
+                  continue;
+                }
+          }
+          found = true;
+          break;
+        }
+
+        if (!found) {
+          flushAndFail('did not fail with expected error with message: [' + 
+            expectedError.message + '] and stacktrace: [' + 
+            expectedError.stacktrace + ']');
+        } else {
+          actualErrors.splice(i, 1);
+        }
+      });
+
+      if (actualErrors.length > 0) {
+        flushAndFail('failed with ' + actualErrors.length + ' unexpected failures');
+      }
+
+      if (self.expectedMinTestDuration_
+          && duration < self.expectedMinTestDuration_) {
+            flushAndFail('expecting test min duration: ' + 
+                self.expectedMinTestDuration_ + ', actual: ' + duration);
+      }
+      if (self.expectedMaxTestDuration_ 
+          && duration > self.expectedMaxTestDuration_) {
+            flushAndFail('expecting test max duration: ' + 
+                self.expectedMaxTestDuration_ + ', actual: ' + duration);
+      }
+    });
+  };
+};
+
+/**
+ * Util for running tests and testing functionalities including:
+ *   exitCode, test durations, expected errors, and expected stacktrace
+ * Note, this will work with any commandline tests, but only if it supports
+ *   the flag '--testResultOutput', unless only exitCode is being tested.
+ *   For now, this means only protractor-jasmine.
+ */
+exports.Executor = function() {
+  var tests = [];
+  this.addCommandlineTest = function(command) {
+    var test = new CommandlineTest(command);
+    tests.push(test);
+    return test; 
+  };
+
+  this.execute = function() {
+    var failed = false;
+
+    (function runTests(i) {
+      if (i < tests.length) {
+        console.log('running: ' + tests[i].command_);
+        tests[i].run().then(function() {
+          console.log('>>> \033[1;32mpass\033[0m');
+        }, function(err) {
+          failed = true;
+          console.log('>>> \033[1;31mfail: ' + err.toString() + '\033[0m');
+        }).fin(function() {
+          runTests(i + 1);
+        }).done();
+      } else {
+        console.log('Summary: ' + (failed ? 'fail' : 'pass'));
+        process.exit(failed ? 1 : 0);
+      }
+    }(0));
+  };
+};

spec/errorTest/afterLaunchChangesExitCodeConf.js

@@ -0,0 +1,25 @@
+var env = require('../environment.js');
+
+// The main suite of Protractor tests to be run on CI servers.
+exports.config = {
+  specs: [
+    'baseCase/single_failure_spec1.js'
+  ],
+
+  multiCapabilities: [{
+    'browserName': 'chrome'
+  }],
+
+  baseUrl: env.baseUrl,
+
+  jasmineNodeOpts: {
+    isVerbose: true,
+    showTiming: true,
+    defaultTimeoutInterval: 90000
+  },
+
+  afterLaunch: function(exitCode) {
+    return exitCode + 10;
+  },
+
+};

spec/errorTest/baseCase/error_spec.js

@@ -0,0 +1,6 @@
+describe('error spec', function() {
+  it('should throw an error', function() {
+    browser.get('index.html');
+    var greeting = element(by.binding('INVALID'));
+  });
+});

spec/errorTest/baseCase/mocha_failure_spec.js

@@ -0,0 +1,13 @@
+// Use the external Chai As Promised to deal with resolving promises in
+// expectations.
+var chai = require('chai');
+var chaiAsPromised = require('chai-as-promised');
+chai.use(chaiAsPromised);
+var expect = chai.expect;
+
+describe('protractor library', function() {
+  it('should wrap webdriver', function() {
+    browser.get('index.html');
+    expect(browser.getTitle()).to.eventually.equal('INTENTIONALLY INCORRECT');
+  });
+});

spec/errorTest/baseCase/multi_failure_spec.js

@@ -0,0 +1,13 @@
+describe('multi failure spec', function() {
+  it('should fail expectation', function() {
+    browser.get('index.html');
+    var greeting = element(by.binding('greeting'));
+    expect(greeting.getText()).toEqual('INTENTIONALLY INCORRECT');
+  });
+
+  it('should fail expectation again', function() {
+    browser.get('index.html');
+    var greeting = element(by.binding('greeting'));
+    expect(greeting.getText()).toEqual('INTENTIONALLY INCORRECT AGAIN');
+  });
+});