Make CI run browser tests (#2231)

* Initial prep for browser tests in CI

- ⬆️ [email protected]

  Some fixes that we didn't have were required.

- 🐛 Fix browser not being able to require describe/it

  `support/browser-entry.js` was removing the listener that is in charge
  of keeping `Mocha.describe/it/suite` etc up-to-date.

  That listener is now added everytime a `ui` is set up.

- Correct isatty for browser

* Travis + SauceLabs + Karma setup (first pass)

- add sauce connect
- prepare to use karma on saucelabs; fix potential windows problems
- rewrite `scripts/ensure-compatible-npm.sh` in JS
- detect travis node 0.8 env; just upgrade if found
- symlink `mocha` to itself for `karma-mocha`
- if explicit `--production` flag is present, skip this
- don't know how to skip this part if `mocha` is a devDep of some other package
- use [email protected]; don't try to install npm@3
- browser tests with karma & phantom
- move `support/browser-entry.js` to `browser-entry.js` to keep paths sane
- `karma.conf.js` loads data out of `.karma.conf.js`
  - we have a "main" browser suite
    - then we have suites for each interface
    - each suite is a single run of Karma
    - each suite is executed by the `Makefile`
  - build `./mocha.js` upon test execution for `karma-mocha` to use
  - bonus: 3rd-party browserification should theoretically now be possible as per the `./index.js` entry in `package.json`'s `browser` field
  - add dev deps for karma
  - test changes:
    - split `lookupFiles` test into its own file so we can ignore it in Karma
    - fix broken `test/acceptance/throw.js` for browser
    - fix `test/acceptance/utils.js`'s `type` tests for PhantomJS
- try to reduce build matrix
- fix missing targets
- fix infinite loop in Makefile
- downgrade phantomjs to 1.9.8
- remove some cruft from karma config; try to add sauce labs
- try again w/ the saucelabs
- typo
- remove karma-source-map-support as it's a no-go on IE8
- Require up-to-date mocha.js for any browser tests
- Make CI rebuild mocha.js.  Make won't rebuild when just checked out by CI because timestamps on source and mocha.js are the same.  We could use a script to set the timestamps to the commit time, but it still wouldn't work if mocha.js were committed along with other changes that hadn't been built into it. Which shouldn't happen anyway -- but then again, the point of CI is to see what commits change, it's usually going to need to rebuild mocha.js if it's working right, so it's not going to hurt much to rebuild it every time even on the few times it doesn't need to.

* fix IE8 compatibility for browser tests

- Avoid `Array.prototype.map` in test
- Workaround for missing `Object.create`
- Use a shim for `Date.prototype.toISOString`
- Use simple number math instead of array indexing for interface tests
- Use `expect` instead of `should`
- Avoid builtin function in stringify test (A quick check revealed that stringify does not treat toString specially anyway, and IE8 ignores the toString assigned to the object, so use a different property name)
- Use `karma-expect` for automatic browser tests
- Remove `karma-should`

* Travis + SauceLabs + Karma setup (second pass)

- add note about exporting `global` in `browser-entry.js`
- remove too-short timeout for "throw" unit tests (because SauceLabs)
- revert modification to `isatty()` function of `lib/browser/tty.js`
- switch reporter to `karma-spec-reporter`
- use custom `karma-no-mocha` package
- fixtures in `test/browser-fixtures/`
- use single `karma.conf.js`

Author: dasilvacontin

.travis.yml

@@ -8,22 +8,37 @@ dist: trusty
 
 language: node_js
 
-node_js:
-  - '6'
-  - '5'
-  - '4'
-  - 'iojs'
-  - '0.12'
-  - '0.11'
-  - '0.10'
-  - '0.8'
+matrix:
+  include:
+    - node_js: '6'
+      env: TARGET="clean lint test-node test-browser"
+    - node_js: '5'
+      env: TARGET=test-node
+    - node_js: '4'
+      env: TARGET=test-node
+    - node_js: 'iojs'
+      env: TARGET=test-node
+    - node_js: '0.12'
+      env: TARGET=test-node
+    - node_js: '0.11'
+      env: TARGET=test-node
+    - node_js: '0.10'
+      env: TARGET=test-node
+    - node_js: '0.8'
+      env: TARGET=test-node
 
 before_install:
   # node 0.8 won't install our dev deps with an out-of-box npm;
   # this upgrades it
-  - ./scripts/ensure-compatible-npm.sh
+  - node ./scripts/upgrade-npm.js
 
-script: travis_retry npm test
+script: travis_retry make $TARGET
+
+addons:
+  sauce_connect:
+    username: mochajs
+    access_key:
+      secure: R0HXKtR6F2iDEnItv57BTRyL64XfyIlyyluPLK8G33O/InaQjT3KxGuxevz3nVYIqqnI1MPjYodXcQaqrBOLUVmA2vhBeMHB2OwGc9GAL+HBtB1fh+bQJelkl/XMcTTbC5LIZ6nZjmFnkmjqT3AmUhdDRISgieIFeVY4x48LfiU=
 
 notifications:
   urls:

CHANGELOG.md

@@ -1,3 +1,11 @@
+Unreleased
+==================
+
+* [#2079], [#2231] - Add browser to CI
+
+[#2079]: https://github.com/mochajs/mocha/issues/2079
+[#2231]: https://github.com/mochajs/mocha/pull/2231
+
 2.4.5 / 2016-01-28
 ==================
 

Makefile

@@ -1,15 +1,18 @@
 BROWSERIFY := node_modules/.bin/browserify
 ESLINT := node_modules/.bin/eslint
+KARMA := node_modules/.bin/karma
 
 REPORTER ?= spec
 TM_BUNDLE = JavaScript\ mocha.tmbundle
 SRC = $(shell find lib -name "*.js" -type f | sort)
+TESTS = $(shell find test -name "*.js" -type f | sort)
 SUPPORT = $(wildcard support/*.js)
 
 all: mocha.js
 
 mocha.js: $(SRC) $(SUPPORT)
-	@$(BROWSERIFY) ./support/browser-entry \
+	@printf "==> [Browser :: build]\n"
+	@$(BROWSERIFY) ./browser-entry \
 		--ignore 'fs' \
 		--ignore 'glob' \
 		--ignore 'jade' \
@@ -18,48 +21,75 @@ mocha.js: $(SRC) $(SUPPORT)
 		--exclude './lib-cov/mocha' > $@
 
 clean:
+	@printf "==> [Clean]\n"
 	rm -f mocha.js
 	rm -rf test-outputs
-	rm -fr lib-cov
+	rm -rf lib-cov
 	rm -f coverage.html
 
 test-cov: lib-cov
+	@printf "==> [Test :: Coverage]\n"
 	@COV=1 $(MAKE) test REPORTER=html-cov > coverage.html
 
 lib-cov:
+	@printf "==> [Coverage]\n"
 	@rm -fr ./$@
 	@jscoverage lib $@
 
 lint:
+	@printf "==> [Test :: Lint]\n"
 	@$(ESLINT) $(SRC)
 
-test: lint test-unit
+test-node: test-bdd test-tdd test-qunit test-exports test-unit test-integration test-jsapi test-compilers test-glob test-requires test-reporters test-only
 
-test-all: lint test-bdd test-tdd test-qunit test-exports test-unit test-integration test-jsapi test-compilers test-glob test-requires test-reporters test-only
+test-browser: test-browser-unit test-browser-bdd test-browser-qunit test-browser-tdd test-browser-exports
+
+test: lint test-node test-browser
+
+test-browser-unit: mocha.js
+	@printf "==> [Test :: Browser]\n"
+	@NODE_PATH=. $(KARMA) start
+
+test-browser-bdd:
+	@printf "==> [Test :: Browser :: BDD]\n"
+	@MOCHA_UI=bdd $(MAKE) test-browser-unit
+
+test-browser-qunit:
+	@printf "==> [Test :: Browser :: QUnit]\n"
+	@MOCHA_UI=qunit $(MAKE) test-browser-unit
+
+test-browser-tdd:
+	@printf "==> [Test :: Browser :: TDD]\n"
+	@MOCHA_UI=tdd $(MAKE) test-browser-unit
 
 test-jsapi:
+	@printf "==> [Test :: JS API]\n"
 	@node test/jsapi
 
 test-unit:
+	@printf "==> [Test :: Unit]\n"
 	@./bin/mocha \
 		--reporter $(REPORTER) \
 		test/acceptance/*.js \
 		--growl \
 		test/*.js
 
 test-integration:
+	@printf "==> [Test :: Integrations]\n"
 	@./bin/mocha \
 		--reporter $(REPORTER) \
 		test/integration/*.js
 
 test-compilers:
+	@printf "==> [Test :: Compilers]\n"
 	@./bin/mocha \
 		--reporter $(REPORTER) \
 		--compilers coffee:coffee-script/register,foo:./test/compiler/foo \
 		test/acceptance/test.coffee \
 		test/acceptance/test.foo
 
 test-requires:
+	@printf "==> [Test :: Requires]\n"
 	@./bin/mocha \
 		--reporter $(REPORTER) \
 		--compilers coffee:coffee-script/register \
@@ -70,38 +100,45 @@ test-requires:
 		test/acceptance/require/require.js
 
 test-bdd:
+	@printf "==> [Test :: BDD]\n"
 	@./bin/mocha \
 		--reporter $(REPORTER) \
 		--ui bdd \
 		test/acceptance/interfaces/bdd
 
 test-tdd:
+	@printf "==> [Test :: TDD]\n"
 	@./bin/mocha \
 		--reporter $(REPORTER) \
 		--ui tdd \
 		test/acceptance/interfaces/tdd
 
 test-qunit:
+	@printf "==> [Test :: QUnit]\n"
 	@./bin/mocha \
 		--reporter $(REPORTER) \
 		--ui qunit \
 		test/acceptance/interfaces/qunit
 
 test-exports:
+	@printf "==> [Test :: Exports]\n"
 	@./bin/mocha \
 		--reporter $(REPORTER) \
 		--ui exports \
 		test/acceptance/interfaces/exports
 
 test-glob:
+	@printf "==> [Test :: Glob]\n"
 	@./test/acceptance/glob/glob.sh
 
 test-reporters:
+	@printf "==> [Test :: Reporters]\n"
 	@./bin/mocha \
 		--reporter $(REPORTER) \
 		test/reporters/*.js
 
 test-only:
+	@printf "==> [Test :: Only]\n"
 	@./bin/mocha \
 		--reporter $(REPORTER) \
 		--ui tdd \
@@ -123,11 +160,13 @@ test-only:
 		test/acceptance/misc/only/qunit
 
 test-mocha:
+	@printf "==> [Test :: Mocha]\n"
 	@./bin/mocha \
 		--reporter $(REPORTER) \
 		test/mocha
 
 non-tty:
+	@printf "==> [Test :: Non-TTY]\n"
 	@./bin/mocha \
 		--reporter dot \
 		test/acceptance/interfaces/bdd 2>&1 > /tmp/dot.out
@@ -150,6 +189,7 @@ non-tty:
 	@cat /tmp/spec.out
 
 tm:
+	@printf "==> [TM]\n"
 	@open editors/$(TM_BUNDLE)
 
-.PHONY: test-cov test-jsapi test-compilers watch test test-all test-bdd test-tdd test-qunit test-exports test-unit test-integration non-tty tm clean
+.PHONY: test-cov test-jsapi test-compilers watch test test-node test-bdd test-tdd test-qunit test-exports test-unit test-integration non-tty tm clean test-browser test-browser-unit test-browser-bdd test-browser-qunit test-browser-tdd test-browser-exports lint

support/browser-entry.js renamed to browser-entry.js

@@ -4,7 +4,7 @@
 
 process.stdout = require('browser-stdout')();
 
-var Mocha = require('../');
+var Mocha = require('./lib/mocha');
 
 /**
  * Create a Mocha instance.
@@ -159,3 +159,8 @@ Mocha.process = process;
 
 global.Mocha = Mocha;
 global.mocha = mocha;
+
+// this allows test/acceptance/required-tokens.js to pass; thus,
+// you can now do `const describe = require('mocha').describe` in a
+// browser context (assuming browserification).  should fix #880
+module.exports = global;

karma.conf.js

@@ -0,0 +1,97 @@
+'use strict';
+
+module.exports = function(config) {
+  var cfg = {
+    frameworks: [
+      'browserify',
+      'expect',
+      'mocha'
+    ],
+    files: [
+      'test/browser-fixtures/bdd.js',
+      'test/acceptance/*.js'
+    ],
+    exclude: [
+      'test/acceptance/http.js',
+      'test/acceptance/fs.js',
+      'test/acceptance/lookup-files.js',
+      'test/acceptance/require/**/*.js',
+      'test/acceptance/misc/**/*.js'
+    ],
+    preprocessors: {
+      'test/**/*.js': ['browserify']
+    },
+    browserify: {
+      debug: true,
+      configure: function configure(b) {
+        b.ignore('glob')
+          .ignore('jade')
+          .ignore('supports-color')
+          .exclude('./lib-cov/mocha');
+      }
+    },
+    reporters: ['spec'],
+    colors: true,
+    browsers: ['PhantomJS'],
+    logLevel: config.LOG_INFO,
+    singleRun: true
+  };
+
+  // see https://github.com/saucelabs/karma-sauce-example
+  // TO RUN LOCALLY:
+  // Execute `CI=1 make test-browser`, once you've set the SAUCE_USERNAME and
+  // SAUCE_ACCESS_KEY env vars.
+  if (process.env.CI) {
+    if (!(process.env.SAUCE_USERNAME || process.env.SAUCE_ACCESS_KEY)) {
+      throw new Error('Must set SAUCE_USERNAME and SAUCE_ACCESS_KEY '
+        + 'environment variables!');
+    }
+    cfg.reporters.push('saucelabs');
+    cfg.browsers.push('ie8');
+    cfg.customLaunchers = {
+      ie8: {
+        base: 'SauceLabs',
+        browserName: 'internet explorer',
+        platform: 'Windows XP',
+        version: '8.0'
+      }
+    };
+
+    cfg.sauceLabs = {
+      public: 'public'
+    };
+
+    if (process.env.TRAVIS) {
+      // correlate build/tunnel with Travis
+      cfg.sauceLabs.build = 'TRAVIS #' + process.env.TRAVIS_BUILD_NUMBER
+        + ' (' + process.env.TRAVIS_BUILD_ID + ')';
+      cfg.sauceLabs.tunnelIdentifier = process.env.TRAVIS_JOB_NUMBER;
+      cfg.sauceLabs.startConnect = false;
+    } else {
+      // otherwise just make something up
+      cfg.sauceLabs.build = require('os').hostname() + ' (' + Date.now() + ')';
+    }
+
+    // for slow browser booting, ostensibly
+    cfg.captureTimeout = 120000;
+  }
+
+  // the MOCHA_UI env var will determine if we're running interface-specific
+  // tets.  since you can only load one at a time, each must be run separately.
+  // each has its own set of acceptance tests and a fixture.
+  // the "bdd" fixture is used by default.
+  var ui = process.env.MOCHA_UI;
+  if (ui) {
+    if (cfg.sauceLabs) {
+      cfg.sauceLabs.testName = 'Interface "' + ui + '" integration tests';
+    }
+    cfg.files = [
+      'test/browser-fixtures/' + ui + '.js',
+      'test/acceptance/interfaces/' + ui + '.js'
+    ];
+  } else if (cfg.sauceLabs) {
+    cfg.sauceLabs.testName = 'Unit Tests';
+  }
+
+  config.set(cfg);
+};

lib/mocha.js

@@ -99,22 +99,6 @@ function Mocha(options) {
   if (options.slow) {
     this.slow(options.slow);
   }
-
-  this.suite.on('pre-require', function(context) {
-    exports.afterEach = context.afterEach || context.teardown;
-    exports.after = context.after || context.suiteTeardown;
-    exports.beforeEach = context.beforeEach || context.setup;
-    exports.before = context.before || context.suiteSetup;
-    exports.describe = context.describe || context.suite;
-    exports.it = context.it || context.test;
-    exports.setup = context.setup || context.beforeEach;
-    exports.suiteSetup = context.suiteSetup || context.before;
-    exports.suiteTeardown = context.suiteTeardown || context.after;
-    exports.suite = context.suite || context.describe;
-    exports.teardown = context.teardown || context.afterEach;
-    exports.test = context.test || context.it;
-    exports.run = context.run;
-  });
 }
 
 /**
@@ -202,6 +186,23 @@ Mocha.prototype.ui = function(name) {
     }
   }
   this._ui = this._ui(this.suite);
+
+  this.suite.on('pre-require', function(context) {
+    exports.afterEach = context.afterEach || context.teardown;
+    exports.after = context.after || context.suiteTeardown;
+    exports.beforeEach = context.beforeEach || context.setup;
+    exports.before = context.before || context.suiteSetup;
+    exports.describe = context.describe || context.suite;
+    exports.it = context.it || context.test;
+    exports.setup = context.setup || context.beforeEach;
+    exports.suiteSetup = context.suiteSetup || context.before;
+    exports.suiteTeardown = context.suiteTeardown || context.after;
+    exports.suite = context.suite || context.describe;
+    exports.teardown = context.teardown || context.afterEach;
+    exports.test = context.test || context.it;
+    exports.run = context.run;
+  });
+
   return this;
 };