Enable source maps in Karma

.gitmodules

@@ -1,3 +1,4 @@
 [submodule "test/web-platform-tests"]
-	path = test/web-platform-tests
-	url = https://github.com/w3c/web-platform-tests.git
+  path = test/web-platform-tests
+  url = https://github.com/alancutter/web-platform-tests.git
+  branch = nativeShippingAPI

CONTRIBUTING.md

@@ -15,6 +15,8 @@ You can run the tests in an interactive mode with `grunt debug`. This starts the
 Karma server once for each polyfill target for each test framework.
 Navigate to `http://localhost:9876/debug.html` to open the test runner in your
 browser of choice, all test results appear in the Javascript console.
+Test failures can be accessed via `window.failures` and `window.formattedFailures`
+once the tests have completed.
 
 The polyfill target and tests can be specified as arguments to the `debug` task.  
 Example: `grunt debug:web-animations-next:test/web-platform-tests/web-animations/animation/pause.html`  

Gruntfile.js

@@ -227,11 +227,7 @@ module.exports = function(grunt) {
       });
     }
     function runWebPlatformTests() {
-      if (!config.runWebPlatformTests) {
-        return Promise.resolve(true);
-      }
-
-      var testFiles = filterTests(grunt.file.expand('test/web-platform-tests/web-animations/**/*.html'));
+      var testFiles = filterTests(grunt.file.expand(config.webPlatformTests));
       if (testFiles.length == 0) {
         return Promise.resolve(true);
       }
@@ -241,13 +237,14 @@ module.exports = function(grunt) {
         configCallback(karmaConfig);
         karmaConfig.client.testharnessTests = require('./test/web-platform-tests-expectations.js');
         karmaConfig.client.testharnessTests.testURLList = testFiles;
+        karmaConfig.proxies['/base/polyfill.js'] = '/base/' + task.target + '.min.js';
         karmaConfig.files.push('test/karma-testharness-adapter.js');
         var servedFiles = [
           'test/web-platform-tests/resources/**',
           'test/web-platform-tests/web-animations/**',
           'test/resources/*',
           'src/**',
-          '*.js',
+          '*.js*',
 
           // TODO(alancutter): Make a separate grunt task for serving these imported Blink tests.
           'test/blink/**',

package.json

@@ -41,6 +41,6 @@
     "source-map": "~0.1.40"
   },
   "scripts": {
-    "test": "grunt test gjslint git-status checkrepo"
+    "test": "grunt web-animations web-animations-next test gjslint git-status checkrepo"
   }
 }

target-config.js

@@ -110,6 +110,26 @@
       'test/js/timeline.js',
       'test/js/web-animations-next-animation.js');
 
+  var webAnimations1WebPlatformTests = [
+    'test/web-platform-tests/web-animations/animation-model/animation-types/discrete-animation.html',
+    'test/web-platform-tests/web-animations/animation-model/keyframe-effects/effect-value-context.html',
+    'test/web-platform-tests/web-animations/animation-model/keyframe-effects/the-effect-value-of-a-keyframe-effect.html',
+    'test/web-platform-tests/web-animations/interfaces/Animatable/animate-basic.html',
+    'test/web-platform-tests/web-animations/interfaces/Animation/cancel.html',
+    'test/web-platform-tests/web-animations/interfaces/Animation/id.html',
+    'test/web-platform-tests/web-animations/interfaces/Animation/pause.html',
+    'test/web-platform-tests/web-animations/interfaces/Animation/play.html',
+    'test/web-platform-tests/web-animations/interfaces/Animation/playState.html',
+    'test/web-platform-tests/web-animations/interfaces/Animation/startTime.html',
+    'test/web-platform-tests/web-animations/interfaces/KeyframeEffect/effect-easing.html',
+    'test/web-platform-tests/web-animations/timing-model/animation-effects/active-time.html',
+    'test/web-platform-tests/web-animations/timing-model/animation-effects/simple-iteration-progress.html',
+  ];
+
+  var webAnimationsNextWebPlatformTests = [
+    'test/web-platform-tests/web-animations/**/*.html',
+  ];
+
   // This object specifies the source and test files for different Web Animation build targets.
   var targetConfig = {
     'web-animations': {
@@ -120,7 +140,7 @@
       webAnimationsNextSrc: [],
       src: scopeSrc.concat(sharedSrc).concat(webAnimations1Src).concat(webAnimations1BonusSrc),
       polyfillTests: webAnimations1PolyfillTests,
-      runWebPlatformTests: false,
+      webPlatformTests: webAnimations1WebPlatformTests,
     },
     'web-animations-next': {
       scopeSrc: scopeSrc,
@@ -130,7 +150,7 @@
       webAnimationsNextSrc: webAnimationsNextSrc,
       src: scopeSrc.concat(sharedSrc).concat(webAnimations1Src).concat(webAnimations1BonusSrc).concat(webAnimationsNextSrc),
       polyfillTests: webAnimationsNextPolyfillTests,
-      runWebPlatformTests: true,
+      webPlatformTests: webAnimationsNextWebPlatformTests,
     },
     'web-animations-next-lite': {
       scopeSrc: scopeSrc,
@@ -140,7 +160,7 @@
       webAnimationsNextSrc: webAnimationsNextSrc,
       src: scopeSrc.concat(sharedSrc).concat(liteWebAnimations1Src).concat(webAnimations1BonusSrc).concat(webAnimationsNextSrc),
       polyfillTests: [],
-      runWebPlatformTests: false,
+      webPlatformTests: [],
     },
   };
 

test/karma-config.js

@@ -21,7 +21,6 @@ module.exports = function(config) {
       '/resources/WebIDLParser.js': '/base/test/web-platform-tests/resources/webidl2/lib/webidl2.js',
 
       '/test/': '/base/test/',
-      '/polyfill/': '/base/',
     },
   });
 };

test/karma-testharness-adapter.js

@@ -19,7 +19,7 @@
     return '\'' + string.replace(/\\/g, '\\\\').replace(/\n/g, '\\n').replace(/'/g, '\\\'') + '\'';
   }
 
-  function checkExpectations(testURL, passes, failures, expectedFailures, flakyTestIndicator) {
+  function checkExpectations(testURL, passes, failures, expectedFailures) {
     expectedFailures = expectedFailures || {};
 
     var failedDifferently = false;
@@ -37,7 +37,7 @@
     for (var name in failures) {
       var message = failures[name];
       if (name in expectedFailures) {
-        if (expectedFailures[name] != flakyTestIndicator && message != expectedFailures[name]) {
+        if (message != expectedFailures[name]) {
           failedDifferently = true;
           differentFailures[name] = message;
         }
@@ -47,7 +47,7 @@
       }
     }
     for (var name in expectedFailures) {
-      if (name in passes && expectedFailures[name] != flakyTestIndicator) {
+      if (name in passes) {
         passedUnexpectedly = true;
         unexpectedPasses.push(name);
       } else if (!(name in failures)) {
@@ -122,9 +122,29 @@
     return true;
   }
 
-  function runRemainingTests(remainingTestURLs, config, testNameDiv, iframe) {
+  // Serialises the failures suitable for pasting into expectedFailures: {} in web-platform-tests-expectations.js
+  function formatFailures(failures) {
+    var testURLs = Object.keys(failures);
+    testURLs.sort();
+    return testURLs.map(function(testURL) {
+      var tests = Object.keys(failures[testURL]);
+      tests.sort();
+      return (
+          '    ' + stringify(testURL) + ': {\n' +
+          tests.map(function(test) {
+            return (
+                '      ' + stringify(test) + ':\n' +
+                '          ' + stringify(failures[testURL][test]) + ',\n');
+          }).join('\n') +
+          '    },\n');
+    }).join('\n');
+  }
+
+  function runRemainingTests(remainingTestURLs, config, testNameDiv, iframe, outputFailures) {
     if (remainingTestURLs.length == 0) {
       karma.complete();
+      window.failures = outputFailures;
+      window.formattedFailures = formatFailures(outputFailures);
       return;
     }
 
@@ -136,17 +156,22 @@
         description: '',
         skipped: true,
       });
-      runRemainingTests(remainingTestURLs.slice(1), config, testNameDiv, iframe);
+      runRemainingTests(remainingTestURLs.slice(1), config, testNameDiv, iframe, outputFailures);
       return;
     }
 
     // This expects testharnessreport.js in the iframe to look for this function on the
     // parent window and call it once testharness.js has loaded.
     window.onTestharnessLoaded = function(innerWindow) {
       innerWindow.add_completion_callback(function(results) {
+        var expectations = config.expectedFailures[testURL];
         var failures = {};
         var passes = {};
         results.forEach(function(result) {
+          if (expectations && expectations[result.name] == config.flakyTestIndicator) {
+            failures[result.name] = config.flakyTestIndicator;
+            return;
+          }
           if (result.status == 0) {
             passes[result.name] = true;
           } else {
@@ -156,16 +181,18 @@
             failures[result.name] = result.message;
           }
         });
+        if (Object.keys(failures).length > 0) {
+          outputFailures[testURL] = failures;
+        }
 
-        karma.result(checkExpectations(testURL, passes, failures, config.expectedFailures[testURL], config.flakyTestIndicator));
-        runRemainingTests(remainingTestURLs.slice(1), config, testNameDiv, iframe);
+        karma.result(checkExpectations(testURL, passes, failures, expectations));
+        runRemainingTests(remainingTestURLs.slice(1), config, testNameDiv, iframe, outputFailures);
       });
     };
     testNameDiv.textContent = testURL;
     iframe.src = testURL;
   }
 
-
   karma.start = function() {
     // Karma's config.client object appears as karma.config here.
     var config = karma.config.testharnessTests;
@@ -182,6 +209,6 @@
     iframe.style.height = 'calc(100vh - 60px)';
     document.body.appendChild(iframe);
 
-    runRemainingTests(config.testURLList, config, testNameDiv, iframe);
+    runRemainingTests(config.testURLList, config, testNameDiv, iframe, {});
   };
 })();

test/resources/testharnessreport.js

@@ -406,9 +406,8 @@ function dump_test_results(tests, status) {
 }
 
 /* BEGIN WEB ANIMATIONS POLYFILL EXTRAS */
-// TODO(alancutter): Use a generic script injection method to allow grunt to choose what
-// version of the polyfill to load for testing.
-document.write('<script src="/polyfill/web-animations-next.dev.js"></script>');
+// The path /base/polyfill.js is expected to be a proxy for the appropriate polyfill script configured in Karma.
+document.write('<script src="/base/polyfill.js"></script>');
 if (window.parent && parent.window.onTestharnessLoaded) {
     parent.window.onTestharnessLoaded(window);
 } else {