Dump testharness test failures on window for debug mode

CONTRIBUTING.md

@@ -15,6 +15,8 @@ You can run the tests in an interactive mode with `grunt debug`. This starts the
 Karma server once for each polyfill target for each test framework.
 Navigate to `http://localhost:9876/debug.html` to open the test runner in your
 browser of choice, all test results appear in the Javascript console.
+Test failures can be accessed via `window.failures` and `window.formattedFailures`
+once the tests have completed.
 
 The polyfill target and tests can be specified as arguments to the `debug` task.  
 Example: `grunt debug:web-animations-next:test/web-platform-tests/web-animations/animation/pause.html`  

test/karma-testharness-adapter.js

@@ -15,11 +15,14 @@
 (function() {
   var karma = window.__karma__;
 
+  // Behaves like JSON.stringify() except only for strings and outputs strings with single quotes
+  // instead of double quotes.
+  // This so we can paste test results as expectations while keeping our linter happy.
   function stringify(string) {
     return '\'' + string.replace(/\\/g, '\\\\').replace(/\n/g, '\\n').replace(/'/g, '\\\'') + '\'';
   }
 
-  function checkExpectations(testURL, passes, failures, expectedFailures, flakyTestIndicator) {
+  function checkExpectations(testURL, passes, failures, expectedFailures) {
     expectedFailures = expectedFailures || {};
 
     var failedDifferently = false;
@@ -37,7 +40,7 @@
     for (var name in failures) {
       var message = failures[name];
       if (name in expectedFailures) {
-        if (expectedFailures[name] != flakyTestIndicator && message != expectedFailures[name]) {
+        if (message != expectedFailures[name]) {
           failedDifferently = true;
           differentFailures[name] = message;
         }
@@ -47,7 +50,7 @@
       }
     }
     for (var name in expectedFailures) {
-      if (name in passes && expectedFailures[name] != flakyTestIndicator) {
+      if (name in passes) {
         passedUnexpectedly = true;
         unexpectedPasses.push(name);
       } else if (!(name in failures)) {
@@ -122,9 +125,29 @@
     return true;
   }
 
-  function runRemainingTests(remainingTestURLs, config, testNameDiv, iframe) {
+  // Serialises the failures suitable for pasting into expectedFailures: {} in web-platform-tests-expectations.js
+  function formatFailures(failures) {
+    var testURLs = Object.keys(failures);
+    testURLs.sort();
+    return testURLs.map(function(testURL) {
+      var tests = Object.keys(failures[testURL]);
+      tests.sort();
+      return (
+          '    ' + stringify(testURL) + ': {\n' +
+          tests.map(function(test) {
+            return (
+                '      ' + stringify(test) + ':\n' +
+                '          ' + stringify(failures[testURL][test]) + ',\n');
+          }).join('\n') +
+          '    },\n');
+    }).join('\n');
+  }
+
+  function runRemainingTests(remainingTestURLs, config, testNameDiv, iframe, outputFailures) {
     if (remainingTestURLs.length == 0) {
       karma.complete();
+      window.failures = outputFailures;
+      window.formattedFailures = formatFailures(outputFailures);
       return;
     }
 
@@ -136,17 +159,22 @@
         description: '',
         skipped: true,
       });
-      runRemainingTests(remainingTestURLs.slice(1), config, testNameDiv, iframe);
+      runRemainingTests(remainingTestURLs.slice(1), config, testNameDiv, iframe, outputFailures);
       return;
     }
 
     // This expects testharnessreport.js in the iframe to look for this function on the
     // parent window and call it once testharness.js has loaded.
     window.onTestharnessLoaded = function(innerWindow) {
       innerWindow.add_completion_callback(function(results) {
+        var expectations = config.expectedFailures[testURL];
         var failures = {};
         var passes = {};
         results.forEach(function(result) {
+          if (expectations && expectations[result.name] == config.flakyTestIndicator) {
+            failures[result.name] = config.flakyTestIndicator;
+            return;
+          }
           if (result.status == 0) {
             passes[result.name] = true;
           } else {
@@ -156,16 +184,18 @@
             failures[result.name] = result.message;
           }
         });
+        if (Object.keys(failures).length > 0) {
+          outputFailures[testURL] = failures;
+        }
 
-        karma.result(checkExpectations(testURL, passes, failures, config.expectedFailures[testURL], config.flakyTestIndicator));
-        runRemainingTests(remainingTestURLs.slice(1), config, testNameDiv, iframe);
+        karma.result(checkExpectations(testURL, passes, failures, expectations));
+        runRemainingTests(remainingTestURLs.slice(1), config, testNameDiv, iframe, outputFailures);
       });
     };
     testNameDiv.textContent = testURL;
     iframe.src = testURL;
   }
 
-
   karma.start = function() {
     // Karma's config.client object appears as karma.config here.
     var config = karma.config.testharnessTests;
@@ -182,6 +212,6 @@
     iframe.style.height = 'calc(100vh - 60px)';
     document.body.appendChild(iframe);
 
-    runRemainingTests(config.testURLList, config, testNameDiv, iframe);
+    runRemainingTests(config.testURLList, config, testNameDiv, iframe, {});
   };
 })();