test_runner: add code coverage support to spec reporter

PR-URL: #46674 Reviewed-By: Benjamin Gruenbaum <[email protected]> Reviewed-By: Moshe Atlow <[email protected]>
nodejs · Apr 8, 2023 · 6b87f29 · 6b87f29
1 parent 23a69d9
commit 6b87f29
Show file tree

Hide file tree

Showing 4 changed files with 163 additions and 89 deletions.
diff --git a/lib/internal/test_runner/reporter/spec.js b/lib/internal/test_runner/reporter/spec.js
@@ -15,7 +15,7 @@ const assert = require('assert');
 const Transform = require('internal/streams/transform');
 const { inspectWithNoCustomRetry } = require('internal/errors');
 const { green, blue, red, white, gray } = require('internal/util/colors');
-
+const { getCoverageReport } = require('internal/test_runner/utils');
 
 const inspectOptions = { __proto__: null, colors: true, breakLength: Infinity };
 
@@ -30,6 +30,7 @@ const symbols = {
   'test:fail': '\u2716 ',
   'test:pass': '\u2714 ',
   'test:diagnostic': '\u2139 ',
+  'test:coverage': '\u2139 ',
   'arrow:right': '\u25B6 ',
   'hyphen:minus': '\uFE63 ',
 };
@@ -115,6 +116,8 @@ class SpecReporter extends Transform {
         break;
       case 'test:diagnostic':
         return `${colors[type]}${this.#indent(data.nesting)}${symbols[type]}${data.message}${white}\n`;
+      case 'test:coverage':
+        return getCoverageReport(this.#indent(data.nesting), data.summary, symbols['test:coverage'], blue);
     }
   }
   _transform({ type, data }, encoding, callback) {

diff --git a/lib/internal/test_runner/reporter/tap.js b/lib/internal/test_runner/reporter/tap.js
@@ -3,7 +3,6 @@ const {
   ArrayPrototypeForEach,
   ArrayPrototypeJoin,
   ArrayPrototypePush,
-  NumberPrototypeToFixed,
   ObjectEntries,
   RegExpPrototypeSymbolReplace,
   SafeMap,
@@ -13,7 +12,7 @@ const {
 } = primordials;
 const { inspectWithNoCustomRetry } = require('internal/errors');
 const { isError, kEmptyObject } = require('internal/util');
-const { relative } = require('path');
+const { getCoverageReport } = require('internal/test_runner/utils');
 const kDefaultIndent = '    '; // 4 spaces
 const kFrameStartRegExp = /^ {4}at /;
 const kLineBreakRegExp = /\n|\r\n/;
@@ -49,7 +48,7 @@ async function * tapReporter(source) {
         yield `${indent(data.nesting)}# ${tapEscape(data.message)}\n`;
         break;
       case 'test:coverage':
-        yield reportCoverage(data.nesting, data.summary);
+        yield getCoverageReport(indent(data.nesting), data.summary, '# ', '');
         break;
     }
   }
@@ -73,40 +72,6 @@ function reportTest(nesting, testNumber, status, name, skip, todo) {
   return line;
 }
 
-function reportCoverage(nesting, summary) {
-  const pad = indent(nesting);
-  let report = `${pad}# start of coverage report\n`;
-
-  report += `${pad}# file | line % | branch % | funcs % | uncovered lines\n`;
-
-  for (let i = 0; i < summary.files.length; ++i) {
-    const {
-      path,
-      coveredLinePercent,
-      coveredBranchPercent,
-      coveredFunctionPercent,
-      uncoveredLineNumbers,
-    } = summary.files[i];
-    const relativePath = relative(summary.workingDirectory, path);
-    const lines = NumberPrototypeToFixed(coveredLinePercent, 2);
-    const branches = NumberPrototypeToFixed(coveredBranchPercent, 2);
-    const functions = NumberPrototypeToFixed(coveredFunctionPercent, 2);
-    const uncovered = ArrayPrototypeJoin(uncoveredLineNumbers, ', ');
-
-    report += `${pad}# ${relativePath} | ${lines} | ${branches} | ` +
-              `${functions} | ${uncovered}\n`;
-  }
-
-  const { totals } = summary;
-  report += `${pad}# all files | ` +
-            `${NumberPrototypeToFixed(totals.coveredLinePercent, 2)} | ` +
-            `${NumberPrototypeToFixed(totals.coveredBranchPercent, 2)} | ` +
-            `${NumberPrototypeToFixed(totals.coveredFunctionPercent, 2)} |\n`;
-
-  report += `${pad}# end of coverage report\n`;
-  return report;
-}
-
 function reportDetails(nesting, data = kEmptyObject) {
   const { error, duration_ms } = data;
   const _indent = indent(nesting);

diff --git a/lib/internal/test_runner/utils.js b/lib/internal/test_runner/utils.js
@@ -1,19 +1,22 @@
 'use strict';
 const {
+  ArrayPrototypeJoin,
   ArrayPrototypeMap,
   ArrayPrototypePush,
   ObjectGetOwnPropertyDescriptor,
+  NumberPrototypeToFixed,
   SafePromiseAllReturnArrayLike,
   RegExp,
   RegExpPrototypeExec,
   SafeMap,
 } = primordials;
 
-const { basename } = require('path');
+const { basename, relative } = require('path');
 const { createWriteStream } = require('fs');
 const { pathToFileURL } = require('internal/url');
 const { createDeferredPromise } = require('internal/util');
 const { getOptionValue } = require('internal/options');
+const { green, red, white } = require('internal/util/colors');
 
 const {
   codes: {
@@ -246,6 +249,52 @@ function countCompletedTest(test, harness = test.root.harness) {
   harness.counters.all++;
 }
 
+
+function coverageThreshold(coverage, color) {
+  coverage = NumberPrototypeToFixed(coverage, 2);
+  if (color) {
+    if (coverage > 90) return `${green}${coverage}${color}`;
+    if (coverage < 50) return `${red}${coverage}${color}`;
+  }
+  return coverage;
+}
+
+function getCoverageReport(pad, summary, symbol, color) {
+  let report = `${color}${pad}${symbol}start of coverage report\n`;
+
+  report += `${pad}${symbol}file | line % | branch % | funcs % | uncovered lines\n`;
+
+  for (let i = 0; i < summary.files.length; ++i) {
+    const {
+      path,
+      coveredLinePercent,
+      coveredBranchPercent,
+      coveredFunctionPercent,
+      uncoveredLineNumbers,
+    } = summary.files[i];
+    const relativePath = relative(summary.workingDirectory, path);
+    const lines = coverageThreshold(coveredLinePercent, color);
+    const branches = coverageThreshold(coveredBranchPercent, color);
+    const functions = coverageThreshold(coveredFunctionPercent, color);
+    const uncovered = ArrayPrototypeJoin(uncoveredLineNumbers, ', ');
+
+    report += `${pad}${symbol}${relativePath} | ${lines} | ${branches} | ` +
+              `${functions} | ${uncovered}\n`;
+  }
+
+  const { totals } = summary;
+  report += `${pad}${symbol}all files | ` +
+            `${coverageThreshold(totals.coveredLinePercent, color)} | ` +
+            `${coverageThreshold(totals.coveredBranchPercent, color)} | ` +
+            `${coverageThreshold(totals.coveredFunctionPercent, color)} |\n`;
+
+  report += `${pad}${symbol}end of coverage report\n`;
+  if (color) {
+    report += white;
+  }
+  return report;
+}
+
 module.exports = {
   convertStringToRegExp,
   countCompletedTest,
@@ -255,4 +304,5 @@ module.exports = {
   isTestFailureError,
   parseCommandLine,
   setupTestReporters,
+  getCoverageReport,
 };
diff --git a/test/parallel/test-runner-coverage.js b/test/parallel/test-runner-coverage.js
@@ -18,7 +18,7 @@ function findCoverageFileForPid(pid) {
   });
 }
 
-function getCoverageFixtureReport() {
+function getTapCoverageFixtureReport() {
   const report = [
     '# start of coverage report',
     '# file | line % | branch % | funcs % | uncovered lines',
@@ -37,64 +37,120 @@ function getCoverageFixtureReport() {
   return report;
 }
 
-test('--experimental-test-coverage and --test cannot be combined', () => {
-  // TODO(cjihrig): This test can be removed once multi-process code coverage
-  // is supported.
-  const args = ['--test', '--experimental-test-coverage'];
-  const result = spawnSync(process.execPath, args);
-
-  // 9 is the documented exit code for an invalid CLI argument.
-  assert.strictEqual(result.status, 9);
-  assert.match(
-    result.stderr.toString(),
-    /--experimental-test-coverage cannot be used with --test/
-  );
-});
+function getSpecCoverageFixtureReport() {
+  const report = [
+    '\u2139 start of coverage report',
+    '\u2139 file | line % | branch % | funcs % | uncovered lines',
+    '\u2139 test/fixtures/test-runner/coverage.js | 78.65 | 38.46 | 60.00 | 12, ' +
+    '13, 16, 17, 18, 19, 20, 21, 22, 27, 39, 43, 44, 61, 62, 66, 67, 71, 72',
+    '\u2139 test/fixtures/test-runner/invalid-tap.js | 100.00 | 100.00 | 100.00 | ',
+    '\u2139 test/fixtures/v8-coverage/throw.js | 71.43 | 50.00 | 100.00 | 5, 6',
+    '\u2139 all files | 78.35 | 43.75 | 60.00 |',
+    '\u2139 end of coverage report',
+  ].join('\n');
 
-test('handles the inspector not being available', (t) => {
-  if (process.features.inspector) {
-    return;
+  if (common.isWindows) {
+    return report.replaceAll('/', '\\');
   }
 
-  const fixture = fixtures.path('test-runner', 'coverage.js');
-  const args = ['--experimental-test-coverage', fixture];
-  const result = spawnSync(process.execPath, args);
+  return report;
+}
 
-  assert(!result.stdout.toString().includes('# start of coverage report'));
-  assert(result.stderr.toString().includes('coverage could not be collected'));
-  assert.strictEqual(result.status, 0);
-  assert(!findCoverageFileForPid(result.pid));
-});
+test('test coverage report', async (t) => {
+  await t.test('--experimental-test-coverage and --test cannot be combined', () => {
+    // TODO(cjihrig): This test can be removed once multi-process code coverage
+    // is supported.
+    const args = ['--test', '--experimental-test-coverage'];
+    const result = spawnSync(process.execPath, args);
+
+    // 9 is the documented exit code for an invalid CLI argument.
+    assert.strictEqual(result.status, 9);
+    assert.match(
+      result.stderr.toString(),
+      /--experimental-test-coverage cannot be used with --test/
+    );
+  });
 
-test('coverage is reported and dumped to NODE_V8_COVERAGE if present', (t) => {
-  if (!process.features.inspector) {
-    return;
-  }
+  await t.test('handles the inspector not being available', (t) => {
+    if (process.features.inspector) {
+      return;
+    }
 
-  const fixture = fixtures.path('test-runner', 'coverage.js');
-  const args = ['--experimental-test-coverage', fixture];
-  const options = { env: { ...process.env, NODE_V8_COVERAGE: tmpdir.path } };
-  const result = spawnSync(process.execPath, args, options);
-  const report = getCoverageFixtureReport();
+    const fixture = fixtures.path('test-runner', 'coverage.js');
+    const args = ['--experimental-test-coverage', fixture];
+    const result = spawnSync(process.execPath, args);
 
-  assert(result.stdout.toString().includes(report));
-  assert.strictEqual(result.stderr.toString(), '');
-  assert.strictEqual(result.status, 0);
-  assert(findCoverageFileForPid(result.pid));
+    assert(!result.stdout.toString().includes('# start of coverage report'));
+    assert(result.stderr.toString().includes('coverage could not be collected'));
+    assert.strictEqual(result.status, 0);
+    assert(!findCoverageFileForPid(result.pid));
+  });
 });
 
-test('coverage is reported without NODE_V8_COVERAGE present', (t) => {
-  if (!process.features.inspector) {
-    return;
-  }
+test('test tap coverage reporter', async (t) => {
+  await t.test('coverage is reported and dumped to NODE_V8_COVERAGE if present', (t) => {
+    if (!process.features.inspector) {
+      return;
+    }
+
+    const fixture = fixtures.path('test-runner', 'coverage.js');
+    const args = ['--experimental-test-coverage', '--test-reporter', 'tap', fixture];
+    const options = { env: { ...process.env, NODE_V8_COVERAGE: tmpdir.path } };
+    const result = spawnSync(process.execPath, args, options);
+    const report = getTapCoverageFixtureReport();
+
+    assert(result.stdout.toString().includes(report));
+    assert.strictEqual(result.stderr.toString(), '');
+    assert.strictEqual(result.status, 0);
+    assert(findCoverageFileForPid(result.pid));
+  });
+
+  await t.test('coverage is reported without NODE_V8_COVERAGE present', (t) => {
+    if (!process.features.inspector) {
+      return;
+    }
+
+    const fixture = fixtures.path('test-runner', 'coverage.js');
+    const args = ['--experimental-test-coverage', '--test-reporter', 'tap', fixture];
+    const result = spawnSync(process.execPath, args);
+    const report = getTapCoverageFixtureReport();
 
-  const fixture = fixtures.path('test-runner', 'coverage.js');
-  const args = ['--experimental-test-coverage', fixture];
-  const result = spawnSync(process.execPath, args);
-  const report = getCoverageFixtureReport();
+    assert(result.stdout.toString().includes(report));
+    assert.strictEqual(result.stderr.toString(), '');
+    assert.strictEqual(result.status, 0);
+    assert(!findCoverageFileForPid(result.pid));
+  });
+});
 
-  assert(result.stdout.toString().includes(report));
-  assert.strictEqual(result.stderr.toString(), '');
-  assert.strictEqual(result.status, 0);
-  assert(!findCoverageFileForPid(result.pid));
+test('test spec coverage reporter', async (t) => {
+  await t.test('coverage is reported and dumped to NODE_V8_COVERAGE if present', (t) => {
+    if (!process.features.inspector) {
+      return;
+    }
+    const fixture = fixtures.path('test-runner', 'coverage.js');
+    const args = ['--experimental-test-coverage', '--test-reporter', 'spec', fixture];
+    const options = { env: { ...process.env, NODE_V8_COVERAGE: tmpdir.path } };
+    const result = spawnSync(process.execPath, args, options);
+    const report = getSpecCoverageFixtureReport();
+
+    assert(result.stdout.toString().includes(report));
+    assert.strictEqual(result.stderr.toString(), '');
+    assert.strictEqual(result.status, 0);
+    assert(findCoverageFileForPid(result.pid));
+  });
+
+  await t.test('coverage is reported without NODE_V8_COVERAGE present', (t) => {
+    if (!process.features.inspector) {
+      return;
+    }
+    const fixture = fixtures.path('test-runner', 'coverage.js');
+    const args = ['--experimental-test-coverage', '--test-reporter', 'spec', fixture];
+    const result = spawnSync(process.execPath, args);
+    const report = getSpecCoverageFixtureReport();
+
+    assert(result.stdout.toString().includes(report));
+    assert.strictEqual(result.stderr.toString(), '');
+    assert.strictEqual(result.status, 0);
+    assert(!findCoverageFileForPid(result.pid));
+  });
 });