[v16.x backport] test runner additions

doc/api/test.md

@@ -316,6 +316,40 @@ Otherwise, the test is considered to be a failure. Test files must be
 executable by Node.js, but are not required to use the `node:test` module
 internally.
 
+## `run([options])`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* `options` {Object} Configuration options for running tests. The following
+  properties are supported:
+  * `concurrency` {number|boolean} If a number is provided,
+    then that many files would run in parallel.
+    If truthy, it would run (number of cpu cores - 1)
+    files in parallel.
+    If falsy, it would only run one file at a time.
+    If unspecified, subtests inherit this value from their parent.
+    **Default:** `true`.
+  * `files`: {Array} An array containing the list of files to run.
+    **Default** matching files from [test runner execution model][].
+  * `signal` {AbortSignal} Allows aborting an in-progress test execution.
+  * `timeout` {number} A number of milliseconds the test execution will
+    fail after.
+    If unspecified, subtests inherit this value from their parent.
+    **Default:** `Infinity`.
+  * `inspectPort` {number|Function} Sets inspector port of test child process.
+    This can be a number, or a function that takes no arguments and returns a
+    number. If a nullish value is provided, each process gets its own port,
+    incremented from the primary's `process.debugPort`.
+    **Default:** `undefined`.
+* Returns: {TapStream}
+
+```js
+run({ files: [path.resolve('./tests/test.js')] })
+  .pipe(process.stdout);
+```
+
 ## `test([name][, options][, fn])`
 
 <!-- YAML
@@ -560,6 +594,47 @@ describe('tests', async () => {
 });
 ```
 
+## Class: `TapStream`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* Extends {ReadableStream}
+
+A successful call to [`run()`][] method will return a new {TapStream}
+object, streaming a [TAP][] output
+`TapStream` will emit events, in the order of the tests definition
+
+### Event: `'test:diagnostic'`
+
+* `message` {string} The diagnostic message.
+
+Emitted when [`context.diagnostic`][] is called.
+
+### Event: `'test:fail'`
+
+* `data` {Object}
+  * `duration` {number} The test duration.
+  * `error` {Error} The failure casing test to fail.
+  * `name` {string} The test name.
+  * `testNumber` {number} The ordinal number of the test.
+  * `todo` {string|undefined} Present if [`context.todo`][] is called
+  * `skip` {string|undefined} Present if [`context.skip`][] is called
+
+Emitted when a test fails.
+
+### Event: `'test:pass'`
+
+* `data` {Object}
+  * `duration` {number} The test duration.
+  * `name` {string} The test name.
+  * `testNumber` {number} The ordinal number of the test.
+  * `todo` {string|undefined} Present if [`context.todo`][] is called
+  * `skip` {string|undefined} Present if [`context.skip`][] is called
+
+Emitted when a test passes.
+
 ## Class: `TestContext`
 
 <!-- YAML
@@ -825,6 +900,10 @@ added: v16.17.0
 [`--test`]: cli.md#--test
 [`SuiteContext`]: #class-suitecontext
 [`TestContext`]: #class-testcontext
+[`context.diagnostic`]: #contextdiagnosticmessage
+[`context.skip`]: #contextskipmessage
+[`context.todo`]: #contexttodomessage
+[`run()`]: #runoptions
 [`test()`]: #testname-options-fn
 [describe options]: #describename-options-fn
 [it options]: #testname-options-fn

lib/internal/cluster/primary.js

@@ -6,10 +6,14 @@ const {
   ArrayPrototypeSome,
   ObjectKeys,
   ObjectValues,
-  RegExpPrototypeExec,
   SafeMap,
   StringPrototypeStartsWith,
 } = primordials;
+const {
+  codes: {
+    ERR_SOCKET_BAD_PORT,
+  }
+} = require('internal/errors');
 
 const assert = require('internal/assert');
 const { fork } = require('child_process');
@@ -18,14 +22,12 @@ const EventEmitter = require('events');
 const RoundRobinHandle = require('internal/cluster/round_robin_handle');
 const SharedHandle = require('internal/cluster/shared_handle');
 const Worker = require('internal/cluster/worker');
+const { getInspectPort, isUsingInspector } = require('internal/util/inspector');
 const { internal, sendHelper } = require('internal/cluster/utils');
 const cluster = new EventEmitter();
 const intercom = new EventEmitter();
 const SCHED_NONE = 1;
 const SCHED_RR = 2;
-const minPort = 1024;
-const maxPort = 65535;
-const { validatePort } = require('internal/validators');
 
 module.exports = cluster;
 
@@ -40,7 +42,6 @@ cluster.SCHED_NONE = SCHED_NONE;  // Leave it to the operating system.
 cluster.SCHED_RR = SCHED_RR;      // Primary distributes connections.
 
 let ids = 0;
-let debugPortOffset = 1;
 let initialized = false;
 
 // XXX(bnoordhuis) Fold cluster.schedulingPolicy into cluster.settings?
@@ -117,28 +118,12 @@ function setupSettingsNT(settings) {
 function createWorkerProcess(id, env) {
   const workerEnv = { ...process.env, ...env, NODE_UNIQUE_ID: `${id}` };
   const execArgv = [...cluster.settings.execArgv];
-  const debugArgRegex = /--inspect(?:-brk|-port)?|--debug-port/;
-  const nodeOptions = process.env.NODE_OPTIONS || '';
-
-  if (ArrayPrototypeSome(execArgv,
-                         (arg) => RegExpPrototypeExec(debugArgRegex, arg) !== null) ||
-      RegExpPrototypeExec(debugArgRegex, nodeOptions) !== null) {
-    let inspectPort;
-    if ('inspectPort' in cluster.settings) {
-      if (typeof cluster.settings.inspectPort === 'function')
-        inspectPort = cluster.settings.inspectPort();
-      else
-        inspectPort = cluster.settings.inspectPort;
-
-      validatePort(inspectPort);
-    } else {
-      inspectPort = process.debugPort + debugPortOffset;
-      if (inspectPort > maxPort)
-        inspectPort = inspectPort - maxPort + minPort - 1;
-      debugPortOffset++;
-    }
 
-    ArrayPrototypePush(execArgv, `--inspect-port=${inspectPort}`);
+  if (cluster.settings.inspectPort === null) {
+    throw new ERR_SOCKET_BAD_PORT('Port', null, true);
+  }
+  if (isUsingInspector(cluster.settings.execArgv)) {
+    ArrayPrototypePush(execArgv, `--inspect-port=${getInspectPort(cluster.settings.inspectPort)}`);
   }
 
   return fork(cluster.settings.exec, cluster.settings.args, {

lib/internal/main/test_runner.js

@@ -1,146 +1,25 @@
 'use strict';
-const {
-  ArrayFrom,
-  ArrayPrototypeFilter,
-  ArrayPrototypeIncludes,
-  ArrayPrototypeJoin,
-  ArrayPrototypePush,
-  ArrayPrototypeSlice,
-  ArrayPrototypeSort,
-  SafePromiseAll,
-  SafeSet,
-} = primordials;
 const {
   prepareMainThreadExecution,
 } = require('internal/bootstrap/pre_execution');
-const { spawn } = require('child_process');
-const { readdirSync, statSync } = require('fs');
-const console = require('internal/console/global');
-const {
-  codes: {
-    ERR_TEST_FAILURE,
-  },
-} = require('internal/errors');
-const { test } = require('internal/test_runner/harness');
-const { kSubtestsFailed } = require('internal/test_runner/test');
-const {
-  isSupportedFileType,
-  doesPathMatchFilter,
-} = require('internal/test_runner/utils');
-const { basename, join, resolve } = require('path');
-const { once } = require('events');
-const kFilterArgs = ['--test'];
+const { isUsingInspector } = require('internal/util/inspector');
+const { run } = require('internal/test_runner/runner');
 
 prepareMainThreadExecution(false);
 markBootstrapComplete();
 
-// TODO(cjihrig): Replace this with recursive readdir once it lands.
-function processPath(path, testFiles, options) {
-  const stats = statSync(path);
-
-  if (stats.isFile()) {
-    if (options.userSupplied ||
-        (options.underTestDir && isSupportedFileType(path)) ||
-        doesPathMatchFilter(path)) {
-      testFiles.add(path);
-    }
-  } else if (stats.isDirectory()) {
-    const name = basename(path);
-
-    if (!options.userSupplied && name === 'node_modules') {
-      return;
-    }
-
-    // 'test' directories get special treatment. Recursively add all .js,
-    // .cjs, and .mjs files in the 'test' directory.
-    const isTestDir = name === 'test';
-    const { underTestDir } = options;
-    const entries = readdirSync(path);
-
-    if (isTestDir) {
-      options.underTestDir = true;
-    }
+let concurrency = true;
+let inspectPort;
 
-    options.userSupplied = false;
-
-    for (let i = 0; i < entries.length; i++) {
-      processPath(join(path, entries[i]), testFiles, options);
-    }
-
-    options.underTestDir = underTestDir;
-  }
+if (isUsingInspector()) {
+  process.emitWarning('Using the inspector with --test forces running at a concurrency of 1. ' +
+  'Use the inspectPort option to run with concurrency');
+  concurrency = 1;
+  inspectPort = process.debugPort;
 }
 
-function createTestFileList() {
-  const cwd = process.cwd();
-  const hasUserSuppliedPaths = process.argv.length > 1;
-  const testPaths = hasUserSuppliedPaths ?
-    ArrayPrototypeSlice(process.argv, 1) : [cwd];
-  const testFiles = new SafeSet();
-
-  try {
-    for (let i = 0; i < testPaths.length; i++) {
-      const absolutePath = resolve(testPaths[i]);
-
-      processPath(absolutePath, testFiles, { userSupplied: true });
-    }
-  } catch (err) {
-    if (err?.code === 'ENOENT') {
-      console.error(`Could not find '${err.path}'`);
-      process.exit(1);
-    }
-
-    throw err;
-  }
-
-  return ArrayPrototypeSort(ArrayFrom(testFiles));
-}
-
-function filterExecArgv(arg) {
-  return !ArrayPrototypeIncludes(kFilterArgs, arg);
-}
-
-function runTestFile(path) {
-  return test(path, async (t) => {
-    const args = ArrayPrototypeFilter(process.execArgv, filterExecArgv);
-    ArrayPrototypePush(args, path);
-
-    const child = spawn(process.execPath, args, { signal: t.signal, encoding: 'utf8' });
-    // TODO(cjihrig): Implement a TAP parser to read the child's stdout
-    // instead of just displaying it all if the child fails.
-    let err;
-
-    child.on('error', (error) => {
-      err = error;
-    });
-
-    const { 0: { 0: code, 1: signal }, 1: stdout, 2: stderr } = await SafePromiseAll([
-      once(child, 'exit', { signal: t.signal }),
-      child.stdout.toArray({ signal: t.signal }),
-      child.stderr.toArray({ signal: t.signal }),
-    ]);
-
-    if (code !== 0 || signal !== null) {
-      if (!err) {
-        err = new ERR_TEST_FAILURE('test failed', kSubtestsFailed);
-        err.exitCode = code;
-        err.signal = signal;
-        err.stdout = ArrayPrototypeJoin(stdout, '');
-        err.stderr = ArrayPrototypeJoin(stderr, '');
-        // The stack will not be useful since the failures came from tests
-        // in a child process.
-        err.stack = undefined;
-      }
-
-      throw err;
-    }
-  });
-}
-
-(async function main() {
-  const testFiles = createTestFileList();
-
-  for (let i = 0; i < testFiles.length; i++) {
-    runTestFile(testFiles[i]);
-  }
-})();
+const tapStream = run({ concurrency, inspectPort });
+tapStream.pipe(process.stdout);
+tapStream.once('test:fail', () => {
+  process.exitCode = 1;
+});