test_runner: support programmatically running --test

PR-URL: nodejs/node#44241
Fixes: nodejs/node#44023
Fixes: nodejs/node#43675
Reviewed-By: Benjamin Gruenbaum <[email protected]>
Reviewed-By: Antoine du Hamel <[email protected]>
(cherry picked from commit 59527de13d39327eb3dfa8dedc92241eb40066d5)

Author: MoLow

README.md

@@ -324,6 +324,35 @@ Otherwise, the test is considered to be a failure. Test files must be
 executable by Node.js, but are not required to use the `node:test` module
 internally.
 
+## `run([options])`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* `options` {Object} Configuration options for running tests. The following
+  properties are supported:
+  * `concurrency` {number|boolean} If a number is provided,
+    then that many files would run in parallel.
+    If truthy, it would run (number of cpu cores - 1)
+    files in parallel.
+    If falsy, it would only run one file at a time.
+    If unspecified, subtests inherit this value from their parent.
+    **Default:** `true`.
+  * `files`: {Array} An array containing the list of files to run.
+    **Default** matching files from [test runner execution model][].
+  * `signal` {AbortSignal} Allows aborting an in-progress test execution.
+  * `timeout` {number} A number of milliseconds the test execution will
+    fail after.
+    If unspecified, subtests inherit this value from their parent.
+    **Default:** `Infinity`.
+* Returns: {TapStream}
+
+```js
+run({ files: [path.resolve('./tests/test.js')] })
+  .pipe(process.stdout);
+```
+
 ## `test([name][, options][, fn])`
 
 - `name` {string} The name of the test, which is displayed when reporting test
@@ -541,6 +570,47 @@ describe('tests', async () => {
 });
 ```
 
+## Class: `TapStream`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* Extends {ReadableStream}
+
+A successful call to [`run()`][] method will return a new {TapStream}
+object, streaming a [TAP][] output
+`TapStream` will emit events, in the order of the tests definition
+
+### Event: `'test:diagnostic'`
+
+* `message` {string} The diagnostic message.
+
+Emitted when [`context.diagnostic`][] is called.
+
+### Event: `'test:fail'`
+
+* `data` {Object}
+  * `duration` {number} The test duration.
+  * `error` {Error} The failure casing test to fail.
+  * `name` {string} The test name.
+  * `testNumber` {number} The ordinal number of the test.
+  * `todo` {string|undefined} Present if [`context.todo`][] is called
+  * `skip` {string|undefined} Present if [`context.skip`][] is called
+
+Emitted when a test fails.
+
+### Event: `'test:pass'`
+
+* `data` {Object}
+  * `duration` {number} The test duration.
+  * `name` {string} The test name.
+  * `testNumber` {number} The ordinal number of the test.
+  * `todo` {string|undefined} Present if [`context.todo`][] is called
+  * `skip` {string|undefined} Present if [`context.skip`][] is called
+
+Emitted when a test passes.
+
 ## Class: `TestContext`
 
 An instance of `TestContext` is passed to each test function in order to
@@ -712,6 +782,10 @@ The name of the suite.
 [TAP]: https://testanything.org/
 [`SuiteContext`]: #class-suitecontext
 [`TestContext`]: #class-testcontext
+[`context.diagnostic`]: #contextdiagnosticmessage
+[`context.skip`]: #contextskipmessage
+[`context.todo`]: #contexttodomessage
+[`run()`]: #runoptions
 [`test()`]: #testname-options-fn
 [describe options]: #describename-options-fn
 [it options]: #testname-options-fn

lib/internal/main/test_runner.js

@@ -1,148 +1,15 @@
-// https://github.com/nodejs/node/blob/2fd4c013c221653da2a7921d08fe1aa96aaba504/lib/internal/main/test_runner.js
+// https://github.com/nodejs/node/blob/59527de13d39327eb3dfa8dedc92241eb40066d5/lib/internal/main/test_runner.js
 'use strict'
-const {
-  ArrayFrom,
-  ArrayPrototypeFilter,
-  ArrayPrototypeIncludes,
-  ArrayPrototypeJoin,
-  ArrayPrototypePush,
-  ArrayPrototypeSlice,
-  ArrayPrototypeSort,
-  SafePromiseAll,
-  SafeSet
-} = require('#internal/per_context/primordials')
 const {
   prepareMainThreadExecution
-} = require('#internal/bootstrap/pre_execution')
-const { spawn } = require('child_process')
-const { readdirSync, statSync } = require('fs')
-const {
-  codes: {
-    ERR_TEST_FAILURE
-  }
-} = require('#internal/errors')
-const { toArray } = require('#internal/streams/operators').promiseReturningOperators
-const { test } = require('#internal/test_runner/harness')
-const { kSubtestsFailed } = require('#internal/test_runner/test')
-const {
-  isSupportedFileType,
-  doesPathMatchFilter
-} = require('#internal/test_runner/utils')
-const { basename, join, resolve } = require('path')
-const { once } = require('events')
-const kFilterArgs = ['--test']
+} = require('#internal/process/pre_execution')
+const { run } = require('#internal/test_runner/runner')
 
 prepareMainThreadExecution(false)
 // markBootstrapComplete();
 
-// TODO(cjihrig): Replace this with recursive readdir once it lands.
-function processPath (path, testFiles, options) {
-  const stats = statSync(path)
-
-  if (stats.isFile()) {
-    if (options.userSupplied ||
-        (options.underTestDir && isSupportedFileType(path)) ||
-        doesPathMatchFilter(path)) {
-      testFiles.add(path)
-    }
-  } else if (stats.isDirectory()) {
-    const name = basename(path)
-
-    if (!options.userSupplied && name === 'node_modules') {
-      return
-    }
-
-    // 'test' directories get special treatment. Recursively add all .js,
-    // .cjs, and .mjs files in the 'test' directory.
-    const isTestDir = name === 'test'
-    const { underTestDir } = options
-    const entries = readdirSync(path)
-
-    if (isTestDir) {
-      options.underTestDir = true
-    }
-
-    options.userSupplied = false
-
-    for (let i = 0; i < entries.length; i++) {
-      processPath(join(path, entries[i]), testFiles, options)
-    }
-
-    options.underTestDir = underTestDir
-  }
-}
-
-function createTestFileList () {
-  const cwd = process.cwd()
-  const hasUserSuppliedPaths = process.argv.length > 1
-  const testPaths = hasUserSuppliedPaths
-    ? ArrayPrototypeSlice(process.argv, 1)
-    : [cwd]
-  const testFiles = new SafeSet()
-
-  try {
-    for (let i = 0; i < testPaths.length; i++) {
-      const absolutePath = resolve(testPaths[i])
-
-      processPath(absolutePath, testFiles, { userSupplied: true })
-    }
-  } catch (err) {
-    if (err?.code === 'ENOENT') {
-      console.error(`Could not find '${err.path}'`)
-      process.exit(1)
-    }
-
-    throw err
-  }
-
-  return ArrayPrototypeSort(ArrayFrom(testFiles))
-}
-
-function filterExecArgv (arg) {
-  return !ArrayPrototypeIncludes(kFilterArgs, arg)
-}
-
-function runTestFile (path) {
-  return test(path, async (t) => {
-    const args = ArrayPrototypeFilter(process.execArgv, filterExecArgv)
-    ArrayPrototypePush(args, path)
-
-    const child = spawn(process.execPath, args, { signal: t.signal, encoding: 'utf8' })
-    // TODO(cjihrig): Implement a TAP parser to read the child's stdout
-    // instead of just displaying it all if the child fails.
-    let err
-
-    child.on('error', (error) => {
-      err = error
-    })
-
-    const { 0: { 0: code, 1: signal }, 1: stdout, 2: stderr } = await SafePromiseAll([
-      once(child, 'exit', { signal: t.signal }),
-      toArray.call(child.stdout, { signal: t.signal }),
-      toArray.call(child.stderr, { signal: t.signal })
-    ])
-
-    if (code !== 0 || signal !== null) {
-      if (!err) {
-        err = new ERR_TEST_FAILURE('test failed', kSubtestsFailed)
-        err.exitCode = code
-        err.signal = signal
-        err.stdout = ArrayPrototypeJoin(stdout, '')
-        err.stderr = ArrayPrototypeJoin(stderr, '')
-        // The stack will not be useful since the failures came from tests
-        // in a child process.
-        err.stack = undefined
-      }
-
-      throw err
-    }
-  })
-}
-
-;(async function main () {
-  const testFiles = createTestFileList()
-
-  for (let i = 0; i < testFiles.length; i++) {
-    runTestFile(testFiles[i])
-  }
-})()
+const tapStream = run()
+tapStream.pipe(process.stdout)
+tapStream.once('test:fail', () => {
+  process.exitCode = 1
+})

lib/internal/per_context/primordials.js

@@ -3,6 +3,8 @@
 const replaceAll = require('string.prototype.replaceall')
 
 exports.ArrayFrom = (it, mapFn) => Array.from(it, mapFn)
+exports.ArrayIsArray = Array.isArray
+exports.ArrayPrototypeConcat = (arr, ...el) => arr.concat(...el)
 exports.ArrayPrototypeFilter = (arr, fn) => arr.filter(fn)
 exports.ArrayPrototypeForEach = (arr, fn, thisArg) => arr.forEach(fn, thisArg)
 exports.ArrayPrototypeIncludes = (arr, el, fromIndex) => arr.includes(el, fromIndex)
@@ -20,6 +22,7 @@ exports.FunctionPrototype = Function.prototype
 exports.FunctionPrototypeBind = (fn, obj, ...args) => fn.bind(obj, ...args)
 exports.MathMax = (...args) => Math.max(...args)
 exports.Number = Number
+exports.ObjectAssign = (target, ...sources) => Object.assign(target, ...sources)
 exports.ObjectCreate = obj => Object.create(obj)
 exports.ObjectDefineProperties = (obj, props) => Object.defineProperties(obj, props)
 exports.ObjectDefineProperty = (obj, key, descr) => Object.defineProperty(obj, key, descr)
@@ -41,6 +44,7 @@ exports.SafePromiseAll = (array, mapFn) => Promise.all(mapFn ? array.map(mapFn)
 exports.SafePromiseRace = (array, mapFn) => Promise.race(mapFn ? array.map(mapFn) : array)
 exports.SafeSet = Set
 exports.SafeWeakMap = WeakMap
+exports.SafeWeakSet = WeakSet
 exports.StringPrototypeIncludes = (str, needle) => str.includes(needle)
 exports.StringPrototypeMatch = (str, reg) => str.match(reg)
 exports.StringPrototypeReplace = (str, search, replacement) =>