test_runner: add option to rerun only failed tests

Author: MoLow

doc/api/cli.md

@@ -2647,6 +2647,20 @@ changes:
 The destination for the corresponding test reporter. See the documentation on
 [test reporters][] for more details.
 
+### `--test-rerun`
+
+<!-- YAML
+added:
+  - REPLACEME
+-->
+
+A path to a file allowing the test runner to persist the state of the test
+suite between runs. The test runner will use this file to determine which tests
+have already succeeded or failed, allowing for re-running of failed tests
+without having to re-run the entire test suite. The test runner will create this
+file if it does not exist.
+See the documentation on [test reruns][] for more details.
+
 ### `--test-shard`
 
 <!-- YAML
@@ -3508,6 +3522,7 @@ one is included in the list below.
 * `--test-only`
 * `--test-reporter-destination`
 * `--test-reporter`
+* `--test-rerun`
 * `--test-shard`
 * `--test-skip-pattern`
 * `--throw-deprecation`
@@ -4082,6 +4097,7 @@ node --stack-trace-limit=12 -p -e "Error.stackTraceLimit" # prints 12
 [snapshot testing]: test.md#snapshot-testing
 [syntax detection]: packages.md#syntax-detection
 [test reporters]: test.md#test-reporters
+[test reruns]: test.md#rerunning-failed-tests
 [test runner execution model]: test.md#test-runner-execution-model
 [timezone IDs]: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
 [tracking issue for user-land snapshots]: https://github.com/nodejs/node/issues/44014

doc/api/test.md

@@ -153,6 +153,41 @@ test('skip() method with message', (t) => {
 });
 ```
 
+## Rerunning failed tests
+
+The test runner supports persisting the state of the run to a file, allowing
+the test runner to rerun failed tests without having to re-run the entire test suite.
+Use the [`--test-rerun`][] command-line option to specify a file path where the
+state of the run is stored. if the state file does not exist, the test runner will
+create it.
+the state file is a JSON file that contains an array of run attempts.
+Each run attempt is an object mapping successful tests to the attempt they have passed in.
+The key of a test in this map is the test file path, and the line and column where the test is defined,
+meaning the state file is valid as long as the test definition does not change.
+
+example of a state file:
+
+```json
+[
+  {
+    "test.js:10:5": 0,
+  },
+  {
+    "test.js:10:5": 0,
+    "test.js:20:5": 1
+  }
+]
+```
+
+in this example, there are two run attempts, with two tests defined in `test.js`,
+the first test succeeded on the first attempt, and the second test succeeded on the second attempt.
+
+When the `--test-rerun` option is used, the test runner will only run tests that have not yet passed.
+
+```bash
+node --test-rerun /path/to/state/file
+```
+
 ## TODO tests
 
 Individual tests can be marked as flaky or incomplete by passing the `todo`
@@ -3202,6 +3237,8 @@ Emitted when a test is enqueued for execution.
       * `cause` {Error} The actual error thrown by the test.
     * `type` {string|undefined} The type of the test, used to denote whether
       this is a suite.
+    * `attempt` {number|undefined} The attempt number of the test run,
+      present only when using the [`--test-rerun`][] flag.
   * `file` {string|undefined} The path of the test file,
     `undefined` if test was run through the REPL.
   * `line` {number|undefined} The line number where the test is defined, or
@@ -3226,6 +3263,10 @@ The corresponding execution ordered event is `'test:complete'`.
     * `duration_ms` {number} The duration of the test in milliseconds.
     * `type` {string|undefined} The type of the test, used to denote whether
       this is a suite.
+    * `attempt` {number|undefined} The attempt number of the test run,
+      present only when using the [`--test-rerun`][] flag.
+    * `passed_attempt` {number|undefined} The attempt number the test passed on,
+      present only when using the [`--test-rerun`][] flag.
   * `file` {string|undefined} The path of the test file,
     `undefined` if test was run through the REPL.
   * `line` {number|undefined} The line number where the test is defined, or
@@ -3929,6 +3970,7 @@ Can be used to abort test subtasks when the test has been aborted.
 [`--test-only`]: cli.md#--test-only
 [`--test-reporter-destination`]: cli.md#--test-reporter-destination
 [`--test-reporter`]: cli.md#--test-reporter
+[`--test-rerun`]: cli.md#--test-rerun
 [`--test-skip-pattern`]: cli.md#--test-skip-pattern
 [`--test-update-snapshots`]: cli.md#--test-update-snapshots
 [`--test`]: cli.md#--test

doc/node-config-schema.json

@@ -446,6 +446,9 @@
             }
           ]
         },
+        "test-rerun": {
+          "type": "string"
+        },
         "test-shard": {
           "type": "string"
         },
@@ -698,6 +701,9 @@
             }
           ]
         },
+        "test-rerun": {
+          "type": "string"
+        },
         "test-shard": {
           "type": "string"
         },

doc/node.1

@@ -493,6 +493,10 @@ A test reporter to use when running tests.
 .It Fl -test-reporter-destination
 The destination for the corresponding test reporter.
 .
+.It Fl -test-rerun
+Configures the tests runner to persist the state of tests to allow
+rerunning only failed tests.
+.
 .It Fl -test-only
 Configures the test runner to only execute top level tests that have the `only`
 option set.

lib/internal/test_runner/reporter/rerun.js

@@ -0,0 +1,29 @@
+'use strict';
+
+const {
+  ArrayPrototypePush,
+  JSONStringify,
+} = primordials;
+const { relative } = require('path');
+const { writeFileSync } = require('fs');
+
+function reportReruns(previousRuns, rerunFilePath) {
+  const cwd = process.cwd();
+  return async function reporter(source) {
+    const obj = { __proto__: null };
+
+    for await (const { type, data } of source) {
+      if (type === 'test:pass') {
+        obj[`${relative(cwd, data.file)}:${data.line}:${data.column}`] = data.details.passed_attempt ?? data.details.attempt;
+      }
+    }
+
+    ArrayPrototypePush(previousRuns, obj);
+    writeFileSync(rerunFilePath, JSONStringify(previousRuns, null, 2), 'utf8');
+  };
+};
+
+module.exports = {
+  __proto__: null,
+  reportReruns,
+};

lib/internal/test_runner/runner.js

@@ -148,6 +148,7 @@ function getRunArgs(path, { forceExit,
                             only,
                             argv: suppliedArgs,
                             execArgv,
+                            rerunFilePath,
                             root: { timeout },
                             cwd }) {
   const argv = ArrayPrototypeFilter(process.execArgv, filterExecArgv);
@@ -169,6 +170,9 @@ function getRunArgs(path, { forceExit,
   if (timeout != null) {
     ArrayPrototypePush(argv, `--test-timeout=${timeout}`);
   }
+  if (rerunFilePath != null) {
+    ArrayPrototypePush(argv, `--test-rerun=${rerunFilePath}`);
+  }
 
   ArrayPrototypePushApply(argv, execArgv);
 

lib/internal/test_runner/test.js

@@ -71,6 +71,7 @@ const {
 } = require('timers');
 const { TIMEOUT_MAX } = require('internal/timers');
 const { fileURLToPath } = require('internal/url');
+const { relative } = require('path');
 const { availableParallelism } = require('os');
 const { innerOk } = require('internal/assert/utils');
 const { bigint: hrtime } = process.hrtime;
@@ -290,6 +291,10 @@ class TestContext {
     return this.#test.passed;
   }
 
+  get attempt() {
+    return this.#test.attempt ?? 0;
+  }
+
   diagnostic(message) {
     this.#test.diagnostic(message);
   }
@@ -646,6 +651,8 @@ class Test extends AsyncResource {
     this.endTime = null;
     this.passed = false;
     this.error = null;
+    this.attempt = undefined;
+    this.passedAttempt = undefined;
     this.message = typeof skip === 'string' ? skip :
       typeof todo === 'string' ? todo : null;
     this.activeSubtests = 0;
@@ -690,6 +697,16 @@ class Test extends AsyncResource {
         this.loc.file = fileURLToPath(this.loc.file);
       }
     }
+
+    if (this.loc != null && this.config.previousRuns != null) {
+      const testLocation = `${relative(process.cwd(), this.loc.file)}:${this.loc.line}:${this.loc.column}`;
+      this.attempt = this.config.previousRuns.length;
+      const previousAttempt = this.config.previousRuns[this.attempt - 1]?.[testLocation];
+      if (previousAttempt != null) {
+        this.passedAttempt = previousAttempt;
+        this.fn = noop;
+      }
+    }
   }
 
   applyFilters() {
@@ -1329,6 +1346,12 @@ class Test extends AsyncResource {
     if (!this.passed) {
       details.error = this.error;
     }
+    if (this.attempt !== undefined) {
+      details.attempt = this.attempt;
+    }
+    if (this.passedAttempt !== undefined) {
+      details.passed_attempt = this.passedAttempt;
+    }
     return { __proto__: null, details, directive };
   }
 

lib/internal/test_runner/utils.js

@@ -8,6 +8,7 @@ const {
   ArrayPrototypePush,
   ArrayPrototypeReduce,
   ArrayPrototypeSome,
+  JSONParse,
   MathFloor,
   MathMax,
   MathMin,
@@ -28,9 +29,10 @@ const {
 
 const { AsyncResource } = require('async_hooks');
 const { relative, sep, resolve } = require('path');
-const { createWriteStream } = require('fs');
+const { createWriteStream, readFileSync } = require('fs');
 const { pathToFileURL } = require('internal/url');
 const { getOptionValue } = require('internal/options');
+const { reportReruns } = require('internal/test_runner/reporter/rerun');
 const { green, yellow, red, white, shouldColorize } = require('internal/util/colors');
 
 const {
@@ -40,7 +42,7 @@ const {
   },
   kIsNodeError,
 } = require('internal/errors');
-const { compose } = require('stream');
+const { compose, PassThrough } = require('stream');
 const {
   validateInteger,
   validateFunction,
@@ -150,6 +152,20 @@ function shouldColorizeTestFiles(destinations) {
   });
 }
 
+function parsePreviousRuns(rerunFilePath) {
+  let data;
+  try {
+    data = readFileSync(rerunFilePath, 'utf8');
+  } catch (err) {
+    if (err.code === 'ENOENT') {
+      data = '[]';
+    } else {
+      throw err;
+    }
+  }
+  return JSONParse(data);
+}
+
 async function getReportersMap(reporters, destinations) {
   return SafePromiseAllReturnArrayLike(reporters, async (name, i) => {
     const destination = kBuiltinDestinations.get(destinations[i]) ??
@@ -202,6 +218,7 @@ function parseCommandLine() {
   const updateSnapshots = getOptionValue('--test-update-snapshots');
   const watch = getOptionValue('--watch');
   const timeout = getOptionValue('--test-timeout') || Infinity;
+  const rerunFilePath = getOptionValue('--test-rerun');
   const isChildProcess = process.env.NODE_TEST_CONTEXT === 'child';
   const isChildProcessV8 = process.env.NODE_TEST_CONTEXT === 'child-v8';
   let globalSetupPath;
@@ -308,8 +325,24 @@ function parseCommandLine() {
     validateInteger(functionCoverage, '--test-coverage-functions', 0, 100);
   }
 
+  let previousRuns;
+  if (rerunFilePath) {
+    validatePath(rerunFilePath, '--test-rerun');
+    previousRuns = parsePreviousRuns(rerunFilePath);
+    if (previousRuns === null) {
+      throw new ERR_INVALID_ARG_VALUE('--test-rerun', rerunFilePath, 'is not a valid rerun file');
+    }
+  }
+
   const setup = reporterScope.bind(async (rootReporter) => {
     const reportersMap = await getReportersMap(reporters, destinations);
+    if (previousRuns && rerunFilePath) {
+      ArrayPrototypePush(reportersMap, {
+        __proto__: null,
+        reporter: reportReruns(previousRuns, rerunFilePath),
+        destination: new PassThrough(),
+      });
+    }
 
     for (let i = 0; i < reportersMap.length; i++) {
       const { reporter, destination } = reportersMap[i];
@@ -343,6 +376,8 @@ function parseCommandLine() {
     timeout,
     updateSnapshots,
     watch,
+    rerunFilePath,
+    previousRuns,
   };
 
   return globalTestOptions;
@@ -637,4 +672,5 @@ module.exports = {
   shouldColorizeTestFiles,
   getCoverageReport,
   setupGlobalSetupTeardownFunctions,
+  parsePreviousRuns,
 };

src/node_options.cc

@@ -918,6 +918,11 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() {
             &EnvironmentOptions::test_global_setup_path,
             kAllowedInEnvvar,
             OptionNamespaces::kTestRunnerNamespace);
+  AddOption("--test-rerun",
+            "specifies the path to the rerun state file",
+            &EnvironmentOptions::test_rerun,
+            kAllowedInEnvvar,
+            OptionNamespaces::kTestRunnerNamespace);
   AddOption("--test-udp-no-try-send",
             "",  // For testing only.
             &EnvironmentOptions::test_udp_no_try_send,

src/node_options.h

@@ -198,6 +198,7 @@ class EnvironmentOptions : public Options {
   bool test_runner_update_snapshots = false;
   std::vector<std::string> test_name_pattern;
   std::vector<std::string> test_reporter;
+  std::string test_rerun;
   std::vector<std::string> test_reporter_destination;
   std::string test_global_setup_path;
   bool test_only = false;