test_runner: add option to rerun only failed tests

PR-URL: #59443
Reviewed-By: Benjamin Gruenbaum <[email protected]>
Reviewed-By: Pietro Marchini <[email protected]>
Reviewed-By: Chemi Atlow <[email protected]>

Author: MoLow

doc/api/cli.md

@@ -2647,6 +2647,20 @@ changes:
 The destination for the corresponding test reporter. See the documentation on
 [test reporters][] for more details.
 
+### `--test-rerun-failures`
+
+<!-- YAML
+added:
+  - REPLACEME
+-->
+
+A path to a file allowing the test runner to persist the state of the test
+suite between runs. The test runner will use this file to determine which tests
+have already succeeded or failed, allowing for re-running of failed tests
+without having to re-run the entire test suite. The test runner will create this
+file if it does not exist.
+See the documentation on [test reruns][] for more details.
+
 ### `--test-shard`
 
 <!-- YAML
@@ -3508,6 +3522,7 @@ one is included in the list below.
 * `--test-only`
 * `--test-reporter-destination`
 * `--test-reporter`
+* `--test-rerun-failures`
 * `--test-shard`
 * `--test-skip-pattern`
 * `--throw-deprecation`
@@ -4082,6 +4097,7 @@ node --stack-trace-limit=12 -p -e "Error.stackTraceLimit" # prints 12
 [snapshot testing]: test.md#snapshot-testing
 [syntax detection]: packages.md#syntax-detection
 [test reporters]: test.md#test-reporters
+[test reruns]: test.md#rerunning-failed-tests
 [test runner execution model]: test.md#test-runner-execution-model
 [timezone IDs]: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
 [tracking issue for user-land snapshots]: https://github.com/nodejs/node/issues/44014

doc/api/test.md

@@ -153,6 +153,46 @@ test('skip() method with message', (t) => {
 });
 ```
 
+## Rerunning failed tests
+
+The test runner supports persisting the state of the run to a file, allowing
+the test runner to rerun failed tests without having to re-run the entire test suite.
+Use the [`--test-rerun-failures`][] command-line option to specify a file path where the
+state of the run is stored. if the state file does not exist, the test runner will
+create it.
+the state file is a JSON file that contains an array of run attempts.
+Each run attempt is an object mapping successful tests to the attempt they have passed in.
+The key identifying a test in this map is the test file path, with the line and column where the test is defined.
+in a case where a test defined in a specific location is run multiple times,
+for example within a function or a loop,
+a counter will be appended to the key, to disambiguate the test runs.
+note changing the order of test execution or the location of a test can lead the test runner
+to consider tests as passed on a previous attempt,
+meaning `--test-rerun-failures` should be used when tests run in a deterministic order.
+
+example of a state file:
+
+```json
+[
+  {
+    "test.js:10:5": { "passed_on_attempt": 0, "name": "test 1" },
+  },
+  {
+    "test.js:10:5": { "passed_on_attempt": 0, "name": "test 1" },
+    "test.js:20:5": { "passed_on_attempt": 1, "name": "test 2" }
+  }
+]
+```
+
+in this example, there are two run attempts, with two tests defined in `test.js`,
+the first test succeeded on the first attempt, and the second test succeeded on the second attempt.
+
+When the `--test-rerun-failures` option is used, the test runner will only run tests that have not yet passed.
+
+```bash
+node --test-rerun-failures /path/to/state/file
+```
+
 ## TODO tests
 
 Individual tests can be marked as flaky or incomplete by passing the `todo`
@@ -1342,6 +1382,9 @@ added:
   - v18.9.0
   - v16.19.0
 changes:
+  - version: REPLACEME
+    pr-url: https://github.com/nodejs/node/pull/59443
+    description: Added a rerunFailuresFilePath option.
   - version: v23.0.0
     pr-url: https://github.com/nodejs/node/pull/54705
     description: Added the `cwd` option.
@@ -1432,6 +1475,10 @@ changes:
       that specifies the index of the shard to run. This option is _required_.
     * `total` {number} is a positive integer that specifies the total number
       of shards to split the test files to. This option is _required_.
+  * `rerunFailuresFilePath` {string} A file path where the test runner will
+    store the state of the tests to allow rerunning only the failed tests on a next run.
+    see \[Rerunning failed tests]\[] for more information.
+    **Default:** `undefined`.
   * `coverage` {boolean} enable [code coverage][] collection.
     **Default:** `false`.
   * `coverageExcludeGlobs` {string|Array} Excludes specific files from code coverage
@@ -3220,6 +3267,8 @@ Emitted when a test is enqueued for execution.
       * `cause` {Error} The actual error thrown by the test.
     * `type` {string|undefined} The type of the test, used to denote whether
       this is a suite.
+    * `attempt` {number|undefined} The attempt number of the test run,
+      present only when using the [`--test-rerun-failures`][] flag.
   * `file` {string|undefined} The path of the test file,
     `undefined` if test was run through the REPL.
   * `line` {number|undefined} The line number where the test is defined, or
@@ -3244,6 +3293,10 @@ The corresponding execution ordered event is `'test:complete'`.
     * `duration_ms` {number} The duration of the test in milliseconds.
     * `type` {string|undefined} The type of the test, used to denote whether
       this is a suite.
+    * `attempt` {number|undefined} The attempt number of the test run,
+      present only when using the [`--test-rerun-failures`][] flag.
+    * `passed_on_attempt` {number|undefined} The attempt number the test passed on,
+      present only when using the [`--test-rerun-failures`][] flag.
   * `file` {string|undefined} The path of the test file,
     `undefined` if test was run through the REPL.
   * `line` {number|undefined} The line number where the test is defined, or
@@ -3947,6 +4000,7 @@ Can be used to abort test subtasks when the test has been aborted.
 [`--test-only`]: cli.md#--test-only
 [`--test-reporter-destination`]: cli.md#--test-reporter-destination
 [`--test-reporter`]: cli.md#--test-reporter
+[`--test-rerun-failures`]: cli.md#--test-rerun-failures
 [`--test-skip-pattern`]: cli.md#--test-skip-pattern
 [`--test-update-snapshots`]: cli.md#--test-update-snapshots
 [`--test`]: cli.md#--test

doc/node-config-schema.json

@@ -446,6 +446,9 @@
             }
           ]
         },
+        "test-rerun-failures": {
+          "type": "string"
+        },
         "test-shard": {
           "type": "string"
         },
@@ -698,6 +701,9 @@
             }
           ]
         },
+        "test-rerun-failures": {
+          "type": "string"
+        },
         "test-shard": {
           "type": "string"
         },

doc/node.1

@@ -493,6 +493,10 @@ A test reporter to use when running tests.
 .It Fl -test-reporter-destination
 The destination for the corresponding test reporter.
 .
+.It Fl -test-rerun-failures
+Configures the tests runner to persist the state of tests to allow
+rerunning only failed tests.
+.
 .It Fl -test-only
 Configures the test runner to only execute top level tests that have the `only`
 option set.

lib/internal/test_runner/harness.js

@@ -26,7 +26,10 @@ const {
   reporterScope,
   shouldColorizeTestFiles,
   setupGlobalSetupTeardownFunctions,
+  parsePreviousRuns,
 } = require('internal/test_runner/utils');
+const { PassThrough, compose } = require('stream');
+const { reportReruns } = require('internal/test_runner/reporter/rerun');
 const { queueMicrotask } = require('internal/process/task_queues');
 const { TIMEOUT_MAX } = require('internal/timers');
 const { clearInterval, setInterval } = require('timers');
@@ -69,6 +72,7 @@ function createTestTree(rootTestOptions, globalOptions) {
     shouldColorizeTestFiles: shouldColorizeTestFiles(globalOptions.destinations),
     teardown: null,
     snapshotManager: null,
+    previousRuns: null,
     isFilteringByName,
     isFilteringByOnly,
     async runBootstrap() {
@@ -203,6 +207,25 @@ function collectCoverage(rootTest, coverage) {
   return summary;
 }
 
+function setupFailureStateFile(rootTest, globalOptions) {
+  if (!globalOptions.rerunFailuresFilePath) {
+    return;
+  }
+  rootTest.harness.previousRuns = parsePreviousRuns(globalOptions.rerunFailuresFilePath);
+  if (rootTest.harness.previousRuns === null) {
+    rootTest.diagnostic(`Warning: The rerun failures file at ` +
+      `${globalOptions.rerunFailuresFilePath} is not a valid rerun file. ` +
+      'The test runner will not be able to rerun failed tests.');
+    rootTest.harness.success = false;
+    process.exitCode = kGenericUserError;
+    return;
+  }
+  if (!process.env.NODE_TEST_CONTEXT) {
+    const reporter = reportReruns(rootTest.harness.previousRuns, globalOptions);
+    compose(rootTest.reporter, reporter).pipe(new PassThrough());
+  }
+}
+
 function setupProcessState(root, globalOptions) {
   const hook = createHook({
     __proto__: null,
@@ -230,6 +253,9 @@ function setupProcessState(root, globalOptions) {
   const rejectionHandler =
     createProcessEventHandler('unhandledRejection', root);
   const coverage = configureCoverage(root, globalOptions);
+
+  setupFailureStateFile(root, globalOptions);
+
   const exitHandler = async (kill) => {
     if (root.subtests.length === 0 && (root.hooks.before.length > 0 || root.hooks.after.length > 0)) {
       // Run global before/after hooks in case there are no tests

lib/internal/test_runner/reporter/rerun.js

@@ -0,0 +1,40 @@
+'use strict';
+
+const {
+  ArrayPrototypePush,
+  JSONStringify,
+} = primordials;
+const { relative } = require('path');
+const { writeFileSync } = require('fs');
+
+function reportReruns(previousRuns, globalOptions) {
+  return async function reporter(source) {
+    const obj = { __proto__: null };
+    const disambiguator = { __proto__: null };
+
+    for await (const { type, data } of source) {
+      if (type === 'test:pass') {
+        let identifier = `${relative(globalOptions.cwd, data.file)}:${data.line}:${data.column}`;
+        if (disambiguator[identifier] !== undefined) {
+          identifier += `:(${disambiguator[identifier]})`;
+          disambiguator[identifier] += 1;
+        } else {
+          disambiguator[identifier] = 1;
+        }
+        obj[identifier] = {
+          __proto__: null,
+          name: data.name,
+          passed_on_attempt: data.details.passed_on_attempt ?? data.details.attempt,
+        };
+      }
+    }
+
+    ArrayPrototypePush(previousRuns, obj);
+    writeFileSync(globalOptions.rerunFailuresFilePath, JSONStringify(previousRuns, null, 2), 'utf8');
+  };
+};
+
+module.exports = {
+  __proto__: null,
+  reportReruns,
+};

lib/internal/test_runner/runner.js

@@ -148,6 +148,7 @@ function getRunArgs(path, { forceExit,
                             only,
                             argv: suppliedArgs,
                             execArgv,
+                            rerunFailuresFilePath,
                             root: { timeout },
                             cwd }) {
   const processNodeOptions = getOptionsAsFlagsFromBinding();
@@ -170,6 +171,9 @@ function getRunArgs(path, { forceExit,
   if (timeout != null) {
     ArrayPrototypePush(runArgs, `--test-timeout=${timeout}`);
   }
+  if (rerunFailuresFilePath) {
+    ArrayPrototypePush(runArgs, `--test-rerun-failures=${rerunFailuresFilePath}`);
+  }
 
   ArrayPrototypePushApply(runArgs, execArgv);
 
@@ -588,6 +592,7 @@ function run(options = kEmptyObject) {
     execArgv = [],
     argv = [],
     cwd = process.cwd(),
+    rerunFailuresFilePath,
   } = options;
 
   if (files != null) {
@@ -620,6 +625,10 @@ function run(options = kEmptyObject) {
     );
   }
 
+  if (rerunFailuresFilePath) {
+    validatePath(rerunFailuresFilePath, 'options.rerunFailuresFilePath');
+  }
+
   if (shard != null) {
     validateObject(shard, 'options.shard');
     // Avoid re-evaluating the shard object in case it's a getter
@@ -702,6 +711,7 @@ function run(options = kEmptyObject) {
     coverage,
     coverageExcludeGlobs,
     coverageIncludeGlobs,
+    rerunFailuresFilePath,
     lineCoverage: lineCoverage,
     branchCoverage: branchCoverage,
     functionCoverage: functionCoverage,
@@ -735,6 +745,7 @@ function run(options = kEmptyObject) {
     isolation,
     argv,
     execArgv,
+    rerunFailuresFilePath,
   };
 
   if (isolation === 'process') {

lib/internal/test_runner/test.js

@@ -71,6 +71,7 @@ const {
 } = require('timers');
 const { TIMEOUT_MAX } = require('internal/timers');
 const { fileURLToPath } = require('internal/url');
+const { relative } = require('path');
 const { availableParallelism } = require('os');
 const { innerOk } = require('internal/assert/utils');
 const { bigint: hrtime } = process.hrtime;
@@ -290,6 +291,10 @@ class TestContext {
     return this.#test.passed;
   }
 
+  get attempt() {
+    return this.#test.attempt ?? 0;
+  }
+
   diagnostic(message) {
     this.#test.diagnostic(message);
   }
@@ -537,6 +542,7 @@ class Test extends AsyncResource {
       this.childNumber = 0;
       this.timeout = kDefaultTimeout;
       this.entryFile = entryFile;
+      this.testDisambiguator = new SafeMap();
     } else {
       const nesting = parent.parent === null ? parent.nesting :
         parent.nesting + 1;
@@ -646,6 +652,8 @@ class Test extends AsyncResource {
     this.endTime = null;
     this.passed = false;
     this.error = null;
+    this.attempt = undefined;
+    this.passedAttempt = undefined;
     this.message = typeof skip === 'string' ? skip :
       typeof todo === 'string' ? todo : null;
     this.activeSubtests = 0;
@@ -690,6 +698,23 @@ class Test extends AsyncResource {
         this.loc.file = fileURLToPath(this.loc.file);
       }
     }
+
+    if (this.loc != null && this.root.harness.previousRuns != null) {
+      let testIdentifier = `${relative(this.config.cwd, this.loc.file)}:${this.loc.line}:${this.loc.column}`;
+      const disambiguator = this.root.testDisambiguator.get(testIdentifier);
+      if (disambiguator !== undefined) {
+        testIdentifier += `:(${disambiguator})`;
+        this.root.testDisambiguator.set(testIdentifier, disambiguator + 1);
+      } else {
+        this.root.testDisambiguator.set(testIdentifier, 1);
+      }
+      this.attempt = this.root.harness.previousRuns.length;
+      const previousAttempt = this.root.harness.previousRuns[this.attempt - 1]?.[testIdentifier]?.passed_on_attempt;
+      if (previousAttempt != null) {
+        this.passedAttempt = previousAttempt;
+        this.fn = noop;
+      }
+    }
   }
 
   applyFilters() {
@@ -1329,6 +1354,12 @@ class Test extends AsyncResource {
     if (!this.passed) {
       details.error = this.error;
     }
+    if (this.attempt !== undefined) {
+      details.attempt = this.attempt;
+    }
+    if (this.passedAttempt !== undefined) {
+      details.passed_on_attempt = this.passedAttempt;
+    }
     return { __proto__: null, details, directive };
   }