test: refactor test_runner tests to change default reporter

This commit updates the test runner tests in order to switch the
default reporter from tap to spec. This commit can be backported,
while changing the default reporter cannot.

Refs: nodejs#54540
PR-URL: nodejs#54547
Reviewed-By: Matteo Collina <[email protected]>
Reviewed-By: Benjamin Gruenbaum <[email protected]>
Reviewed-By: James M Snell <[email protected]>
Reviewed-By: Moshe Atlow <[email protected]>
Reviewed-By: Jake Yuesong Li <[email protected]>

test/common/assertSnapshot.js

@@ -77,7 +77,11 @@ async function spawnAndAssert(filename, transform = (x) => x, { tty = false, ...
     test({ skip: 'Skipping pseudo-tty tests, as pseudo terminals are not available on Windows.' });
     return;
   }
-  const flags = common.parseTestFlags(filename);
+  let flags = common.parseTestFlags(filename);
+  if (options.flags) {
+    flags = [...options.flags, ...flags];
+  }
+
   const executable = tty ? (process.env.PYTHON || 'python3') : process.execPath;
   const args =
     tty ?

test/parallel/test-runner-cli.js

@@ -26,7 +26,8 @@ for (const isolation of ['none', 'process']) {
   {
     // Default behavior. node_modules is ignored. Files that don't match the
     // pattern are ignored except in test/ directories.
-    const args = ['--test', `--experimental-test-isolation=${isolation}`];
+    const args = ['--test', '--test-reporter=tap',
+                  `--experimental-test-isolation=${isolation}`];
     const child = spawnSync(process.execPath, args, { cwd: join(testFixtures, 'default-behavior') });
 
     assert.strictEqual(child.status, 1);
@@ -47,6 +48,7 @@ for (const isolation of ['none', 'process']) {
     const args = [
       '--require', join(testFixtures, 'protoMutation.js'),
       '--test',
+      '--test-reporter=tap',
       `--experimental-test-isolation=${isolation}`,
     ];
     const child = spawnSync(process.execPath, args, { cwd: join(testFixtures, 'default-behavior') });
@@ -67,6 +69,7 @@ for (const isolation of ['none', 'process']) {
     // User specified files that don't match the pattern are still run.
     const args = [
       '--test',
+      '--test-reporter=tap',
       `--experimental-test-isolation=${isolation}`,
       join(testFixtures, 'index.js'),
     ];
@@ -83,6 +86,7 @@ for (const isolation of ['none', 'process']) {
     // Searches node_modules if specified.
     const args = [
       '--test',
+      '--test-reporter=tap',
       `--experimental-test-isolation=${isolation}`,
       join(testFixtures, 'default-behavior/node_modules/*.js'),
     ];
@@ -105,18 +109,19 @@ for (const isolation of ['none', 'process']) {
     assert.strictEqual(child.signal, null);
     assert.strictEqual(child.stderr.toString(), '');
     const stdout = child.stdout.toString();
-    assert.match(stdout, /ok 1 - this should pass/);
-    assert.match(stdout, /not ok 2 - this should fail/);
-    assert.match(stdout, /ok 3 - subdir.+subdir_test\.js/);
-    assert.match(stdout, /ok 4 - this should pass/);
-    assert.match(stdout, /ok 5 - this should be skipped/);
-    assert.match(stdout, /ok 6 - this should be executed/);
+    assert.match(stdout, /this should pass/);
+    assert.match(stdout, /this should fail/);
+    assert.match(stdout, /subdir.+subdir_test\.js/);
+    assert.match(stdout, /this should pass/);
+    assert.match(stdout, /this should be skipped/);
+    assert.match(stdout, /this should be executed/);
   }
 
   {
     // Test combined stream outputs
     const args = [
       '--test',
+      '--test-reporter=tap',
       `--experimental-test-isolation=${isolation}`,
       'test/fixtures/test-runner/default-behavior/index.test.js',
       'test/fixtures/test-runner/nested.js',
@@ -189,6 +194,7 @@ for (const isolation of ['none', 'process']) {
   // Test user logging in tests.
   const args = [
     '--test',
+    '--test-reporter=tap',
     'test/fixtures/test-runner/user-logs.js',
   ];
   const child = spawnSync(process.execPath, args);
@@ -223,7 +229,7 @@ for (const isolation of ['none', 'process']) {
   assert.strictEqual(child.status, 0);
   assert.strictEqual(child.signal, null);
   const stdout = child.stdout.toString();
-  assert.match(stdout, /ok 1 - this should pass/);
+  assert.match(stdout, /this should pass/);
 }
 
 {
@@ -290,6 +296,7 @@ for (const isolation of ['none', 'process']) {
   // --test-shard option, first shard
   const args = [
     '--test',
+    '--test-reporter=tap',
     '--test-shard=1/2',
     join(testFixtures, 'shards/*.cjs'),
   ];
@@ -324,6 +331,7 @@ for (const isolation of ['none', 'process']) {
   // --test-shard option, last shard
   const args = [
     '--test',
+    '--test-reporter=tap',
     '--test-shard=2/2',
     join(testFixtures, 'shards/*.cjs'),
   ];

test/parallel/test-runner-exit-code.js

@@ -12,7 +12,7 @@ async function runAndKill(file) {
     return;
   }
   let stdout = '';
-  const child = spawn(process.execPath, ['--test', file]);
+  const child = spawn(process.execPath, ['--test', '--test-reporter=tap', file]);
   child.stdout.setEncoding('utf8');
   child.stdout.on('data', (chunk) => {
     if (!stdout.length) child.kill('SIGINT');
@@ -58,10 +58,10 @@ if (process.argv[2] === 'child') {
   assert.strictEqual(child.status, 0);
   assert.strictEqual(child.signal, null);
   const stdout = child.stdout.toString();
-  assert.match(stdout, /# tests 3/);
-  assert.match(stdout, /# pass 0/);
-  assert.match(stdout, /# fail 0/);
-  assert.match(stdout, /# todo 3/);
+  assert.match(stdout, /tests 3/);
+  assert.match(stdout, /pass 0/);
+  assert.match(stdout, /fail 0/);
+  assert.match(stdout, /todo 3/);
 
   child = spawnSync(process.execPath, [__filename, 'child', 'fail']);
   assert.strictEqual(child.status, 1);

test/parallel/test-runner-extraneous-async-activity.js

@@ -10,10 +10,10 @@ const { spawnSync } = require('child_process');
     fixtures.path('test-runner', 'extraneous_set_immediate_async.mjs'),
   ]);
   const stdout = child.stdout.toString();
-  assert.match(stdout, /^# Error: Test "extraneous async activity test" at .+extraneous_set_immediate_async\.mjs:3:1 generated asynchronous activity after the test ended/m);
-  assert.match(stdout, /^# pass 1/m);
-  assert.match(stdout, /^# fail 1$/m);
-  assert.match(stdout, /^# cancelled 0$/m);
+  assert.match(stdout, /Error: Test "extraneous async activity test" at .+extraneous_set_immediate_async\.mjs:3:1 generated asynchronous activity after the test ended/m);
+  assert.match(stdout, /pass 1/m);
+  assert.match(stdout, /fail 1$/m);
+  assert.match(stdout, /cancelled 0$/m);
   assert.strictEqual(child.status, 1);
   assert.strictEqual(child.signal, null);
 }
@@ -24,10 +24,10 @@ const { spawnSync } = require('child_process');
     fixtures.path('test-runner', 'extraneous_set_timeout_async.mjs'),
   ]);
   const stdout = child.stdout.toString();
-  assert.match(stdout, /^# Error: Test "extraneous async activity test" at .+extraneous_set_timeout_async\.mjs:3:1 generated asynchronous activity after the test ended/m);
-  assert.match(stdout, /^# pass 1$/m);
-  assert.match(stdout, /^# fail 1$/m);
-  assert.match(stdout, /^# cancelled 0$/m);
+  assert.match(stdout, /Error: Test "extraneous async activity test" at .+extraneous_set_timeout_async\.mjs:3:1 generated asynchronous activity after the test ended/m);
+  assert.match(stdout, /pass 1$/m);
+  assert.match(stdout, /fail 1$/m);
+  assert.match(stdout, /cancelled 0$/m);
   assert.strictEqual(child.status, 1);
   assert.strictEqual(child.signal, null);
 }
@@ -38,13 +38,13 @@ const { spawnSync } = require('child_process');
     fixtures.path('test-runner', 'async-error-in-test-hook.mjs'),
   ]);
   const stdout = child.stdout.toString();
-  assert.match(stdout, /^# Error: Test hook "before" at .+async-error-in-test-hook\.mjs:3:1 generated asynchronous activity after the test ended/m);
-  assert.match(stdout, /^# Error: Test hook "beforeEach" at .+async-error-in-test-hook\.mjs:9:1 generated asynchronous activity after the test ended/m);
-  assert.match(stdout, /^# Error: Test hook "after" at .+async-error-in-test-hook\.mjs:15:1 generated asynchronous activity after the test ended/m);
-  assert.match(stdout, /^# Error: Test hook "afterEach" at .+async-error-in-test-hook\.mjs:21:1 generated asynchronous activity after the test ended/m);
-  assert.match(stdout, /^# pass 1$/m);
-  assert.match(stdout, /^# fail 1$/m);
-  assert.match(stdout, /^# cancelled 0$/m);
+  assert.match(stdout, /Error: Test hook "before" at .+async-error-in-test-hook\.mjs:3:1 generated asynchronous activity after the test ended/m);
+  assert.match(stdout, /Error: Test hook "beforeEach" at .+async-error-in-test-hook\.mjs:9:1 generated asynchronous activity after the test ended/m);
+  assert.match(stdout, /Error: Test hook "after" at .+async-error-in-test-hook\.mjs:15:1 generated asynchronous activity after the test ended/m);
+  assert.match(stdout, /Error: Test hook "afterEach" at .+async-error-in-test-hook\.mjs:21:1 generated asynchronous activity after the test ended/m);
+  assert.match(stdout, /pass 1$/m);
+  assert.match(stdout, /fail 1$/m);
+  assert.match(stdout, /cancelled 0$/m);
   assert.strictEqual(child.status, 1);
   assert.strictEqual(child.signal, null);
 }
@@ -56,13 +56,13 @@ const { spawnSync } = require('child_process');
     fixtures.path('test-runner', 'async-error-in-test-hook.mjs'),
   ]);
   const stdout = child.stdout.toString();
-  assert.match(stdout, /^# Error: Test hook "before" at .+async-error-in-test-hook\.mjs:3:1 generated asynchronous activity after the test ended/m);
-  assert.match(stdout, /^# Error: Test hook "beforeEach" at .+async-error-in-test-hook\.mjs:9:1 generated asynchronous activity after the test ended/m);
-  assert.match(stdout, /^# Error: Test hook "after" at .+async-error-in-test-hook\.mjs:15:1 generated asynchronous activity after the test ended/m);
-  assert.match(stdout, /^# Error: Test hook "afterEach" at .+async-error-in-test-hook\.mjs:21:1 generated asynchronous activity after the test ended/m);
-  assert.match(stdout, /^# pass 1$/m);
-  assert.match(stdout, /^# fail 0$/m);
-  assert.match(stdout, /^# cancelled 0$/m);
+  assert.match(stdout, /Error: Test hook "before" at .+async-error-in-test-hook\.mjs:3:1 generated asynchronous activity after the test ended/m);
+  assert.match(stdout, /Error: Test hook "beforeEach" at .+async-error-in-test-hook\.mjs:9:1 generated asynchronous activity after the test ended/m);
+  assert.match(stdout, /Error: Test hook "after" at .+async-error-in-test-hook\.mjs:15:1 generated asynchronous activity after the test ended/m);
+  assert.match(stdout, /Error: Test hook "afterEach" at .+async-error-in-test-hook\.mjs:21:1 generated asynchronous activity after the test ended/m);
+  assert.match(stdout, /pass 1$/m);
+  assert.match(stdout, /fail 0$/m);
+  assert.match(stdout, /cancelled 0$/m);
   assert.strictEqual(child.status, 1);
   assert.strictEqual(child.signal, null);
 }

test/parallel/test-runner-force-exit-failure.js

@@ -8,6 +8,7 @@ const fixture = fixtures.path('test-runner/throws_sync_and_async.js');
 for (const isolation of ['none', 'process']) {
   const args = [
     '--test',
+    '--test-reporter=spec',
     '--test-force-exit',
     `--experimental-test-isolation=${isolation}`,
     fixture,
@@ -19,6 +20,6 @@ for (const isolation of ['none', 'process']) {
   strictEqual(r.stderr.toString(), '');
 
   const stdout = r.stdout.toString();
-  match(stdout, /error: 'fails'/);
+  match(stdout, /Error: fails/);
   doesNotMatch(stdout, /this should not have a chance to be thrown/);
 }

test/parallel/test-runner-inspect.mjs

@@ -12,7 +12,7 @@ tmpdir.refresh();
 
 {
   const child = new NodeInstance(
-    ['--test', '--inspect-brk=0'],
+    ['--test', '--test-reporter=tap', '--inspect-brk=0'],
     undefined,
     fixtures.path('test-runner/default-behavior/index.test.js')
   );
@@ -38,7 +38,10 @@ tmpdir.refresh();
 
 
 {
-  const args = ['--test', '--inspect=0', fixtures.path('test-runner/index.js')];
+  const args = [
+    '--test', '--test-reporter=tap', '--inspect=0',
+    fixtures.path('test-runner/index.js'),
+  ];
   const { stderr, stdout, code, signal } = await common.spawnPromisified(process.execPath, args);
 
   assert.match(stderr,

test/parallel/test-runner-misc.js

@@ -33,9 +33,9 @@ if (process.argv[2] === 'child') {
 } else {
   const child = spawnSync(process.execPath, [__filename, 'child', 'abortSignal']);
   const stdout = child.stdout.toString();
-  assert.match(stdout, /^# pass 2$/m);
-  assert.match(stdout, /^# fail 0$/m);
-  assert.match(stdout, /^# cancelled 1$/m);
+  assert.match(stdout, /pass 2$/m);
+  assert.match(stdout, /fail 0$/m);
+  assert.match(stdout, /cancelled 1$/m);
   assert.strictEqual(child.status, 1);
   assert.strictEqual(child.signal, null);
 }

test/parallel/test-runner-module-mocking.js

@@ -571,7 +571,7 @@ test('node_modules can be used by both module systems', async (t) => {
 
   assert.strictEqual(code, 0);
   assert.strictEqual(signal, null);
-  assert.match(stdout, /# pass 1/);
+  assert.match(stdout, /pass 1/);
 });
 
 test('file:// imports are supported in ESM only', async (t) => {

test/parallel/test-runner-no-isolation-filtering.js

@@ -11,6 +11,7 @@ const fixture2 = fixtures.path('test-runner', 'no-isolation', 'two.test.js');
 test('works with --test-only', () => {
   const args = [
     '--test',
+    '--test-reporter=tap',
     '--experimental-test-isolation=none',
     '--test-only',
     fixture1,
@@ -33,6 +34,7 @@ test('works with --test-only', () => {
 test('works with --test-name-pattern', () => {
   const args = [
     '--test',
+    '--test-reporter=tap',
     '--experimental-test-isolation=none',
     '--test-name-pattern=/test one/',
     fixture1,
@@ -52,6 +54,7 @@ test('works with --test-name-pattern', () => {
 test('works with --test-skip-pattern', () => {
   const args = [
     '--test',
+    '--test-reporter=tap',
     '--experimental-test-isolation=none',
     '--test-skip-pattern=/one/',
     fixture1,