Skip to content

Commit

Permalink
test: refactor test_runner tests to change default reporter
Browse files Browse the repository at this point in the history 
This commit updates the test runner tests in order to switch the
default reporter from tap to spec. This commit can be backported,
while changing the default reporter cannot.

Refs: #54540
  • Loading branch information
@cjihrig
cjihrig committed Aug 24, 2024
1 parent 4e68b54 commit efd6176
Show file tree
Hide file tree
Showing 16 changed files with 233 additions and 114 deletions.
6 changes: 5 additions & 1 deletion test/common/assertSnapshot.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,11 @@ async function spawnAndAssert(filename, transform = (x) => x, { tty = false, ...
test({ skip: 'Skipping pseudo-tty tests, as pseudo terminals are not available on Windows.' });
return;
}
const flags = common.parseTestFlags(filename);
let flags = common.parseTestFlags(filename);
if (options.flags) {
flags = [...options.flags, ...flags];
}

const executable = tty ? (process.env.PYTHON || 'python3') : process.execPath;
const args =
tty ?
Expand Down
24 changes: 16 additions & 8 deletions test/parallel/test-runner-cli.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ for (const isolation of ['none', 'process']) {
{
// Default behavior. node_modules is ignored. Files that don't match the
// pattern are ignored except in test/ directories.
const args = ['--test', `--experimental-test-isolation=${isolation}`];
const args = ['--test', '--test-reporter=tap',
`--experimental-test-isolation=${isolation}`];
const child = spawnSync(process.execPath, args, { cwd: join(testFixtures, 'default-behavior') });

assert.strictEqual(child.status, 1);
Expand All @@ -47,6 +48,7 @@ for (const isolation of ['none', 'process']) {
const args = [
'--require', join(testFixtures, 'protoMutation.js'),
'--test',
'--test-reporter=tap',
`--experimental-test-isolation=${isolation}`,
];
const child = spawnSync(process.execPath, args, { cwd: join(testFixtures, 'default-behavior') });
Expand All @@ -67,6 +69,7 @@ for (const isolation of ['none', 'process']) {
// User specified files that don't match the pattern are still run.
const args = [
'--test',
'--test-reporter=tap',
`--experimental-test-isolation=${isolation}`,
join(testFixtures, 'index.js'),
];
Expand All @@ -83,6 +86,7 @@ for (const isolation of ['none', 'process']) {
// Searches node_modules if specified.
const args = [
'--test',
'--test-reporter=tap',
`--experimental-test-isolation=${isolation}`,
join(testFixtures, 'default-behavior/node_modules/*.js'),
];
Expand All @@ -105,18 +109,19 @@ for (const isolation of ['none', 'process']) {
assert.strictEqual(child.signal, null);
assert.strictEqual(child.stderr.toString(), '');
const stdout = child.stdout.toString();
assert.match(stdout, /ok 1 - this should pass/);
assert.match(stdout, /not ok 2 - this should fail/);
assert.match(stdout, /ok 3 - subdir.+subdir_test\.js/);
assert.match(stdout, /ok 4 - this should pass/);
assert.match(stdout, /ok 5 - this should be skipped/);
assert.match(stdout, /ok 6 - this should be executed/);
assert.match(stdout, /this should pass/);
assert.match(stdout, /this should fail/);
assert.match(stdout, /subdir.+subdir_test\.js/);
assert.match(stdout, /this should pass/);
assert.match(stdout, /this should be skipped/);
assert.match(stdout, /this should be executed/);
}

{
// Test combined stream outputs
const args = [
'--test',
'--test-reporter=tap',
`--experimental-test-isolation=${isolation}`,
'test/fixtures/test-runner/default-behavior/index.test.js',
'test/fixtures/test-runner/nested.js',
Expand Down Expand Up @@ -189,6 +194,7 @@ for (const isolation of ['none', 'process']) {
// Test user logging in tests.
const args = [
'--test',
'--test-reporter=tap',
'test/fixtures/test-runner/user-logs.js',
];
const child = spawnSync(process.execPath, args);
Expand Down Expand Up @@ -223,7 +229,7 @@ for (const isolation of ['none', 'process']) {
assert.strictEqual(child.status, 0);
assert.strictEqual(child.signal, null);
const stdout = child.stdout.toString();
assert.match(stdout, /ok 1 - this should pass/);
assert.match(stdout, /this should pass/);
}

{
Expand Down Expand Up @@ -290,6 +296,7 @@ for (const isolation of ['none', 'process']) {
// --test-shard option, first shard
const args = [
'--test',
'--test-reporter=tap',
'--test-shard=1/2',
join(testFixtures, 'shards/*.cjs'),
];
Expand Down Expand Up @@ -324,6 +331,7 @@ for (const isolation of ['none', 'process']) {
// --test-shard option, last shard
const args = [
'--test',
'--test-reporter=tap',
'--test-shard=2/2',
join(testFixtures, 'shards/*.cjs'),
];
Expand Down
10 changes: 5 additions & 5 deletions test/parallel/test-runner-exit-code.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ async function runAndKill(file) {
return;
}
let stdout = '';
const child = spawn(process.execPath, ['--test', file]);
const child = spawn(process.execPath, ['--test', '--test-reporter=tap', file]);
child.stdout.setEncoding('utf8');
child.stdout.on('data', (chunk) => {
if (!stdout.length) child.kill('SIGINT');
Expand Down Expand Up @@ -58,10 +58,10 @@ if (process.argv[2] === 'child') {
assert.strictEqual(child.status, 0);
assert.strictEqual(child.signal, null);
const stdout = child.stdout.toString();
assert.match(stdout, /# tests 3/);
assert.match(stdout, /# pass 0/);
assert.match(stdout, /# fail 0/);
assert.match(stdout, /# todo 3/);
assert.match(stdout, /tests 3/);
assert.match(stdout, /pass 0/);
assert.match(stdout, /fail 0/);
assert.match(stdout, /todo 3/);

child = spawnSync(process.execPath, [__filename, 'child', 'fail']);
assert.strictEqual(child.status, 1);
Expand Down
44 changes: 22 additions & 22 deletions test/parallel/test-runner-extraneous-async-activity.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,10 @@ const { spawnSync } = require('child_process');
fixtures.path('test-runner', 'extraneous_set_immediate_async.mjs'),
]);
const stdout = child.stdout.toString();
assert.match(stdout, /^# Error: Test "extraneous async activity test" at .+extraneous_set_immediate_async\.mjs:3:1 generated asynchronous activity after the test ended/m);
assert.match(stdout, /^# pass 1/m);
assert.match(stdout, /^# fail 1$/m);
assert.match(stdout, /^# cancelled 0$/m);
assert.match(stdout, /Error: Test "extraneous async activity test" at .+extraneous_set_immediate_async\.mjs:3:1 generated asynchronous activity after the test ended/m);
assert.match(stdout, /pass 1/m);
assert.match(stdout, /fail 1$/m);
assert.match(stdout, /cancelled 0$/m);
assert.strictEqual(child.status, 1);
assert.strictEqual(child.signal, null);
}
Expand All @@ -24,10 +24,10 @@ const { spawnSync } = require('child_process');
fixtures.path('test-runner', 'extraneous_set_timeout_async.mjs'),
]);
const stdout = child.stdout.toString();
assert.match(stdout, /^# Error: Test "extraneous async activity test" at .+extraneous_set_timeout_async\.mjs:3:1 generated asynchronous activity after the test ended/m);
assert.match(stdout, /^# pass 1$/m);
assert.match(stdout, /^# fail 1$/m);
assert.match(stdout, /^# cancelled 0$/m);
assert.match(stdout, /Error: Test "extraneous async activity test" at .+extraneous_set_timeout_async\.mjs:3:1 generated asynchronous activity after the test ended/m);
assert.match(stdout, /pass 1$/m);
assert.match(stdout, /fail 1$/m);
assert.match(stdout, /cancelled 0$/m);
assert.strictEqual(child.status, 1);
assert.strictEqual(child.signal, null);
}
Expand All @@ -38,13 +38,13 @@ const { spawnSync } = require('child_process');
fixtures.path('test-runner', 'async-error-in-test-hook.mjs'),
]);
const stdout = child.stdout.toString();
assert.match(stdout, /^# Error: Test hook "before" at .+async-error-in-test-hook\.mjs:3:1 generated asynchronous activity after the test ended/m);
assert.match(stdout, /^# Error: Test hook "beforeEach" at .+async-error-in-test-hook\.mjs:9:1 generated asynchronous activity after the test ended/m);
assert.match(stdout, /^# Error: Test hook "after" at .+async-error-in-test-hook\.mjs:15:1 generated asynchronous activity after the test ended/m);
assert.match(stdout, /^# Error: Test hook "afterEach" at .+async-error-in-test-hook\.mjs:21:1 generated asynchronous activity after the test ended/m);
assert.match(stdout, /^# pass 1$/m);
assert.match(stdout, /^# fail 1$/m);
assert.match(stdout, /^# cancelled 0$/m);
assert.match(stdout, /Error: Test hook "before" at .+async-error-in-test-hook\.mjs:3:1 generated asynchronous activity after the test ended/m);
assert.match(stdout, /Error: Test hook "beforeEach" at .+async-error-in-test-hook\.mjs:9:1 generated asynchronous activity after the test ended/m);
assert.match(stdout, /Error: Test hook "after" at .+async-error-in-test-hook\.mjs:15:1 generated asynchronous activity after the test ended/m);
assert.match(stdout, /Error: Test hook "afterEach" at .+async-error-in-test-hook\.mjs:21:1 generated asynchronous activity after the test ended/m);
assert.match(stdout, /pass 1$/m);
assert.match(stdout, /fail 1$/m);
assert.match(stdout, /cancelled 0$/m);
assert.strictEqual(child.status, 1);
assert.strictEqual(child.signal, null);
}
Expand All @@ -56,13 +56,13 @@ const { spawnSync } = require('child_process');
fixtures.path('test-runner', 'async-error-in-test-hook.mjs'),
]);
const stdout = child.stdout.toString();
assert.match(stdout, /^# Error: Test hook "before" at .+async-error-in-test-hook\.mjs:3:1 generated asynchronous activity after the test ended/m);
assert.match(stdout, /^# Error: Test hook "beforeEach" at .+async-error-in-test-hook\.mjs:9:1 generated asynchronous activity after the test ended/m);
assert.match(stdout, /^# Error: Test hook "after" at .+async-error-in-test-hook\.mjs:15:1 generated asynchronous activity after the test ended/m);
assert.match(stdout, /^# Error: Test hook "afterEach" at .+async-error-in-test-hook\.mjs:21:1 generated asynchronous activity after the test ended/m);
assert.match(stdout, /^# pass 1$/m);
assert.match(stdout, /^# fail 0$/m);
assert.match(stdout, /^# cancelled 0$/m);
assert.match(stdout, /Error: Test hook "before" at .+async-error-in-test-hook\.mjs:3:1 generated asynchronous activity after the test ended/m);
assert.match(stdout, /Error: Test hook "beforeEach" at .+async-error-in-test-hook\.mjs:9:1 generated asynchronous activity after the test ended/m);
assert.match(stdout, /Error: Test hook "after" at .+async-error-in-test-hook\.mjs:15:1 generated asynchronous activity after the test ended/m);
assert.match(stdout, /Error: Test hook "afterEach" at .+async-error-in-test-hook\.mjs:21:1 generated asynchronous activity after the test ended/m);
assert.match(stdout, /pass 1$/m);
assert.match(stdout, /fail 0$/m);
assert.match(stdout, /cancelled 0$/m);
assert.strictEqual(child.status, 1);
assert.strictEqual(child.signal, null);
}
2 changes: 1 addition & 1 deletion test/parallel/test-runner-force-exit-failure.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,6 @@ for (const isolation of ['none', 'process']) {
strictEqual(r.stderr.toString(), '');

const stdout = r.stdout.toString();
match(stdout, /error: 'fails'/);
match(stdout, /Error: fails/);

Check failure on line 22 in test/parallel/test-runner-force-exit-failure.js

View workflow job for this annotation

GitHub Actions / test-linux 

--- stderr ---
node:assert:1035
    throw err;
    ^

AssertionError [ERR_ASSERTION]: The input did not match the regular expression /Error: fails/. Input:

'TAP version 13\n' +
  '# Subtest: fails and schedules more work\n' +
  'not ok 1 - fails and schedules more work\n' +
  '  ---\n' +
  '  duration_ms: 5.824801\n' +
  "  location: '/home/runner/work/node/node/test/fixtures/test-runner/throws_sync_and_async.js:4:1'\n" +
  "  failureType: 'testCodeFailure'\n" +
  "  error: 'fails'\n" +
  "  code: 'ERR_TEST_FAILURE'\n" +
  '  stack: |-\n' +
  '    TestContext.<anonymous> (/home/runner/work/node/node/test/fixtures/test-runner/throws_sync_and_async.js:9:9)\n' +
  '    Test.runInAsyncScope (node:async_hooks:211:14)\n' +
  '    Test.run (node:internal/test_runner/test:887:25)\n' +
  '    Test.start (node:internal/test_runner/test:786:17)\n' +
  '    startSubtestAfterBootstrap (node:internal/test_runner/harness:280:17)\n' +
  '  ...\n' +
  '1..1\n' +
  '# tests 1\n' +
  '# suites 0\n' +
  '# pass 0\n' +
  '# fail 1\n' +
  '# cancelled 0\n' +
  '# skipped 0\n' +
  '# todo 0\n'

    at Object.<anonymous> (/home/runner/work/node/node/test/parallel/test-runner-force-exit-failure.js:22:3)
    at Module._compile (node:internal/modules/cjs/loader:1546:14)
    at Module._extensions..js (node:internal/modules/cjs/loader:1691:10)
    at Module.load (node:internal/modules/cjs/loader:1317:32)
    at Module._load (node:internal/modules/cjs/loader:1127:12)
    at TracingChannel.traceSync (node:diagnostics_channel:315:14)
    at wrapModuleLoad (node:internal/modules/cjs/loader:217:24)
    at Function.executeUserEntryPoint [as runMain] (node:internal/modules/run_main:166:5)
    at node:internal/main/run_main_module:30:49 {
  generatedMessage: true,
  code: 'ERR_ASSERTION',
  actual: 'TAP version 13\n' +
    '# Subtest: fails and schedules more work\n' +
    'not ok 1 - fails and schedules more work\n' +
    '  ---\n' +
    '  duration_ms: 5.824801\n' +
    "  location: '/home/runner/work/node/node/test/fixtures/test-runner/throws_sync_and_async.js:4:1'\n" +
    "  failureType: 'testCodeFailure'\n" +
    "  error: 'fails'\n" +
    "  code: 'ERR_TEST_FAILURE'\n" +
    '  stack: |-\n' +
    '    TestContext.<anonymous> (/home/runner/work/node/node/test/fixtures/test-runner/throws_sync_and_async.js:9:9)\n' +
    '    Test.runInAsyncScope (node:async_hooks:211:14)\n' +
    '    Test.run (node:internal/test_runner/test:887:25)\n' +
    '    Test.start (node:internal/test_runner/test:786:17)\n' +
    '    startSubtestAfterBootstrap (node:internal/test_runner/harness:280:17)\n' +
    '  ...\n' +
    '1..1\n' +
    '# tests 1\n' +
    '# suites 0\n' +
    '# pass 0\n' +
    '# fail 1\n' +
    '# cancelled 0\n' +
    '# skipped 0\n' +
    '# todo 0\n',
  expected: /Error: fails/,
  operator: 'match'
}

Node.js v23.0.0-pre
Command: out/Release/node --test-reporter=spec --test-reporter-destination=stdout --test-reporter=./tools/github_reporter/index.js --test-reporter-destination=stdout /home/runner/work/node/node/test/parallel/test-runner-force-exit-failure.js

Check failure on line 22 in test/parallel/test-runner-force-exit-failure.js

View workflow job for this annotation

GitHub Actions / test-macOS 

--- stderr ---
node:assert:1035
    throw err;
    ^

AssertionError [ERR_ASSERTION]: The input did not match the regular expression /Error: fails/. Input:

'TAP version 13\n' +
  '# Subtest: fails and schedules more work\n' +
  'not ok 1 - fails and schedules more work\n' +
  '  ---\n' +
  '  duration_ms: 0.876458\n' +
  "  location: '/Users/runner/work/node/node/test/fixtures/test-runner/throws_sync_and_async.js:4:1'\n" +
  "  failureType: 'testCodeFailure'\n" +
  "  error: 'fails'\n" +
  "  code: 'ERR_TEST_FAILURE'\n" +
  '  stack: |-\n' +
  '    TestContext.<anonymous> (/Users/runner/work/node/node/test/fixtures/test-runner/throws_sync_and_async.js:9:9)\n' +
  '    Test.runInAsyncScope (node:async_hooks:211:14)\n' +
  '    Test.run (node:internal/test_runner/test:887:25)\n' +
  '    Test.start (node:internal/test_runner/test:786:17)\n' +
  '    startSubtestAfterBootstrap (node:internal/test_runner/harness:280:17)\n' +
  '  ...\n' +
  '1..1\n' +
  '# tests 1\n' +
  '# suites 0\n' +
  '# pass 0\n' +
  '# fail 1\n' +
  '# cancelled 0\n' +
  '# skipped 0\n' +
  '# todo 0\n'

    at Object.<anonymous> (/Users/runner/work/node/node/test/parallel/test-runner-force-exit-failure.js:22:3)
    at Module._compile (node:internal/modules/cjs/loader:1546:14)
    at Module._extensions..js (node:internal/modules/cjs/loader:1691:10)
    at Module.load (node:internal/modules/cjs/loader:1317:32)
    at Module._load (node:internal/modules/cjs/loader:1127:12)
    at TracingChannel.traceSync (node:diagnostics_channel:315:14)
    at wrapModuleLoad (node:internal/modules/cjs/loader:217:24)
    at Function.executeUserEntryPoint [as runMain] (node:internal/modules/run_main:166:5)
    at node:internal/main/run_main_module:30:49 {
  generatedMessage: true,
  code: 'ERR_ASSERTION',
  actual: 'TAP version 13\n' +
    '# Subtest: fails and schedules more work\n' +
    'not ok 1 - fails and schedules more work\n' +
    '  ---\n' +
    '  duration_ms: 0.876458\n' +
    "  location: '/Users/runner/work/node/node/test/fixtures/test-runner/throws_sync_and_async.js:4:1'\n" +
    "  failureType: 'testCodeFailure'\n" +
    "  error: 'fails'\n" +
    "  code: 'ERR_TEST_FAILURE'\n" +
    '  stack: |-\n' +
    '    TestContext.<anonymous> (/Users/runner/work/node/node/test/fixtures/test-runner/throws_sync_and_async.js:9:9)\n' +
    '    Test.runInAsyncScope (node:async_hooks:211:14)\n' +
    '    Test.run (node:internal/test_runner/test:887:25)\n' +
    '    Test.start (node:internal/test_runner/test:786:17)\n' +
    '    startSubtestAfterBootstrap (node:internal/test_runner/harness:280:17)\n' +
    '  ...\n' +
    '1..1\n' +
    '# tests 1\n' +
    '# suites 0\n' +
    '# pass 0\n' +
    '# fail 1\n' +
    '# cancelled 0\n' +
    '# skipped 0\n' +
    '# todo 0\n',
  expected: /Error: fails/,
  operator: 'match'
}

Node.js v23.0.0-pre
Command: out/Release/node --test-reporter=spec --test-reporter-destination=stdout --test-reporter=./tools/github_reporter/index.js --test-reporter-destination=stdout /Users/runner/work/node/node/test/parallel/test-runner-force-exit-failure.js
doesNotMatch(stdout, /this should not have a chance to be thrown/);
}
7 changes: 5 additions & 2 deletions test/parallel/test-runner-inspect.mjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ tmpdir.refresh();

{
const child = new NodeInstance(
['--test', '--inspect-brk=0'],
['--test', '--test-reporter=tap', '--inspect-brk=0'],
undefined,
fixtures.path('test-runner/default-behavior/index.test.js')
);
Expand All @@ -38,7 +38,10 @@ tmpdir.refresh();


{
const args = ['--test', '--inspect=0', fixtures.path('test-runner/index.js')];
const args = [
'--test', '--test-reporter=tap', '--inspect=0',
fixtures.path('test-runner/index.js'),
];
const { stderr, stdout, code, signal } = await common.spawnPromisified(process.execPath, args);

assert.match(stderr,
Expand Down
6 changes: 3 additions & 3 deletions test/parallel/test-runner-misc.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,9 @@ if (process.argv[2] === 'child') {
} else {
const child = spawnSync(process.execPath, [__filename, 'child', 'abortSignal']);
const stdout = child.stdout.toString();
assert.match(stdout, /^# pass 2$/m);
assert.match(stdout, /^# fail 0$/m);
assert.match(stdout, /^# cancelled 1$/m);
assert.match(stdout, /pass 2$/m);
assert.match(stdout, /fail 0$/m);
assert.match(stdout, /cancelled 1$/m);
assert.strictEqual(child.status, 1);
assert.strictEqual(child.signal, null);
}
2 changes: 1 addition & 1 deletion test/parallel/test-runner-module-mocking.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -571,7 +571,7 @@ test('node_modules can be used by both module systems', async (t) => {

assert.strictEqual(code, 0);
assert.strictEqual(signal, null);
assert.match(stdout, /# pass 1/);
assert.match(stdout, /pass 1/);
});

test('file:// imports are supported in ESM only', async (t) => {
Expand Down
3 changes: 3 additions & 0 deletions test/parallel/test-runner-no-isolation-filtering.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ const fixture2 = fixtures.path('test-runner', 'no-isolation', 'two.test.js');
test('works with --test-only', () => {
const args = [
'--test',
'--test-reporter=tap',
'--experimental-test-isolation=none',
'--test-only',
fixture1,
Expand All @@ -33,6 +34,7 @@ test('works with --test-only', () => {
test('works with --test-name-pattern', () => {
const args = [
'--test',
'--test-reporter=tap',
'--experimental-test-isolation=none',
'--test-name-pattern=/test one/',
fixture1,
Expand All @@ -52,6 +54,7 @@ test('works with --test-name-pattern', () => {
test('works with --test-skip-pattern', () => {
const args = [
'--test',
'--test-reporter=tap',
'--experimental-test-isolation=none',
'--test-skip-pattern=/one/',
fixture1,
Expand Down
Loading

0 comments on commit efd6176

Please sign in to comment.