From 87a0e866040a568b2180ea7fb8a75ffc71304cde Mon Sep 17 00:00:00 2001 From: Moshe Atlow Date: Tue, 13 Dec 2022 16:51:35 +0200 Subject: [PATCH] test_runner: parse yaml PR-URL: https://github.com/nodejs/node/pull/45815 Reviewed-By: Colin Ihrig Reviewed-By: Benjamin Gruenbaum Reviewed-By: Antoine du Hamel --- lib/internal/test_runner/runner.js | 18 +- lib/internal/test_runner/tap_stream.js | 5 +- lib/internal/test_runner/test.js | 2 +- lib/internal/test_runner/yaml_parser.js | 119 +++++ test/message/test_runner_output_cli.js | 6 + test/message/test_runner_output_cli.out | 654 ++++++++++++++++++++++++ 6 files changed, 785 insertions(+), 19 deletions(-) create mode 100644 lib/internal/test_runner/yaml_parser.js create mode 100644 test/message/test_runner_output_cli.js create mode 100644 test/message/test_runner_output_cli.out diff --git a/lib/internal/test_runner/runner.js b/lib/internal/test_runner/runner.js index 74cb4c150f9c5c..1e5c0dda0fabd9 100644 --- a/lib/internal/test_runner/runner.js +++ b/lib/internal/test_runner/runner.js @@ -4,7 +4,6 @@ const { ArrayPrototypeFilter, ArrayPrototypeForEach, ArrayPrototypeIncludes, - ArrayPrototypeJoin, ArrayPrototypePush, ArrayPrototypeSlice, ArrayPrototypeSort, @@ -35,6 +34,7 @@ const { kEmptyObject } = require('internal/util'); const { createTestTree } = require('internal/test_runner/harness'); const { kDefaultIndent, kSubtestsFailed, Test } = require('internal/test_runner/test'); const { TapParser } = require('internal/test_runner/tap_parser'); +const { YAMLToJs } = require('internal/test_runner/yaml_parser'); const { TokenKind } = require('internal/test_runner/tap_lexer'); const { @@ -129,18 +129,6 @@ class FileTest extends Test { #handleReportItem({ kind, node, nesting = 0 }) { const indent = StringPrototypeRepeat(kDefaultIndent, nesting + 1); - const details = (diagnostic) => { - return ( - diagnostic && { - __proto__: null, - yaml: - `${indent} ` + - ArrayPrototypeJoin(diagnostic, `\n${indent} `) + - '\n', - } - ); - }; - switch (kind) { case TokenKind.TAP_VERSION: // TODO(manekinekko): handle TAP version coming from the parser. @@ -174,7 +162,7 @@ class FileTest extends Test { indent, node.id, node.description, - details(node.diagnostics), + YAMLToJs(node.diagnostics), directive ); } else { @@ -182,7 +170,7 @@ class FileTest extends Test { indent, node.id, node.description, - details(node.diagnostics), + YAMLToJs(node.diagnostics), directive ); } diff --git a/lib/internal/test_runner/tap_stream.js b/lib/internal/test_runner/tap_stream.js index a5834ee46f2014..052f8284c8d931 100644 --- a/lib/internal/test_runner/tap_stream.js +++ b/lib/internal/test_runner/tap_stream.js @@ -83,11 +83,10 @@ class TapStream extends Readable { } #details(indent, data = kEmptyObject) { - const { error, duration, yaml } = data; + const { error, duration_ms } = data; let details = `${indent} ---\n`; - details += `${yaml ? yaml : ''}`; - details += jsToYaml(indent, 'duration_ms', duration); + details += jsToYaml(indent, 'duration_ms', duration_ms); details += jsToYaml(indent, null, error); details += `${indent} ...\n`; this.#tryPush(details); diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index 62f157486a023b..1570b9c55320e6 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -680,7 +680,7 @@ class Test extends AsyncResource { this.reportSubtest(); } let directive; - const details = { __proto__: null, duration: this.#duration() }; + const details = { __proto__: null, duration_ms: this.#duration() }; if (this.skipped) { directive = this.reporter.getSkip(this.message); diff --git a/lib/internal/test_runner/yaml_parser.js b/lib/internal/test_runner/yaml_parser.js new file mode 100644 index 00000000000000..0642501c275989 --- /dev/null +++ b/lib/internal/test_runner/yaml_parser.js @@ -0,0 +1,119 @@ +'use strict'; +const { + codes: { + ERR_TEST_FAILURE, + } +} = require('internal/errors'); +const AssertionError = require('internal/assert/assertion_error'); +const { + ArrayPrototypeJoin, + ArrayPrototypePush, + Error, + Number, + NumberIsNaN, + RegExpPrototypeExec, + StringPrototypeEndsWith, + StringPrototypeRepeat, + StringPrototypeSlice, + StringPrototypeStartsWith, + StringPrototypeSubstring, +} = primordials; + +const kYamlKeyRegex = /^(\s+)?(\w+):(\s)+([>|][-+])?(.*)$/; +const kStackDelimiter = ' at '; + +function reConstructError(parsedYaml) { + if (!('error' in parsedYaml)) { + return parsedYaml; + } + const isAssertionError = parsedYaml.code === 'ERR_ASSERTION' || + 'actual' in parsedYaml || 'expected' in parsedYaml || 'operator' in parsedYaml; + const isTestFailure = parsedYaml.code === 'ERR_TEST_FAILURE' || 'failureType' in parsedYaml; + const stack = parsedYaml.stack ? kStackDelimiter + ArrayPrototypeJoin(parsedYaml.stack, `\n${kStackDelimiter}`) : ''; + let error, cause; + + if (isAssertionError) { + cause = new AssertionError({ + message: parsedYaml.error, + actual: parsedYaml.actual, + expected: parsedYaml.expected, + operator: parsedYaml.operator + }); + } else { + // eslint-disable-next-line no-restricted-syntax + cause = new Error(parsedYaml.error); + cause.code = parsedYaml.code; + } + cause.stack = stack; + + if (isTestFailure) { + error = new ERR_TEST_FAILURE(cause, parsedYaml.failureType); + error.stack = stack; + } + + parsedYaml.error = error ?? cause; + delete parsedYaml.stack; + delete parsedYaml.code; + delete parsedYaml.failureType; + delete parsedYaml.actual; + delete parsedYaml.expected; + delete parsedYaml.operator; + + return parsedYaml; +} + +function getYamlValue(value) { + if (StringPrototypeStartsWith(value, "'") && StringPrototypeEndsWith(value, "'")) { + return StringPrototypeSlice(value, 1, -1); + } + if (value === 'true') { + return true; + } + if (value === 'false') { + return false; + } + if (value !== '') { + const valueAsNumber = Number(value); + return NumberIsNaN(valueAsNumber) ? value : valueAsNumber; + } + return value; +} + +// This parses the YAML generated by the built-in TAP reporter, +// which is a subset of the full YAML spec. There are some +// YAML features that won't be parsed here. This function should not be exposed publicly. +function YAMLToJs(lines) { + if (lines == null) { + return undefined; + } + const result = { __proto__: null }; + let isInYamlBlock = false; + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + if (isInYamlBlock && !StringPrototypeStartsWith(line, StringPrototypeRepeat(' ', isInYamlBlock.indent))) { + result[isInYamlBlock.key] = isInYamlBlock.key === 'stack' ? + result[isInYamlBlock.key] : ArrayPrototypeJoin(result[isInYamlBlock.key], '\n'); + isInYamlBlock = false; + } + if (isInYamlBlock) { + const blockLine = StringPrototypeSubstring(line, isInYamlBlock.indent); + ArrayPrototypePush(result[isInYamlBlock.key], blockLine); + continue; + } + const match = RegExpPrototypeExec(kYamlKeyRegex, line); + if (match !== null) { + const { 1: leadingSpaces, 2: key, 4: block, 5: value } = match; + if (block) { + isInYamlBlock = { key, indent: (leadingSpaces?.length ?? 0) + 2 }; + result[key] = []; + } else { + result[key] = getYamlValue(value); + } + } + } + return reConstructError(result); +} + +module.exports = { + YAMLToJs, +}; diff --git a/test/message/test_runner_output_cli.js b/test/message/test_runner_output_cli.js new file mode 100644 index 00000000000000..1058d903c5fee4 --- /dev/null +++ b/test/message/test_runner_output_cli.js @@ -0,0 +1,6 @@ +// Flags: --no-warnings +'use strict'; +require('../common'); +const spawn = require('node:child_process').spawn; +spawn(process.execPath, + ['--no-warnings', '--test', 'test/message/test_runner_output.js'], { stdio: 'inherit' }); diff --git a/test/message/test_runner_output_cli.out b/test/message/test_runner_output_cli.out new file mode 100644 index 00000000000000..b33d3e0fbf50b1 --- /dev/null +++ b/test/message/test_runner_output_cli.out @@ -0,0 +1,654 @@ +TAP version 13 +# Subtest: *test_runner_output.js + # Subtest: sync pass todo + ok 1 - sync pass todo # TODO + --- + duration_ms: * + ... + # Subtest: sync pass todo with message + ok 2 - sync pass todo with message # TODO this is a passing todo + --- + duration_ms: * + ... + # Subtest: sync fail todo + not ok 3 - sync fail todo # TODO + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'thrown from sync fail todo' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + * + * + * + * + * + ... + # Subtest: sync fail todo with message + not ok 4 - sync fail todo with message # TODO this is a failing todo + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'thrown from sync fail todo with message' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + * + * + * + * + * + ... + # Subtest: sync skip pass + ok 5 - sync skip pass # SKIP + --- + duration_ms: * + ... + # Subtest: sync skip pass with message + ok 6 - sync skip pass with message # SKIP this is skipped + --- + duration_ms: * + ... + # Subtest: sync pass + ok 7 - sync pass + --- + duration_ms: * + ... + # Subtest: sync throw fail + not ok 8 - sync throw fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'thrown from sync throw fail' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + * + * + * + * + * + ... + # Subtest: async skip pass + ok 9 - async skip pass # SKIP + --- + duration_ms: * + ... + # Subtest: async pass + ok 10 - async pass + --- + duration_ms: * + ... + # Subtest: async throw fail + not ok 11 - async throw fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'thrown from async throw fail' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + * + * + * + * + * + ... + # Subtest: async skip fail + not ok 12 - async skip fail # SKIP + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'thrown from async throw fail' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + * + * + * + * + * + ... + # Subtest: async assertion fail + not ok 13 - async assertion fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: |- + Expected values to be strictly equal: + + true !== false + + code: 'ERR_ASSERTION' + expected: false + actual: true + operator: 'strictEqual' + stack: |- + * + * + * + * + * + * + * + * + ... + # Subtest: resolve pass + ok 14 - resolve pass + --- + duration_ms: * + ... + # Subtest: reject fail + not ok 15 - reject fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'rejected from reject fail' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + * + * + * + * + * + ... + # Subtest: unhandled rejection - passes but warns + ok 16 - unhandled rejection - passes but warns + --- + duration_ms: * + ... + # Subtest: async unhandled rejection - passes but warns + ok 17 - async unhandled rejection - passes but warns + --- + duration_ms: * + ... + # Subtest: immediate throw - passes but warns + ok 18 - immediate throw - passes but warns + --- + duration_ms: * + ... + # Subtest: immediate reject - passes but warns + ok 19 - immediate reject - passes but warns + --- + duration_ms: * + ... + # Subtest: immediate resolve pass + ok 20 - immediate resolve pass + --- + duration_ms: * + ... + # Subtest: subtest sync throw fail + # Subtest: +sync throw fail + not ok 1 - +sync throw fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'thrown from subtest sync throw fail' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + * + * + * + * + * + * + * + ... + 1..1 + not ok 21 - subtest sync throw fail + --- + duration_ms: * + failureType: 'subtestsFailed' + error: '1 subtest failed' + code: 'ERR_TEST_FAILURE' + ... + # Subtest: sync throw non-error fail + not ok 22 - sync throw non-error fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'Symbol(thrown symbol from sync throw non-error fail)' + code: 'ERR_TEST_FAILURE' + ... + # Subtest: level 0a + # Subtest: level 1a + ok 1 - level 1a + --- + duration_ms: * + ... + # Subtest: level 1b + ok 2 - level 1b + --- + duration_ms: * + ... + # Subtest: level 1c + ok 3 - level 1c + --- + duration_ms: * + ... + # Subtest: level 1d + ok 4 - level 1d + --- + duration_ms: * + ... + 1..4 + ok 23 - level 0a + --- + duration_ms: * + ... + # Subtest: top level + # Subtest: +long running + not ok 1 - +long running + --- + duration_ms: * + failureType: 'cancelledByParent' + error: 'test did not finish before its parent and was cancelled' + code: 'ERR_TEST_FAILURE' + ... + # Subtest: +short running + # Subtest: ++short running + ok 1 - ++short running + --- + duration_ms: * + ... + 1..1 + ok 2 - +short running + --- + duration_ms: * + ... + 1..2 + not ok 24 - top level + --- + duration_ms: * + failureType: 'subtestsFailed' + error: '1 subtest failed' + code: 'ERR_TEST_FAILURE' + ... + # Subtest: invalid subtest - pass but subtest fails + ok 25 - invalid subtest - pass but subtest fails + --- + duration_ms: * + ... + # Subtest: sync skip option + ok 26 - sync skip option # SKIP + --- + duration_ms: * + ... + # Subtest: sync skip option with message + ok 27 - sync skip option with message # SKIP this is skipped + --- + duration_ms: * + ... + # Subtest: sync skip option is false fail + not ok 28 - sync skip option is false fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'this should be executed' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + * + * + * + * + ... + # Subtest: + ok 29 - + --- + duration_ms: * + ... + # Subtest: functionOnly + ok 30 - functionOnly + --- + duration_ms: * + ... + # Subtest: + ok 31 - + --- + duration_ms: * + ... + # Subtest: test with only a name provided + ok 32 - test with only a name provided + --- + duration_ms: * + ... + # Subtest: + ok 33 - + --- + duration_ms: * + ... + # Subtest: + ok 34 - # SKIP + --- + duration_ms: * + ... + # Subtest: test with a name and options provided + ok 35 - test with a name and options provided # SKIP + --- + duration_ms: * + ... + # Subtest: functionAndOptions + ok 36 - functionAndOptions # SKIP + --- + duration_ms: * + ... + # Subtest: escaped description \\ \# \\\#\\ n \\t f \\v b \\r + ok 37 - escaped description \\ \# \\\#\\ n \\t f \\v b \\r + --- + duration_ms: * + ... + # Subtest: escaped skip message + ok 38 - escaped skip message # SKIP \#skip + --- + duration_ms: * + ... + # Subtest: escaped todo message + ok 39 - escaped todo message # TODO \#todo + --- + duration_ms: * + ... + # Subtest: escaped diagnostic + ok 40 - escaped diagnostic + --- + duration_ms: * + ... + # Subtest: callback pass + ok 41 - callback pass + --- + duration_ms: * + ... + # Subtest: callback fail + not ok 42 - callback fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'callback failure' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + ... + # Subtest: sync t is this in test + ok 43 - sync t is this in test + --- + duration_ms: * + ... + # Subtest: async t is this in test + ok 44 - async t is this in test + --- + duration_ms: * + ... + # Subtest: callback t is this in test + ok 45 - callback t is this in test + --- + duration_ms: * + ... + # Subtest: callback also returns a Promise + not ok 46 - callback also returns a Promise + --- + duration_ms: * + failureType: 'callbackAndPromisePresent' + error: 'passed a callback but also returned a Promise' + code: 'ERR_TEST_FAILURE' + ... + # Subtest: callback throw + not ok 47 - callback throw + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'thrown from callback throw' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + * + * + * + * + ... + # Subtest: callback called twice + not ok 48 - callback called twice + --- + duration_ms: * + failureType: 'multipleCallbackInvocations' + error: 'callback invoked multiple times' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + ... + # Subtest: callback called twice in different ticks + ok 49 - callback called twice in different ticks + --- + duration_ms: * + ... + # Subtest: callback called twice in future tick + not ok 50 - callback called twice in future tick + --- + duration_ms: * + failureType: 'uncaughtException' + error: 'callback invoked multiple times' + code: 'ERR_TEST_FAILURE' + stack: |- + * + ... + # Subtest: callback async throw + not ok 51 - callback async throw + --- + duration_ms: * + failureType: 'uncaughtException' + error: 'thrown from callback async throw' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + ... + # Subtest: callback async throw after done + ok 52 - callback async throw after done + --- + duration_ms: * + ... + # Subtest: only is set but not in only mode + # Subtest: running subtest 1 + ok 1 - running subtest 1 + --- + duration_ms: * + ... + # Subtest: running subtest 2 + ok 2 - running subtest 2 + --- + duration_ms: * + ... + # Subtest: running subtest 3 + ok 3 - running subtest 3 + --- + duration_ms: * + ... + # Subtest: running subtest 4 + ok 4 - running subtest 4 + --- + duration_ms: * + ... + 1..4 + ok 53 - only is set but not in only mode + --- + duration_ms: * + ... + # Subtest: custom inspect symbol fail + not ok 54 - custom inspect symbol fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'customized' + code: 'ERR_TEST_FAILURE' + ... + # Subtest: custom inspect symbol that throws fail + not ok 55 - custom inspect symbol that throws fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: |- + { + foo: 1, + [Symbol(nodejs.util.inspect.custom)]: [Function: [nodejs.util.inspect.custom]] + } + code: 'ERR_TEST_FAILURE' + ... + # Subtest: subtest sync throw fails + # Subtest: sync throw fails at first + not ok 1 - sync throw fails at first + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'thrown from subtest sync throw fails at first' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + * + * + * + * + * + * + * + ... + # Subtest: sync throw fails at second + not ok 2 - sync throw fails at second + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'thrown from subtest sync throw fails at second' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + * + * + * + * + * + * + * + ... + 1..2 + not ok 56 - subtest sync throw fails + --- + duration_ms: * + failureType: 'subtestsFailed' + error: '2 subtests failed' + code: 'ERR_TEST_FAILURE' + ... + # Subtest: timed out async test + not ok 57 - timed out async test + --- + duration_ms: * + failureType: 'testTimeoutFailure' + error: 'test timed out after 5ms' + code: 'ERR_TEST_FAILURE' + ... + # Subtest: timed out callback test + not ok 58 - timed out callback test + --- + duration_ms: * + failureType: 'testTimeoutFailure' + error: 'test timed out after 5ms' + code: 'ERR_TEST_FAILURE' + ... + # Subtest: large timeout async test is ok + ok 59 - large timeout async test is ok + --- + duration_ms: * + ... + # Subtest: large timeout callback test is ok + ok 60 - large timeout callback test is ok + --- + duration_ms: * + ... + # Subtest: successful thenable + ok 61 - successful thenable + --- + duration_ms: * + ... + # Subtest: rejected thenable + not ok 62 - rejected thenable + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'custom error' + code: 'ERR_TEST_FAILURE' + ... + # Subtest: unfinished test with uncaughtException + not ok 63 - unfinished test with uncaughtException + --- + duration_ms: * + failureType: 'uncaughtException' + error: 'foo' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + ... + # Subtest: unfinished test with unhandledRejection + not ok 64 - unfinished test with unhandledRejection + --- + duration_ms: * + failureType: 'unhandledRejection' + error: 'bar' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + ... + # Subtest: invalid subtest fail + not ok 65 - invalid subtest fail + --- + duration_ms: * + failureType: 'parentAlreadyFinished' + error: 'test could not be started because its parent finished' + code: 'ERR_TEST_FAILURE' + stack: |- + * + ... + 1..65 +not ok 1 - *test_runner_output.js + --- + duration_ms: * + failureType: 'subtestsFailed' + exitCode: 1 + error: 'test failed' + code: 'ERR_TEST_FAILURE' + ... +1..1 +# tests 1 +# pass 0 +# fail 1 +# cancelled 0 +# skipped 0 +# todo 0 +# duration_ms *