Skip to content

Commit 4e21b20

Browse files
palmerj3palmerj3
authored
Merge pull request #144 from SamTheisens/report-no-results-as-error
Report no results as error
2 parents 808f4a0 + 2d1d722 commit 4e21b20

File tree

9 files changed

+348
-23
lines changed

9 files changed

+348
-23
lines changed

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ Reporter options should also be strings exception for suiteNameTemplate, classNa
6767
| `JEST_JUNIT_ADD_FILE_ATTRIBUTE` | `addFileAttribute` | Add file attribute to the output. This config is primarily for Circle CI. This setting provides richer details but may break on other CI platforms. Must be a string. | `"false"` | N/A
6868
| `JEST_JUNIT_INCLUDE_CONSOLE_OUTPUT` | `includeConsoleOutput` | Adds console output to any testSuite that generates stdout during a test run. | `false` | N/A
6969
| `JEST_JUNIT_INCLUDE_SHORT_CONSOLE_OUTPUT` | `includeShortConsoleOutput` | Adds short console output (only message value) to any testSuite that generates stdout during a test run. | `false` | N/A
70+
| `JEST_JUNIT_REPORT_TEST_SUITE_ERRORS` | `reportTestSuiteErrors` | Reports test suites that failed to execute altogether as `error`. _Note:_ since the suite name cannot be determined from files that fail to load, it will default to file path.| `false` | N/A
7071
| `JEST_USE_PATH_FOR_SUITE_NAME` | `usePathForSuiteName` | **DEPRECATED. Use `suiteNameTemplate` instead.** Use file path as the `name` attribute of `<testsuite>` | `"false"` | N/A
7172

7273

‎__mocks__/empty-suite.json‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
{
2+
"numFailedTestSuites": 1,
3+
"numFailedTests": 0,
4+
"numPassedTestSuites": 0,
5+
"numPassedTests": 0,
6+
"numPendingTestSuites": 0,
7+
"numPendingTests": 0,
8+
"numRuntimeErrorTestSuites": 1,
9+
"numTodoTests": 0,
10+
"numTotalTestSuites": 1,
11+
"numTotalTests": 0,
12+
"openHandles": [],
13+
"snapshot": {
14+
"added": 0,
15+
"didUpdate": false,
16+
"failure": false,
17+
"filesAdded": 0,
18+
"filesRemoved": 0,
19+
"filesRemovedList": [],
20+
"filesUnmatched": 0,
21+
"filesUpdated": 0,
22+
"matched": 0,
23+
"total": 0,
24+
"unchecked": 0,
25+
"uncheckedKeysByFile": [],
26+
"unmatched": 0,
27+
"updated": 0
28+
},
29+
"startTime": 1601808216222,
30+
"success": false,
31+
"testResults": [
32+
{
33+
"failureMessage": " \u001b[1m● \u001b[22mTest suite failed to run\n\n Your test suite must contain at least one test.\n\n \u001b[2mat onResult (\u001b[22mnode_modules/@jest/core/build/TestScheduler.js\u001b[2m:175:18)\u001b[22m\n \u001b[2mat testRunner.on (\u001b[22mnode_modules/@jest/core/build/TestScheduler.js\u001b[2m:304:17)\u001b[22m\n \u001b[2mat Promise.all.staticListeners.map (\u001b[22mnode_modules/emittery/index.js\u001b[2m:260:13)\u001b[22m\n at Array.map (<anonymous>)\n \u001b[2mat Emittery.Typed.emit (\u001b[22mnode_modules/emittery/index.js\u001b[2m:258:23)\u001b[22m\n",
34+
"leaks": false,
35+
"numFailingTests": 0,
36+
"numPassingTests": 0,
37+
"numPendingTests": 0,
38+
"numTodoTests": 0,
39+
"openHandles": [],
40+
"perfStats": {
41+
"end": 0,
42+
"runtime": 0,
43+
"slow": false,
44+
"start": 0
45+
},
46+
"skipped": false,
47+
"snapshot": {
48+
"added": 0,
49+
"fileDeleted": false,
50+
"matched": 0,
51+
"unchecked": 0,
52+
"uncheckedKeys": [],
53+
"unmatched": 0,
54+
"updated": 0
55+
},
56+
"testExecError": {
57+
"message": "Your test suite must contain at least one test.",
58+
"stack": "Error: Your test suite must contain at least one test.\n at onResult ../mpl-modules/test/docker/node_modules/@jest/core/build/TestScheduler.js:175:18)\n at testRunner.on ../mpl-modules/test/docker/node_modules/@jest/core/build/TestScheduler.js:304:17)\n at Promise.all.staticListeners.map ../mpl-modules/test/docker/node_modules/emittery/index.js:260:13)\n at Array.map (<anonymous>)\n at Emittery.Typed.emit ../mpl-modules/test/docker/node_modules/emittery/index.js:258:23)"
59+
},
60+
"testFilePath": "/path/to/spec/test.spec.ts",
61+
"testResults": []
62+
}
63+
],
64+
"wasInterrupted": false
65+
}

‎__mocks__/failing-compilation.json‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
{
2+
"numFailedTestSuites": 1,
3+
"numFailedTests": 0,
4+
"numPassedTestSuites": 0,
5+
"numPassedTests": 0,
6+
"numPendingTestSuites": 0,
7+
"numPendingTests": 0,
8+
"numRuntimeErrorTestSuites": 1,
9+
"numTodoTests": 0,
10+
"numTotalTestSuites": 1,
11+
"numTotalTests": 0,
12+
"openHandles": [],
13+
"snapshot": {
14+
"added": 0,
15+
"didUpdate": false,
16+
"failure": false,
17+
"filesAdded": 0,
18+
"filesRemoved": 0,
19+
"filesRemovedList": [],
20+
"filesUnmatched": 0,
21+
"filesUpdated": 0,
22+
"matched": 0,
23+
"total": 0,
24+
"unchecked": 0,
25+
"uncheckedKeysByFile": [],
26+
"unmatched": 0,
27+
"updated": 0
28+
},
29+
"startTime": 1489712747092,
30+
"success": false,
31+
"testResults": [
32+
{
33+
"displayName": "",
34+
"failureMessage": " \u001b[1m● \u001b[22mTest suite failed to run\n\n TypeScript diagnostics (customize using `[jest-config].globals.ts-jest.diagnostics` option):\n \u001b[96mspec/test.spec.ts\u001b[0m:\u001b[93m10\u001b[0m:\u001b[93m35\u001b[0m - \u001b[91merror\u001b[0m\u001b[90m TS2339: \u001b[0mProperty 'hello' does not exist on type 'HelloScreamer'.\n\n \u001b[7m10\u001b[0m const screamed = screamer.hello();\n \u001b[7m \u001b[0m \u001b[91m ~~~~~\u001b[0m\n",
35+
"leaks": false,
36+
"numFailingTests": 0,
37+
"numPassingTests": 0,
38+
"numPendingTests": 0,
39+
"numTodoTests": 0,
40+
"openHandles": [],
41+
"perfStats": {
42+
"end": 1499904221109,
43+
"start": 1499904215586
44+
},
45+
"skipped": false,
46+
"snapshot": {
47+
"added": 0,
48+
"fileDeleted": false,
49+
"matched": 0,
50+
"unchecked": 0,
51+
"unmatched": 0,
52+
"updated": 0
53+
},
54+
"sourceMaps": {},
55+
"testExecError": "spec/test.spec.ts:10:35 - error TS2339: Property 'hello' does not exist on type 'HelloScreamer\n\n 10 const screamed = screamer.hello();\n~~~~~",
56+
"testFilePath": "/path/to/spec/test.spec.ts",
57+
"testResults": [],
58+
"coverage": {
59+
}
60+
},
61+
{
62+
"console": [],
63+
"failureMessage": null,
64+
"numFailingTests": 0,
65+
"numPassingTests": 1,
66+
"numPendingTests": 0,
67+
"perfStats": {
68+
"end": 1518274351347,
69+
"start": 1518274351274
70+
},
71+
"snapshot": {
72+
"added": 0,
73+
"fileDeleted": false,
74+
"matched": 0,
75+
"unchecked": 0,
76+
"unmatched": 0,
77+
"updated": 0,
78+
"uncheckedKeys": []
79+
},
80+
"testFilePath": "/path/to/project2/__tests__/test2.test.js",
81+
"testResults": [
82+
{
83+
"ancestorTitles": [
84+
"another thing"
85+
],
86+
"duration": 1,
87+
"failureMessages": [],
88+
"fullName": "another thing should foo",
89+
"location": null,
90+
"numPassingAsserts": 0,
91+
"status": "passed",
92+
"title": "should foo"
93+
}
94+
],
95+
"sourceMaps": {},
96+
"skipped": false,
97+
"displayName": "project2",
98+
"leaks": false
99+
}
100+
],
101+
"wasInterrupted": false
102+
}

‎__mocks__/failing-import.json‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
{
2+
"numFailedTestSuites": 1,
3+
"numFailedTests": 0,
4+
"numPassedTestSuites": 1,
5+
"numPassedTests": 1,
6+
"numPendingTestSuites": 0,
7+
"numPendingTests": 0,
8+
"numRuntimeErrorTestSuites": 1,
9+
"numTodoTests": 0,
10+
"numTotalTestSuites": 2,
11+
"numTotalTests": 1,
12+
"openHandles": [],
13+
"snapshot": {
14+
"added": 0,
15+
"didUpdate": false,
16+
"failure": false,
17+
"filesAdded": 0,
18+
"filesRemoved": 0,
19+
"filesRemovedList": [],
20+
"filesUnmatched": 0,
21+
"filesUpdated": 0,
22+
"matched": 0,
23+
"total": 0,
24+
"unchecked": 0,
25+
"uncheckedKeysByFile": [],
26+
"unmatched": 0,
27+
"updated": 0
28+
},
29+
"startTime": 1601544556519,
30+
"success": false,
31+
"testResults": [
32+
{
33+
"displayName": "",
34+
"failureMessage": " \u001b[1m● \u001b[22mTest suite failed to run\n\n Cannot find module './mult' from 'mul.test.js'\n\n \u001b[0m\u001b[31m\u001b[1m>\u001b[22m\u001b[39m\u001b[90m 1 | \u001b[39m\u001b[36mconst\u001b[39m mul \u001b[33m=\u001b[39m require(\u001b[32m'./mult'\u001b[39m)\u001b[33m;\u001b[39m\u001b[0m\n \u001b[0m \u001b[90m | \u001b[39m\u001b[31m\u001b[1m^\u001b[22m\u001b[39m\u001b[0m\n \u001b[0m \u001b[90m 2 | \u001b[39m\u001b[0m\n \u001b[0m \u001b[90m 3 | \u001b[39mtest(\u001b[32m'multplies 2 * 3 to equal 6'\u001b[39m\u001b[33m,\u001b[39m () \u001b[33m=>\u001b[39m {\u001b[0m\n \u001b[0m \u001b[90m 4 | \u001b[39m expect(mul(\u001b[35m2\u001b[39m\u001b[33m,\u001b[39m \u001b[35m3\u001b[39m))\u001b[33m.\u001b[39mtoBe(\u001b[35m6\u001b[39m)\u001b[33m;\u001b[39m\u001b[0m\n\n \u001b[2mat Resolver.resolveModule (\u001b[22mnode_modules/jest-resolve/build/index.js\u001b[2m:259:17)\u001b[22m\n \u001b[2mat Object.<anonymous> (\u001b[22m\u001b[0m\u001b[36msrc/mul.test.js\u001b[39m\u001b[0m\u001b[2m:1:1)\u001b[22m\n",
35+
"leaks": false,
36+
"numFailingTests": 0,
37+
"numPassingTests": 0,
38+
"numPendingTests": 0,
39+
"numTodoTests": 0,
40+
"openHandles": [],
41+
"perfStats": {
42+
"end": 0,
43+
"start": 0
44+
},
45+
"skipped": false,
46+
"snapshot": {
47+
"added": 0,
48+
"fileDeleted": false,
49+
"matched": 0,
50+
"unchecked": 0,
51+
"uncheckedKeys": [],
52+
"unmatched": 0,
53+
"updated": 0
54+
},
55+
"testExecError": {
56+
"code": "MODULE_NOT_FOUND"
57+
},
58+
"testFilePath": "/path/to/spec/test.spec.ts",
59+
"testResults": []
60+
}
61+
],
62+
"wasInterrupted": false
63+
}

‎__tests__/__snapshots__/buildJsonResults.test.js.snap‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ Object {
55
"testsuites": Array [
66
Object {
77
"_attr": Object {
8+
"errors": 0,
89
"failures": 0,
910
"name": "jest tests",
1011
"tests": 2,
@@ -41,7 +42,7 @@ Object {
4142
Object {
4243
"_attr": Object {
4344
"classname": "a thing should foo",
44-
"name": "project1-foo",
45+
"name": "project1-bar",
4546
"time": 0.003,
4647
},
4748
},
@@ -79,7 +80,7 @@ Object {
7980
Object {
8081
"_attr": Object {
8182
"classname": "another thing should foo",
82-
"name": "project2-foo",
83+
"name": "project2-bar",
8384
"time": 0.001,
8485
},
8586
},

‎__tests__/buildJsonResults.test.js‎

Lines changed: 67 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,72 @@ describe('buildJsonResults', () => {
134134
.toBe('function called with vars: filepath, filename, suitename, classname, title, displayName');
135135
});
136136

137+
it('should report no results as error', () => {
138+
const failingTestsReport = require('../__mocks__/failing-compilation.json');
139+
140+
jsonResults = buildJsonResults(failingTestsReport, '/path/to/test',
141+
Object.assign({}, constants.DEFAULT_OPTIONS, {
142+
reportTestSuiteErrors: "true"
143+
}));
144+
145+
const totals = jsonResults.testsuites[0]._attr;
146+
expect(totals.tests).toEqual(1);
147+
expect(totals.errors).toEqual(1);
148+
expect(totals.failures).toEqual(0);
149+
150+
const suiteResult = jsonResults.testsuites[1].testsuite[0]._attr;
151+
expect(suiteResult.name).toEqual('../spec/test.spec.ts')
152+
expect(suiteResult.errors).toEqual(1);
153+
expect(suiteResult.tests).toEqual(0);
154+
155+
const errorSuite = jsonResults.testsuites[1].testsuite[2];
156+
expect(errorSuite.testcase[0]._attr.name).toEqual(suiteResult.name);
157+
expect(errorSuite.testcase[0]._attr.classname).toEqual('Test suite failed to run');
158+
expect(errorSuite.testcase[1].error).toContain("Property 'hello' does not exist");
159+
160+
});
161+
162+
it('should report failureMessage if testExecErrorNotSet', () => {
163+
const failingTestsReport = require('../__mocks__/failing-import.json');
164+
165+
jsonResults = buildJsonResults(failingTestsReport, '/path/to/test',
166+
Object.assign({}, constants.DEFAULT_OPTIONS, {
167+
reportTestSuiteErrors: "true"
168+
}));
169+
170+
const errorSuite = jsonResults.testsuites[1].testsuite[2];
171+
expect(errorSuite.testcase[0]._attr.name).toEqual('../spec/test.spec.ts');
172+
expect(errorSuite.testcase[0]._attr.classname).toEqual('Test suite failed to run');
173+
expect(errorSuite.testcase[1].error).toContain("Cannot find module './mult'");
174+
});
175+
176+
it('should report empty suites as error', () => {
177+
const failingTestsReport = require('../__mocks__/empty-suite.json');
178+
179+
jsonResults = buildJsonResults(failingTestsReport, '/path/to/test',
180+
Object.assign({}, constants.DEFAULT_OPTIONS, {
181+
reportTestSuiteErrors: "true"
182+
}));
183+
184+
const errorSuite = jsonResults.testsuites[1].testsuite[2];
185+
expect(errorSuite.testcase[0]._attr.name).toEqual('../spec/test.spec.ts');
186+
expect(errorSuite.testcase[0]._attr.classname).toEqual('Test suite failed to run');
187+
expect(errorSuite.testcase[1].error).toContain("Your test suite must contain at least one test");
188+
});
189+
190+
it('should honor templates when test has errors', () => {
191+
const failingTestsReport = require('../__mocks__/failing-compilation.json');
192+
193+
jsonResults = buildJsonResults(failingTestsReport, '/path/to/test',
194+
Object.assign({}, constants.DEFAULT_OPTIONS, {
195+
reportTestSuiteErrors: "true",
196+
suiteNameTemplate: "{displayName}-foo",
197+
titleTemplate: "{title}-bar"
198+
}));
199+
200+
expect(jsonResults.testsuites[2].testsuite[2].testcase[0]._attr.name).toEqual('should foo-bar');
201+
});
202+
137203
it('should return the proper filepath when titleTemplate is "{filepath}"', () => {
138204
const noFailingTestsReport = require('../__mocks__/no-failing-tests.json');
139205
jsonResults = buildJsonResults(noFailingTestsReport, '/',
@@ -216,7 +282,7 @@ describe('buildJsonResults', () => {
216282
jsonResults = buildJsonResults(multiProjectNoFailingTestsReport, '/',
217283
Object.assign({}, constants.DEFAULT_OPTIONS, {
218284
suiteNameTemplate: "{displayName}-foo",
219-
titleTemplate: "{displayName}-foo"
285+
titleTemplate: "{displayName}-bar"
220286
}));
221287

222288
expect(jsonResults).toMatchSnapshot();

‎constants/index.js‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ module.exports = {
1313
JEST_JUNIT_ADD_FILE_ATTRIBUTE: 'addFileAttribute',
1414
JEST_JUNIT_INCLUDE_CONSOLE_OUTPUT: 'includeConsoleOutput',
1515
JEST_JUNIT_INCLUDE_SHORT_CONSOLE_OUTPUT: 'includeShortConsoleOutput',
16+
JEST_JUNIT_REPORT_TEST_SUITE_ERRORS: 'reportTestSuiteErrors',
1617
JEST_USE_PATH_FOR_SUITE_NAME: 'usePathForSuiteName',
1718
JEST_JUNIT_TEST_SUITE_PROPERTIES_JSON_FILE: 'testSuitePropertiesFile'
1819
},
@@ -29,6 +30,7 @@ module.exports = {
2930
addFileAttribute: 'false',
3031
includeConsoleOutput: 'false',
3132
includeShortConsoleOutput: 'false',
33+
reportTestSuiteErrors: 'false',
3234
testSuitePropertiesFile: 'junitProperties.js'
3335
},
3436
SUITENAME_VAR: 'suitename',

0 commit comments

Comments
 (0)