Skip to content

Commit

Permalink
Merge pull request #1238 from saturninoabril/set-output-retried
Browse files Browse the repository at this point in the history 
Add `retried` to output
  • Loading branch information
@mikepenz
mikepenz authored Nov 28, 2024
2 parents 0bec6a4 + 319aaf3 commit 1e85223
Show file tree
Hide file tree
Showing 6 changed files with 122 additions and 5 deletions.
66 changes: 66 additions & 0 deletions __tests__/testParser.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1269,6 +1269,71 @@ describe('parseTestReports', () => {
])
})

it('should parse retried tests', async () => {
const {checkName, summary, totalCount, skipped, failed, passed, retried, globalAnnotations} =
await parseTestReports(
'checkName',
'summary',
'test_results/junit-server-test/report.xml',
'*',
true,
true,
true,
[],
'{{SUITE_NAME}}/{{TEST_NAME}}',
'/'
)

expect(checkName).toBe('checkName')
expect(summary).toBe('summary')
expect(totalCount).toBe(3)
expect(skipped).toBe(1)
expect(failed).toBe(0)
expect(passed).toBe(2)
expect(retried).toBe(1)
expect(globalAnnotations).toStrictEqual([
{
path: 'com/example/example/server/v8/channels/api4',
start_line: 1,
end_line: 1,
start_column: 0,
end_column: 0,
retries: 1,
annotation_level: 'notice',
status: 'success',
title: 'github.com/example/example/server/v8/channels/api4/TestWebSocketReconnectRace',
message: 'TestWebSocketReconnectRace',
raw_details: ''
},
{
annotation_level: 'notice',
end_column: 0,
end_line: 1,
message: 'TestCreateChannelBookmark',
path: 'com/example/example/server/v8/channels/api4',
raw_details: '',
retries: 0,
start_column: 0,
start_line: 1,
status: 'skipped',
title: 'github.com/example/example/server/v8/channels/api4/TestCreateChannelBookmark'
},
{
annotation_level: 'notice',
end_column: 0,
end_line: 1,
message: 'TestWebSocketUpgrade',
path: 'com/example/example/server/v8/channels/api4',
raw_details: '',
retries: 0,
start_column: 0,
start_line: 1,
status: 'success',
title: 'github.com/example/example/server/v8/channels/api4/TestWebSocketUpgrade'
}
])
})

it('parse corrupt test output', async () => {
const result = await parseTestReports(
'',
Expand Down Expand Up @@ -1296,6 +1361,7 @@ describe('parseTestReports', () => {
foundFiles: 1,
globalAnnotations: [],
passed: 0,
retried: 0,
testResults: []
})
})
Expand Down
20 changes: 18 additions & 2 deletions dist/index.js
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion dist/index.js.map
View file

Large diffs are not rendered by default.

3 changes: 3 additions & 0 deletions src/main.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ export async function run(): Promise<void> {
skipped: 0,
failed: 0,
passed: 0,
retried: 0,
foundFiles: 0,
globalAnnotations: [],
testResults: []
Expand Down Expand Up @@ -96,13 +97,15 @@ export async function run(): Promise<void> {
mergedResult.skipped += testResult.skipped
mergedResult.failed += testResult.failed
mergedResult.passed += testResult.passed
mergedResult.retried += testResult.retried
testResults.push(testResult)
}

core.setOutput('total', mergedResult.totalCount)
core.setOutput('passed', mergedResult.passed)
core.setOutput('skipped', mergedResult.skipped)
core.setOutput('failed', mergedResult.failed)
core.setOutput('retried', mergedResult.retried)

if (!(mergedResult.totalCount > 0 || mergedResult.skipped > 0) && requireTests) {
core.setFailed(`❌ No test results found for ${checkName}`)
Expand Down
20 changes: 18 additions & 2 deletions src/testParser.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ export interface ActualTestResult {
name: string
totalCount: number
skippedCount: number
retriedCount: number
annotations: Annotation[]
globalAnnotations: Annotation[]
testResults: ActualTestResult[]
Expand All @@ -17,6 +18,7 @@ export interface ActualTestResult {
interface TestCasesResult {
totalCount: number
skippedCount: number
retriedCount: number
annotations: Annotation[]
}

Expand All @@ -27,6 +29,7 @@ export interface TestResult {
skipped: number
failed: number
passed: number
retried: number
foundFiles: number
globalAnnotations: Annotation[]
testResults: ActualTestResult[]
Expand Down Expand Up @@ -271,6 +274,7 @@ async function parseSuite(

let totalCount = 0
let skippedCount = 0
let retriedCount = 0
const annotations: Annotation[] = []

// parse testCases
Expand Down Expand Up @@ -300,6 +304,7 @@ async function parseSuite(
// expand global annotations array
totalCount += parsedTestCases.totalCount
skippedCount += parsedTestCases.skippedCount
retriedCount += parsedTestCases.retriedCount
annotations.push(...parsedTestCases.annotations)
globalAnnotations.push(...parsedTestCases.annotations)
}
Expand All @@ -309,6 +314,7 @@ async function parseSuite(
name: suiteName,
totalCount,
skippedCount,
retriedCount,
annotations,
globalAnnotations,
testResults: []
Expand Down Expand Up @@ -349,6 +355,7 @@ async function parseSuite(
childSuiteResults.push(childSuiteResult)
totalCount += childSuiteResult.totalCount
skippedCount += childSuiteResult.skippedCount
retriedCount += childSuiteResult.retriedCount
}

// skip out if we reached our annotations limit
Expand All @@ -357,6 +364,7 @@ async function parseSuite(
name: suiteName,
totalCount,
skippedCount,
retriedCount,
annotations,
globalAnnotations,
testResults: childSuiteResults
Expand All @@ -368,6 +376,7 @@ async function parseSuite(
name: suiteName,
totalCount,
skippedCount,
retriedCount,
annotations,
globalAnnotations,
testResults: childSuiteResults
Expand All @@ -394,6 +403,7 @@ async function parseTestCases(
const annotations: Annotation[] = []
let totalCount = 0
let skippedCount = 0
let retriedCount = 0
if (checkRetries) {
// identify duplicates, in case of flaky tests, and remove them
const testcaseMap = new Map<string, any>()
Expand All @@ -407,12 +417,14 @@ async function parseTestCases(
if (failed && !previousFailed) {
// previous is a success, drop failure
previous.retries = (previous.retries || 0) + 1
retriedCount += 1
core.debug(`Drop flaky test failure for (1): ${key}`)
} else if (!failed && previousFailed) {
// previous failed, new one not, replace
testcase.retries = (previous.retries || 0) + 1
testcaseMap.set(key, testcase)
core.debug(`Drop flaky test failure for (2): ${key}`)
retriedCount += 1
core.debug(`Drop flaky test failure for (2): ${JSON.stringify(testcase)}`)
}
} else {
testcaseMap.set(key, testcase)
Expand Down Expand Up @@ -551,6 +563,7 @@ async function parseTestCases(
return {
totalCount,
skippedCount,
retriedCount,
annotations
}
}
Expand Down Expand Up @@ -586,6 +599,7 @@ export async function parseTestReports(
const testResults: ActualTestResult[] = []
let totalCount = 0
let skipped = 0
let retried = 0
let foundFiles = 0
for await (const file of globber.globGenerator()) {
foundFiles++
Expand All @@ -610,9 +624,10 @@ export async function parseTestReports(
)

if (!testResult) continue
const {totalCount: c, skippedCount: s} = testResult
const {totalCount: c, skippedCount: s, retriedCount: r} = testResult
totalCount += c
skipped += s
retried += r
testResults.push(testResult)

if (annotationsLimit > 0 && globalAnnotations.length >= annotationsLimit) {
Expand All @@ -631,6 +646,7 @@ export async function parseTestReports(
skipped,
failed,
passed,
retried,
foundFiles,
globalAnnotations,
testResults
Expand Down
16 changes: 16 additions & 0 deletions test_results/junit-server-test/report.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="3" failures="1" errors="0" time="24.733407">
<testsuite tests="3" failures="1" time="10.715000" name="github.com/example/example/server/v8/channels/api4" timestamp="2024-10-24T12:07:54Z">
<properties>
<property name="go.version" value="go1.22.6 linux/amd64"></property>
</properties>
<testcase classname="github.com/example/example/server/v8/channels/api4" name="TestWebSocketReconnectRace" time="3.320000">
<failure message="Failed" type="">=== RUN TestWebSocketReconnectRace --- FAIL: TestWebSocketReconnectRace (3.32s)</failure>
</testcase>
<testcase classname="github.com/example/example/server/v8/channels/api4" name="TestCreateChannelBookmark" time="0.000000">
<skipped message="=== RUN TestCreateChannelBookmark --- SKIP: TestCreateChannelBookmark (0.00s)"></skipped>
</testcase>
<testcase classname="github.com/example/example/server/v8/channels/api4" name="TestWebSocketUpgrade" time="2.170000"></testcase>
<testcase classname="github.com/example/example/server/v8/channels/api4" name="TestWebSocketReconnectRace" time="7.780000"></testcase>
</testsuite>
</testsuites>

0 comments on commit 1e85223

Please sign in to comment.