Skip to content

Commit 24558d4

Browse files
komishkomish
committed
Adds support for writing supplemental junitxml reports
Signed-off-by: Jose R. Gonzalez <[email protected]>
1 parent 6b10abd commit 24558d4

File tree

4 files changed

+192
-2
lines changed

4 files changed

+192
-2
lines changed

‎cmd/verify.go

+20
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,11 @@ package cmd
1919
import (
2020
"errors"
2121
"fmt"
22+
"os"
2223
"strings"
2324
"time"
2425

26+
"github.com/redhat-certification/chart-verifier/internal/chartverifier/junitxml"
2527
"github.com/redhat-certification/chart-verifier/internal/chartverifier/utils"
2628
"github.com/redhat-certification/chart-verifier/internal/tool"
2729
apiChecks "github.com/redhat-certification/chart-verifier/pkg/chartverifier/checks"
@@ -67,6 +69,8 @@ var (
6769
pgpPublicKeyFile string
6870
// helm install timeout
6971
helmInstallTimeout time.Duration
72+
// writeJUnitXMLTo is where to write an additional junitxml representation of the outcome
73+
writeJUnitXMLTo string
7074
)
7175

7276
func buildChecks(enabled []string, unEnabled []string) ([]apiChecks.CheckName, []apiChecks.CheckName, error) {
@@ -215,6 +219,20 @@ func NewVerifyCmd(config *viper.Viper) *cobra.Command {
215219
return reportErr
216220
}
217221

222+
// Failure to write JUnitXML result is non-fatal because junitxml reports are considered extra.
223+
if writeJUnitXMLTo != "" {
224+
utils.LogInfo(fmt.Sprintf("user requested additional junitxml report be written to %s", writeJUnitXMLTo))
225+
junitOutput, err := junitxml.Format(*verifier.GetReport())
226+
if err != nil {
227+
utils.LogError(fmt.Sprintf("failed to convert report content to junitxml: %s", err))
228+
} else {
229+
err = os.WriteFile(writeJUnitXMLTo, junitOutput, 0644)
230+
if err != nil {
231+
utils.LogError(fmt.Sprintf("failed to write junitxml output to specified path %s: %s", writeJUnitXMLTo, err))
232+
}
233+
}
234+
}
235+
218236
utils.WriteStdOut(report)
219237

220238
utils.WriteLogs(outputFormatFlag)
@@ -250,6 +268,8 @@ func NewVerifyCmd(config *viper.Viper) *cobra.Command {
250268
cmd.Flags().BoolVarP(&webCatalogOnly, "web-catalog-only", "W", false, "set this to indicate that the distribution method is web catalog only (default: false)")
251269
cmd.Flags().StringVarP(&pgpPublicKeyFile, "pgp-public-key", "k", "", "file containing gpg public key of the key used to sign the chart")
252270
cmd.Flags().DurationVar(&helmInstallTimeout, "helm-install-timeout", 5*time.Minute, "helm install timeout")
271+
cmd.Flags().StringVar(&writeJUnitXMLTo, "write-junitxml-to", "", "If set, will write a junitXML representation of the result to the specified path in addition to the configured output format")
272+
253273
return cmd
254274
}
255275

‎docs/helm-chart-checks.md

+20
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,7 @@ This section provides help on the basic usage of Helm chart checks with the podm
148148
-f, --set-values strings specify application and check configuration values in a YAML file or a URL (can specify multiple)
149149
-E, --suppress-error-log suppress the error log (default: written to ./chartverifier/verifier-<timestamp>.log)
150150
--timeout duration time to wait for completion of chart install and test (default 30m0s)
151+
--write-junitxml-to string If set, will write a junitXML representation of the result to the specified path in addition to the configured output format
151152
-w, --write-to-file write report to ./chartverifier/report.yaml (default: stdout)
152153
Global Flags:
153154
--config string config file (default is $HOME/.chart-verifier.yaml)
@@ -239,6 +240,25 @@ Alternatively, use the ```-w``` option to write the report directly to the file
239240
```
240241
If the file already exists it is overwritten.
241242
243+
An additional report can be written in JUnit XML format if requested with the
244+
`--write-junitxml-to` flag, passing in the desired output filename.
245+
246+
```
247+
$ podman run --rm -i \
248+
-e KUBECONFIG=/.kube/config \
249+
-v "${HOME}/.kube":/.kube:z \
250+
-v $(pwd)/chartverifier:/app/chartverifier:z \
251+
-w \
252+
"quay.io/redhat-certification/chart-verifier" \
253+
verify \
254+
--write-junitxml-to /app/chartverifier/report-junit.xml \
255+
<chart-uri>
256+
```
257+
258+
JUnitXML is not an additional report format that can be used for certification
259+
or validation using chart-verifier, and is only intended to be consumed by user
260+
tooling. The YAML or JSON report is always written as specified.
261+
242262
### The error log
243263
244264
By default an error log is written to file ```./chartverifier/verify-<timestamp>.yaml```. It includes any error messages, the results of each check and additional information around chart testing. To get a copy of the error log a volume mount is required to ```/app/chartverifer```. For example:

‎internal/chartverifier/junitxml/junitxml.go

+150
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
package junitxml
2+
3+
import (
4+
"encoding/xml"
5+
"fmt"
6+
"strconv"
7+
8+
"github.com/redhat-certification/chart-verifier/pkg/chartverifier/report"
9+
)
10+
11+
type JUnitTestSuites struct {
12+
XMLName xml.Name `xml:"testsuites"`
13+
Suites []JUnitTestSuite `xml:"testsuite"`
14+
}
15+
16+
type JUnitTestSuite struct {
17+
XMLName xml.Name `xml:"testsuite"`
18+
Tests int `xml:"tests,attr"`
19+
Failures int `xml:"failures,attr"`
20+
Skipped int `xml:"skipped,attr"`
21+
Unknown int `xml:"unknown,attr"`
22+
ReportDigest string `xml:"reportDigest,attr"`
23+
Name string `xml:"name,attr"`
24+
Properties []JUnitProperty `xml:"properties>property,omitempty"`
25+
TestCases []JUnitTestCase `xml:"testcase"`
26+
}
27+
28+
type JUnitTestCase struct {
29+
XMLName xml.Name `xml:"testcase"`
30+
Classname string `xml:"classname,attr"`
31+
Name string `xml:"name,attr"`
32+
SkipMessage *JUnitSkipMessage `xml:"skipped,omitempty"`
33+
Failure *JUnitMessage `xml:"failure,omitempty"`
34+
Warning *JUnitMessage `xml:"warning,omitempty"`
35+
SystemOut string `xml:"system-out,omitempty"`
36+
Message string `xml:",chardata"`
37+
}
38+
39+
type JUnitSkipMessage struct {
40+
Message string `xml:"message,attr"`
41+
}
42+
43+
type JUnitProperty struct {
44+
Name string `xml:"name,attr"`
45+
Value string `xml:"value,attr"`
46+
}
47+
48+
type JUnitMessage struct {
49+
Message string `xml:"message,attr"`
50+
Type string `xml:"type,attr"`
51+
Contents string `xml:",chardata"`
52+
}
53+
54+
func Format(r report.Report) ([]byte, error) {
55+
results := r.Results
56+
checksByOutcome := map[string][]report.CheckReport{}
57+
58+
for i, result := range results {
59+
checksByOutcome[result.Outcome] = append(checksByOutcome[result.Outcome], *results[i])
60+
}
61+
62+
digest, err := r.GetReportDigest()
63+
if err != nil {
64+
// Prefer to continue even if digest calculation fails for some reason.
65+
digest = "unknown"
66+
}
67+
68+
testsuite := JUnitTestSuite{
69+
Tests: len(results),
70+
Failures: len(checksByOutcome[report.FailOutcomeType]),
71+
Skipped: len(checksByOutcome[report.SkippedOutcomeType]),
72+
Unknown: len(checksByOutcome[report.UnknownOutcomeType]),
73+
ReportDigest: digest,
74+
Name: "Red Hat Helm Chart Certification",
75+
Properties: []JUnitProperty{
76+
{Name: "profileType", Value: r.Metadata.ToolMetadata.Profile.VendorType},
77+
{Name: "profileVersion", Value: r.Metadata.ToolMetadata.Profile.Version},
78+
{Name: "webCatalogOnly", Value: strconv.FormatBool(r.Metadata.ToolMetadata.ProviderDelivery || r.Metadata.ToolMetadata.WebCatalogOnly)},
79+
{Name: "verifierVersion", Value: r.Metadata.ToolMetadata.Version},
80+
},
81+
TestCases: []JUnitTestCase{},
82+
}
83+
84+
for _, tc := range checksByOutcome[report.PassOutcomeType] {
85+
c := JUnitTestCase{
86+
Classname: r.Metadata.ToolMetadata.ChartUri,
87+
Name: string(tc.Check),
88+
Failure: nil,
89+
Message: tc.Reason,
90+
}
91+
testsuite.TestCases = append(testsuite.TestCases, c)
92+
}
93+
94+
for _, tc := range checksByOutcome[report.FailOutcomeType] {
95+
c := JUnitTestCase{
96+
Classname: r.Metadata.ToolMetadata.ChartUri,
97+
Name: string(tc.Check),
98+
Failure: &JUnitMessage{
99+
Message: "Failed",
100+
Type: string(tc.Type),
101+
Contents: tc.Reason,
102+
},
103+
Message: tc.Reason,
104+
}
105+
testsuite.TestCases = append(testsuite.TestCases, c)
106+
}
107+
108+
for _, tc := range checksByOutcome[report.UnknownOutcomeType] {
109+
c := JUnitTestCase{
110+
Classname: r.Metadata.ToolMetadata.ChartUri,
111+
Name: string(tc.Check),
112+
Failure: &JUnitMessage{
113+
Message: "Unknown",
114+
Type: string(tc.Type),
115+
Contents: tc.Reason,
116+
},
117+
Message: tc.Reason,
118+
}
119+
testsuite.TestCases = append(testsuite.TestCases, c)
120+
}
121+
122+
for _, tc := range checksByOutcome[report.SkippedOutcomeType] {
123+
c := JUnitTestCase{
124+
Classname: r.Metadata.ToolMetadata.ChartUri,
125+
Name: string(tc.Check),
126+
Failure: nil,
127+
Message: tc.Reason,
128+
SkipMessage: &JUnitSkipMessage{
129+
Message: tc.Reason,
130+
},
131+
}
132+
testsuite.TestCases = append(testsuite.TestCases, c)
133+
}
134+
135+
suites := JUnitTestSuites{
136+
Suites: []JUnitTestSuite{testsuite},
137+
}
138+
139+
bytes, err := xml.MarshalIndent(suites, "", "\t")
140+
if err != nil {
141+
o := fmt.Errorf("error formatting results with formatter %s: %v",
142+
"junitxml",
143+
err,
144+
)
145+
146+
return nil, o
147+
}
148+
149+
return bytes, nil
150+
}

‎pkg/chartverifier/report/types.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@ import (
99
)
1010

1111
type (
12-
ReportFormat string
13-
OutcomeType string
12+
ReportFormat = string
13+
OutcomeType = string
1414
)
1515

1616
type ShaValue struct{}

0 commit comments

Comments
 (0)