refactor: --html-report instead of format=html

README.md

@@ -145,18 +145,19 @@ Examples:
   Run all testsuites containing in files ending with *.yml or *.yaml: venom run
   Run a single testsuite: venom run mytestfile.yml
   Run a single testsuite and export the result in JSON format in test/ folder: venom run mytestfile.yml --format=json --output-dir=test
-  Run a single testsuite and export the result in XML and HTML formats in test/ folder: venom run mytestfile.yml --format=xml,html --output-dir=test
+  Run a single testsuite and export the result in XML and HTML formats in test/ folder: venom run mytestfile.yml --format=xml --output-dir=test --html-report
   Run a single testsuite and specify a variable: venom run mytestfile.yml --var="foo=bar"
   Run a single testsuite and load all variables from a file: venom run mytestfile.yml --var-from-file variables.yaml
   Run all testsuites containing in files ending with *.yml or *.yaml with verbosity: VENOM_VERBOSE=2 venom run
-
+  
   Notice that variables initialized with -var-from-file argument can be overrided with -var argument
-
+  
   More info: https://github.com/ovh/venom
 
 Flags:
-      --format string           --format:html, json, tap, xml, yaml (default "xml")
+      --format string           --format:json, tap, xml, yaml (default "xml")
   -h, --help                    help for run
+      --html-report             Generate HTML Report
       --lib-dir string          Lib Directory: can contain user executors. example:/etc/venom/lib:$HOME/venom.d/lib
       --output-dir string       Output Directory: create tests results file inside this directory
       --stop-on-failure         Stop running Test Suite on first Test Case failure
@@ -229,8 +230,9 @@ List of available flags for `venom run` command:
 
 ```
 Flags:
-      --format string           --format:html, json, tap, xml, yaml (default "xml")
+      --format string           --format:json, tap, xml, yaml (default "xml")
   -h, --help                    help for run
+      --html-report             Generate HTML Report
       --lib-dir string          Lib Directory: can contain user executors. example:/etc/venom/lib:$HOME/venom.d/lib
       --output-dir string       Output Directory: create tests results file inside this directory
       --stop-on-failure         Stop running Test Suite on first Test Case failure
@@ -720,8 +722,8 @@ You can specify the output directory with the `--output-dir` flag and the format
 ```bash
 $ venom run --format=xml --output-dir="."
 
-# xml and html export
-$ venom run --format=xml,html --output-dir="."
+# html export
+$ venom run --output-dir="." --html-report
 ```
 
 Reports exported in XML can be visualized with a xUnit/jUnit Viewer, directly in your favorite CI/CD stack for example in order to see results run after run.

cmd/venom/run/cmd.go

@@ -30,6 +30,7 @@ var (
 	varFiles      []string
 	outputDir     string
 	libDir        string
+	htmlReport    bool
 	stopOnFailure bool
 	verbose       int = 0 // Set the default value for verboseFlag
 
@@ -39,12 +40,14 @@ var (
 	outputDirFlag     *string
 	libDirFlag        *string
 	stopOnFailureFlag *bool
+	htmlReportFlag    *bool
 	verboseFlag       *int
 )
 
 func init() {
-	formatFlag = Cmd.Flags().String("format", "xml", "--format:html, json, tap, xml, yaml")
+	formatFlag = Cmd.Flags().String("format", "xml", "--format:json, tap, xml, yaml")
 	stopOnFailureFlag = Cmd.Flags().Bool("stop-on-failure", false, "Stop running Test Suite on first Test Case failure")
+	htmlReportFlag = Cmd.Flags().Bool("html-report", false, "Generate HTML Report")
 	verboseFlag = Cmd.Flags().CountP("verbose", "v", "verbose. -vv to very verbose and -vvv to very verbose with CPU Profiling")
 	varFilesFlag = Cmd.Flags().StringSlice("var-from-file", []string{""}, "--var-from-file filename.yaml --var-from-file filename2.yaml: yaml, must contains a dictionnary")
 	variablesFlag = Cmd.Flags().StringArray("var", nil, "--var cds='cds -f config.json' --var cds2='cds -f config.json'")
@@ -81,6 +84,10 @@ func initFromCommandArguments(f *pflag.Flag) {
 		if stopOnFailureFlag != nil {
 			stopOnFailure = *stopOnFailureFlag
 		}
+	case "html-report":
+		if htmlReportFlag != nil {
+			htmlReport = *htmlReportFlag
+		}
 	case "output-dir":
 		if outputDirFlag != nil {
 			outputDir = *outputDirFlag
@@ -148,6 +155,7 @@ type ConfigFileData struct {
 	LibDir         *string   `json:"lib_dir,omitempty" yaml:"lib_dir,omitempty"`
 	OutputDir      *string   `json:"output_dir,omitempty" yaml:"output_dir,omitempty"`
 	StopOnFailure  *bool     `json:"stop_on_failure,omitempty" yaml:"stop_on_failure,omitempty"`
+	HtmlReport     *bool     `json:"html_report,omitempty" yaml:"html_report,omitempty"`
 	Variables      *[]string `json:"variables,omitempty" yaml:"variables,omitempty"`
 	VariablesFiles *[]string `json:"variables_files,omitempty" yaml:"variables_files,omitempty"`
 	Verbosity      *int      `json:"verbosity,omitempty" yaml:"verbosity,omitempty"`
@@ -177,6 +185,9 @@ func initFromReaderConfigFile(reader io.Reader) error {
 	if configFileData.StopOnFailure != nil {
 		stopOnFailure = *configFileData.StopOnFailure
 	}
+	if configFileData.HtmlReport != nil {
+		htmlReport = *configFileData.HtmlReport
+	}
 	if configFileData.Variables != nil {
 		for _, varFromFile := range *configFileData.Variables {
 			variables = mergeVariables(varFromFile, variables)
@@ -240,6 +251,13 @@ func initFromEnv(environ []string) ([]string, error) {
 			return nil, fmt.Errorf("invalid value for VENOM_STOP_ON_FAILURE")
 		}
 	}
+	if os.Getenv("VENOM_HTML_REPORT") != "" {
+		var err error
+		htmlReport, err = strconv.ParseBool(os.Getenv("VENOM_HTML_REPORT"))
+		if err != nil {
+			return nil, fmt.Errorf("invalid value for VENOM_HTML_REPORT")
+		}
+	}
 	if os.Getenv("VENOM_LIB_DIR") != "" {
 		libDir = os.Getenv("VENOM_LIB_DIR")
 	}
@@ -277,6 +295,7 @@ func displayArg(ctx context.Context) {
 	venom.Debug(ctx, "option libDir=%v", libDir)
 	venom.Debug(ctx, "option outputDir=%v", outputDir)
 	venom.Debug(ctx, "option stopOnFailure=%v", stopOnFailure)
+	venom.Debug(ctx, "option htmlReport=%v", htmlReport)
 	venom.Debug(ctx, "option variables=%v", strings.Join(variables, " "))
 	venom.Debug(ctx, "option varFiles=%v", strings.Join(varFiles, " "))
 	venom.Debug(ctx, "option verbose=%v", verbose)
@@ -289,7 +308,7 @@ var Cmd = &cobra.Command{
 	Example: `  Run all testsuites containing in files ending with *.yml or *.yaml: venom run
   Run a single testsuite: venom run mytestfile.yml
   Run a single testsuite and export the result in JSON format in test/ folder: venom run mytestfile.yml --format=json --output-dir=test
-  Run a single testsuite and export the result in XML and HTML formats in test/ folder: venom run mytestfile.yml --format=xml,html --output-dir=test
+  Run a single testsuite and export the result in XML and HTML formats in test/ folder: venom run mytestfile.yml --format=xml --output-dir=test --html-report
   Run a single testsuite and specify a variable: venom run mytestfile.yml --var="foo=bar"
   Run a single testsuite and load all variables from a file: venom run mytestfile.yml --var-from-file variables.yaml
   Run all testsuites containing in files ending with *.yml or *.yaml with verbosity: VENOM_VERBOSE=2 venom run
@@ -317,6 +336,7 @@ var Cmd = &cobra.Command{
 		v.LibDir = libDir
 		v.OutputFormat = format
 		v.StopOnFailure = stopOnFailure
+		v.HtmlReport = htmlReport
 		v.Verbose = verbose
 
 		if err := v.InitLogger(); err != nil {

process_files.go

@@ -67,18 +67,18 @@ type partialTestSuite struct {
 }
 
 func (v *Venom) readFiles(ctx context.Context, filesPath []string) (err error) {
-	for _, f := range filesPath {
-		log.Info("Reading ", f)
-		btes, err := os.ReadFile(f)
+	for _, filePath := range filesPath {
+		log.Info("Reading ", filePath)
+		btes, err := os.ReadFile(filePath)
 		if err != nil {
-			return errors.Wrapf(err, "unable to read file %q", f)
+			return errors.Wrapf(err, "unable to read file %q", filePath)
 		}
 
 		varCloned := v.variables.Clone()
 
 		fromPartial, err := getVarFromPartialYML(ctx, btes)
 		if err != nil {
-			return errors.Wrapf(err, "unable to get vars from file %q", f)
+			return errors.Wrapf(err, "unable to get vars from file %q", filePath)
 		}
 
 		var varsFromPartial map[string]string
@@ -120,7 +120,7 @@ func (v *Venom) readFiles(ctx context.Context, filesPath []string) (err error) {
 		var testSuiteInput TestSuiteInput
 		if err := yaml.Unmarshal([]byte(content), &testSuiteInput); err != nil {
 			Error(context.Background(), "file content: %s", content)
-			return errors.Wrapf(err, "error while unmarshal file %q", f)
+			return errors.Wrapf(err, "error while unmarshal file %q", filePath)
 		}
 
 		ts := TestSuite{
@@ -135,17 +135,17 @@ func (v *Venom) readFiles(ctx context.Context, filesPath []string) (err error) {
 		}
 
 		// Default workdir is testsuite directory
-		ts.WorkDir, err = filepath.Abs(filepath.Dir(f))
+		ts.WorkDir, err = filepath.Abs(filepath.Dir(filePath))
 		if err != nil {
 			return errors.Wrapf(err, "Unable to get testsuite's working directory")
 		}
 
 		// ../foo/a.yml
-		ts.Filepath = f
+		ts.Filepath = filePath
 		// a
-		ts.ShortName = strings.TrimSuffix(filepath.Base(f), filepath.Ext(f))
+		ts.ShortName = strings.TrimSuffix(filepath.Base(filePath), filepath.Ext(filePath))
 		// a.yml
-		ts.Filename = filepath.Base(f)
+		ts.Filename = filepath.Base(filePath)
 		ts.Vars = varCloned
 
 		ts.Vars.Add("venom.testsuite.workdir", ts.WorkDir)

tests/Makefile

@@ -77,7 +77,7 @@ build-test-binary-docker:
 
 run-test: generate-venom-pki
 	VENOM_VAR_MY_ENVAR=foo  ./venom_wrapper.sh run \
-	-vv --format=html,xml --output-dir=. --lib-dir=./lib_custom --var='array_from_var=["biz","buz"]' --var-from-file ./kafka/testVariables.yml --var-from-file=$(PKI_VAR_FILE) --var-from-file ./vars/vars.yml ./*.yml ./assertions/*.yml && \
+	-vv --format=xml --output-dir=. --html-report --lib-dir=./lib_custom --var='array_from_var=["biz","buz"]' --var-from-file ./kafka/testVariables.yml --var-from-file=$(PKI_VAR_FILE) --var-from-file ./vars/vars.yml ./*.yml ./assertions/*.yml && \
 	(cd ./vars_override && VENOM_VAR_foo=from-env VENOM_VAR_FOO2=from-env2 VENOM_VAR_FOO3=from-env3 ../venom_wrapper.sh run -vv --format=xml --output-dir=. --var foo='from-cmd-arg' --var='array_from_var=["biz","buz"]' ./mytc.yml);
 
 .PHONY: wait-for-kafka

types.go

@@ -275,7 +275,7 @@ type FailureXML struct {
 }
 
 func newFailure(ctx context.Context, tc TestCase, stepNumber int, rangedIndex int, assertion string, err error) *Failure {
-	filename := StringVarFromCtx(ctx, "venom.executor.filename")
+	filename := StringVarFromCtx(ctx, "venom.testsuite.filename")
 	var lineNumber = findLineNumber(filename, tc.originalName, stepNumber, assertion, -1)
 	var value string
 	if assertion != "" {

venom.go

@@ -70,6 +70,7 @@ type Venom struct {
 	OutputFormat  string
 	OutputDir     string
 	StopOnFailure bool
+	HtmlReport    bool
 	Verbose       int
 }
 

venom_output.go

@@ -51,42 +51,42 @@ func (v *Venom) OutputResult() error {
 
 		var data []byte
 		var err error
-		for _, f := range strings.Split(v.OutputFormat, ",") {
-			format := strings.TrimSpace(f)
-			switch format {
-			case "json":
-				data, err = json.MarshalIndent(testsResult, "", "  ")
-				if err != nil {
-					log.Fatalf("Error: cannot format output json (%s)", err)
-				}
-			case "tap":
-				data, err = outputTapFormat(*testsResult)
-				if err != nil {
-					log.Fatalf("Error: cannot format output tap (%s)", err)
-				}
-			case "yml", "yaml":
-				data, err = yaml.Marshal(testsResult)
-				if err != nil {
-					log.Fatalf("Error: cannot format output yaml (%s)", err)
-				}
-			case "xml":
-				data, err = outputXMLFormat(*testsResult)
-				if err != nil {
-					log.Fatalf("Error: cannot format output xml (%s)", err)
-				}
-			case "html":
-				continue
-			}
 
-			filename := path.Join(v.OutputDir, "test_results."+v.Tests.TestSuites[i].Filename+"."+format)
-			if err := os.WriteFile(filename, data, 0600); err != nil {
-				return fmt.Errorf("Error while creating file %s: %v", filename, err)
+		switch v.OutputFormat {
+		case "json":
+			data, err = json.MarshalIndent(testsResult, "", "  ")
+			if err != nil {
+				log.Fatalf("Error: cannot format output json (%s)", err)
+			}
+		case "tap":
+			data, err = outputTapFormat(*testsResult)
+			if err != nil {
+				log.Fatalf("Error: cannot format output tap (%s)", err)
 			}
-			v.PrintFunc("Writing file %s\n", filename)
+		case "yml", "yaml":
+			data, err = yaml.Marshal(testsResult)
+			if err != nil {
+				log.Fatalf("Error: cannot format output yaml (%s)", err)
+			}
+		case "xml":
+			data, err = outputXMLFormat(*testsResult)
+			if err != nil {
+				log.Fatalf("Error: cannot format output xml (%s)", err)
+			}
+		case "html":
+			log.Fatalf("Error: you have to use the --html-report flag")
+		}
+
+		fname := strings.TrimSuffix(v.Tests.TestSuites[i].Filepath, filepath.Ext(v.Tests.TestSuites[i].Filepath))
+		fname = strings.ReplaceAll(fname, "/", "_")
+		filename := path.Join(v.OutputDir, "test_results_"+fname+"."+v.OutputFormat)
+		if err := os.WriteFile(filename, data, 0600); err != nil {
+			return fmt.Errorf("Error while creating file %s: %v", filename, err)
 		}
+		v.PrintFunc("Writing file %s\n", filename)
 	}
 
-	if strings.Contains(v.OutputFormat, "html") {
+	if v.HtmlReport {
 		testsResult := &Tests{
 			TestSuites:       v.Tests.TestSuites,
 			Status:           v.Tests.Status,