feat: replace enableProfiling and log-level flags (#301)

* feat: replace enableProfiling and log-level flag

only one flag now:
venom run -v foo.yml: debug mode
venom run -vv foo.yml: debug mode with CPU Profiling

README.md

@@ -11,23 +11,24 @@ Download latest binary release from https://github.com/ovh/venom/releases
 
 ```bash
 $ venom run -h
-Run Tests
+
+$ venom run *.yml
+
+Notice that variables initialized with -var-from-file argument can be overrided with -var argument.
 
 Usage:
   venom run [flags]
 
 Flags:
       --format string           --format:yaml, json, xml, tap (default "xml")
   -h, --help                    help for run
-      --log string              Log Level : debug, info, warn or disable (default "warn")
       --no-check-variables      Don't check variables before run
       --output-dir string       Output Directory: create tests results file inside this directory
-      --parallel int            --parallel=2 : launches 2 Test Suites in parallel (default 1)
-      --profiling               Enable Mem / CPU Profile with pprof
       --stop-on-failure         Stop running Test Suite on first Test Case failure
       --strict                  Exit with an error code if one test fails
       --var strings             --var cds='cds -f config.json' --var cds2='cds -f config.json'
-      --var-from-file strings   --var-from-file filename.yaml --var-from-file filename2.yaml: yaml, must contains a dictionnary'
+      --var-from-file strings   --var-from-file filename.yaml --var-from-file filename2.yaml: yaml, must contains a dictionnary
+  -v, --verbose count           verbose. -vv to very verbose and -vvv to very verbose with CPU Profiling
 ```
 
 ## Docker image

cli/venom/run/cmd.go

@@ -34,29 +34,27 @@ import (
 )
 
 var (
-	path            []string
-	variables       []string
-	format          string
-	varFiles        []string
-	logLevel        string
-	outputDir       string
-	strict          bool
-	noCheckVars     bool
-	stopOnFailure   bool
-	enableProfiling bool
-	v               *venom.Venom
+	path          []string
+	variables     []string
+	format        string
+	varFiles      []string
+	outputDir     string
+	strict        bool
+	noCheckVars   bool
+	stopOnFailure bool
+	verbose       *int
+	v             *venom.Venom
 )
 
 func init() {
 	Cmd.Flags().StringSliceVarP(&variables, "var", "", []string{""}, "--var cds='cds -f config.json' --var cds2='cds -f config.json'")
 	Cmd.Flags().StringSliceVarP(&varFiles, "var-from-file", "", []string{""}, "--var-from-file filename.yaml --var-from-file filename2.yaml: yaml, must contains a dictionnary")
 	Cmd.Flags().StringVarP(&format, "format", "", "xml", "--format:yaml, json, xml, tap")
 	Cmd.Flags().BoolVarP(&strict, "strict", "", false, "Exit with an error code if one test fails")
-	Cmd.Flags().BoolVarP(&stopOnFailure, "stop-on-failure", "", false, "Stop running Test Suite on first Test Case failure")
 	Cmd.Flags().BoolVarP(&noCheckVars, "no-check-variables", "", false, "Don't check variables before run")
-	Cmd.PersistentFlags().StringVarP(&logLevel, "log", "", "warn", "Log Level : debug, info, warn or disable")
+	Cmd.Flags().BoolVarP(&stopOnFailure, "stop-on-failure", "", false, "Stop running Test Suite on first Test Case failure")
 	Cmd.PersistentFlags().StringVarP(&outputDir, "output-dir", "", "", "Output Directory: create tests results file inside this directory")
-	Cmd.PersistentFlags().BoolVarP(&enableProfiling, "profiling", "", false, "Enable Mem / CPU Profile with pprof")
+	verbose = Cmd.Flags().CountP("verbose", "v", "verbose. -vv to very verbose and -vvv to very verbose with CPU Profiling")
 }
 
 // Cmd run
@@ -66,12 +64,6 @@ var Cmd = &cobra.Command{
 	Long: `
 $ venom run *.yml
 
-# to have more information about what's wrong on a test,
-# you can use the output-dir. *.dump files will be created
-# in this directory, with a lot of useful debug lines:
-
-$ venom run *.yml --output-dir=results
-
 Notice that variables initialized with -var-from-file argument can be overrided with -var argument.`,
 	PreRun: func(cmd *cobra.Command, args []string) {
 		if len(args) == 0 {
@@ -97,13 +89,12 @@ Notice that variables initialized with -var-from-file argument can be overrided
 		v.RegisterExecutor(sql.Name, sql.New())
 	},
 	RunE: func(cmd *cobra.Command, args []string) error {
-		v.EnableProfiling = enableProfiling
-		v.LogLevel = logLevel
 		v.OutputDir = outputDir
 		v.OutputFormat = format
 		v.StopOnFailure = stopOnFailure
+		v.Verbose = *verbose
 
-		if v.EnableProfiling {
+		if v.Verbose == 3 {
 			fCPU, err := os.Create(filepath.Join(v.OutputDir, "pprof_cpu_profile.prof"))
 			if err != nil {
 				log.Errorf("error while create profile file %v", err)

executors/imap/imap.go

@@ -146,7 +146,7 @@ func (e *Executor) getMail(ctx context.Context) (*Mail, error) {
 
 		if found {
 			if e.DeleteOnSuccess {
-				venom.Debug(ctx, "Delete message %s", m.UID)
+				venom.Debug(ctx, "Delete message %v", m.UID)
 				if err := m.delete(c); err != nil {
 					return nil, err
 				}

process.go

@@ -15,20 +15,10 @@ import (
 
 func (v *Venom) init() error {
 	v.testsuites = []TestSuite{}
-	switch v.LogLevel {
-	case "disable":
-		v.LogOutput = ioutil.Discard
+	if v.Verbose == 0 {
 		logrus.SetLevel(logrus.WarnLevel)
-		logrus.SetOutput(v.LogOutput)
-		return nil
-	case "debug":
+	} else {
 		logrus.SetLevel(logrus.DebugLevel)
-	case "info":
-		logrus.SetLevel(logrus.InfoLevel)
-	case "error":
-		logrus.SetLevel(logrus.WarnLevel)
-	default:
-		logrus.SetLevel(logrus.WarnLevel)
 	}
 
 	if v.OutputDir != "" {
@@ -37,15 +27,20 @@ func (v *Venom) init() error {
 		}
 	}
 
-	var err error
-	var logFile = filepath.Join(v.OutputDir, "venom.log")
-	_ = os.RemoveAll(logFile)
-	v.LogOutput, err = os.OpenFile(logFile, os.O_CREATE|os.O_RDWR, os.FileMode(0644))
-	if err != nil {
-		return fmt.Errorf("unable to write log file: %v", err)
+	if v.Verbose > 0 {
+		var err error
+		var logFile = filepath.Join(v.OutputDir, "venom.log")
+		_ = os.RemoveAll(logFile)
+		v.LogOutput, err = os.OpenFile(logFile, os.O_CREATE|os.O_RDWR, os.FileMode(0644))
+		if err != nil {
+			return fmt.Errorf("unable to write log file: %v", err)
+		}
+
+		logrus.SetOutput(v.LogOutput)
+	} else {
+		logrus.SetOutput(ioutil.Discard)
 	}
 
-	logrus.SetOutput(v.LogOutput)
 	logrus.SetFormatter(&nested.Formatter{
 		HideKeys:    true,
 		FieldsOrder: []string{"testsuite", "testcase", "step", "executor"},

process_teststep.go

@@ -2,14 +2,24 @@ package venom
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
+	"path"
 	"time"
 
+	"github.com/gosimple/slug"
 	"github.com/ovh/cds/sdk/interpolate"
 
 	"github.com/ovh/venom/executors"
 )
 
+type DumpFile struct {
+	Variables interface{} `json:"variables"`
+	TestStep  TestStep    `json:"step"`
+	Result    interface{} `json:"result"`
+}
+
 //RunTestStep executes a venom testcase is a venom context
 func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, ts *TestSuite, tc *TestCase, stepNumber int, step TestStep) interface{} {
 	ctx = context.WithValue(ctx, ContextKey("executor"), e.Name())
@@ -40,6 +50,26 @@ func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, ts *TestSuite
 		mapResult := GetExecutorResult(result)
 		mapResultString, _ := executors.DumpString(result)
 
+		if v.Verbose == 2 {
+			dumpFile := DumpFile{
+				Result:    result,
+				TestStep:  step,
+				Variables: ts.Vars,
+			}
+			output, _ := json.MarshalIndent(dumpFile, "", " ")
+
+			oDir := v.OutputDir
+			if oDir == "" {
+				oDir = "."
+			}
+			filename := path.Join(oDir, fmt.Sprintf("%s.%s.step.%d.dump.json", slug.Make(ts.ShortName), slug.Make(tc.Name), stepNumber))
+
+			if err := ioutil.WriteFile(filename, []byte(output), 0644); err != nil {
+				return fmt.Errorf("Error while creating file %s: %v", filename, err)
+			}
+			Info(ctx, "File %s is written", filename)
+		}
+
 		for _, i := range e.Info() {
 			info, err := interpolate.Do(i, mapResultString)
 			if err != nil {

process_testsuite.go

@@ -16,7 +16,7 @@ import (
 type ContextKey string
 
 func (v *Venom) runTestSuite(ctx context.Context, ts *TestSuite) {
-	if v.EnableProfiling {
+	if v.Verbose == 3 {
 		var filename, filenameCPU, filenameMem string
 		if v.OutputDir != "" {
 			filename = v.OutputDir + "/"

venom.go

@@ -17,20 +17,17 @@ var (
 
 func New() *Venom {
 	v := &Venom{
-		LogLevel:        "info",
 		LogOutput:       os.Stdout,
 		PrintFunc:       fmt.Printf,
 		executors:       map[string]Executor{},
 		variables:       map[string]interface{}{},
-		EnableProfiling: false,
 		IgnoreVariables: []string{},
 		OutputFormat:    "xml",
 	}
 	return v
 }
 
 type Venom struct {
-	LogLevel  string
 	LogOutput io.Writer
 
 	PrintFunc func(format string, a ...interface{}) (n int, err error)
@@ -40,10 +37,10 @@ type Venom struct {
 	variables       H
 	IgnoreVariables []string
 
-	EnableProfiling bool
-	OutputFormat    string
-	OutputDir       string
-	StopOnFailure   bool
+	OutputFormat  string
+	OutputDir     string
+	StopOnFailure bool
+	Verbose       int
 }
 
 func (v *Venom) Print(format string, a ...interface{}) {

venom_output.go

@@ -6,16 +6,19 @@ import (
 	"encoding/xml"
 	"fmt"
 	"io/ioutil"
+	"path"
 	"time"
 
-	"github.com/gosimple/slug"
 	tap "github.com/mndrix/tap-go"
 	log "github.com/sirupsen/logrus"
 	yaml "gopkg.in/yaml.v2"
 )
 
 // OutputResult output result to sdtout, files...
 func (v *Venom) OutputResult(tests Tests, elapsed time.Duration) error {
+	if v.OutputDir == "" {
+		return nil
+	}
 	var data []byte
 	var err error
 	switch v.OutputFormat {
@@ -42,30 +45,12 @@ func (v *Venom) OutputResult(tests Tests, elapsed time.Duration) error {
 		data = append([]byte(`<?xml version="1.0" encoding="utf-8"?>`), dataxml...)
 	}
 
-	if v.OutputDir != "" {
-		v.PrintFunc("\n") // new line to display files written
-		filename := v.OutputDir + "/test_results." + v.OutputFormat
-		if err := ioutil.WriteFile(filename, data, 0644); err != nil {
-			return fmt.Errorf("Error while creating file %s: %v", filename, err)
-		}
-		v.PrintFunc("Writing file %s\n", filename)
-
-		for _, ts := range tests.TestSuites {
-			for _, tc := range ts.TestCases {
-				for _, f := range tc.Failures {
-					filename := v.OutputDir + "/" + slug.Make(ts.ShortName) + "." + slug.Make(tc.Name) + ".dump"
-					output := f.Value + "\n ------ Variables:\n"
-					for k, v := range ts.Vars {
-						output += fmt.Sprintf("%s:%v\n", k, v)
-					}
-					if err := ioutil.WriteFile(filename, []byte(output), 0644); err != nil {
-						return fmt.Errorf("Error while creating file %s: %v", filename, err)
-					}
-					v.PrintFunc("File %s is written\n", filename)
-				}
-			}
-		}
+	filename := path.Join(v.OutputDir, "test_results."+v.OutputFormat)
+	if err := ioutil.WriteFile(filename, data, 0644); err != nil {
+		return fmt.Errorf("Error while creating file %s: %v", filename, err)
 	}
+	v.PrintFunc("Writing file %s\n", filename)
+
 	return nil
 }