feat: removed result from testcase variables and removed errors warni…

…ng and info from non verbose runs (ovh#670)

* feat: remove tc computedInfo, fix info not showing right variable value in teststep and show info before errors
* feat: removed ranged stdout on non-verbose runs
* fix: update failing tests following breaking change
* chore: review README according to latest change

Signed-off-by: kpaquier <[email protected]>
Signed-off-by: Ivan Velasco <[email protected]>

README.md

@@ -377,10 +377,15 @@ steps:
 - script: echo "{\"hello\":\"{{.input.myarg}}\"}"
   assertions:
   - result.code ShouldEqual 0
+  vars:
+    hello:
+      from: result.systemoutjson.hello
+    all:
+      from: result.systemoutjson
 output:
   display:
-    hello: "{{.result.systemoutjson.hello}}"
-  all: "{{.result.systemoutjson}}"
+    hello: "{{.hello}}"
+  all: "{{.all}}"
 ```
 
 file `testsuite.yml`:
@@ -430,8 +435,11 @@ steps:
   script: {{ .input.script | nindent 4 }}
   assertions:
   - result.code ShouldEqual 0
+  vars:
+    all:
+      from: result.systemoutjson
 output:
-  all: '{{.result.systemoutjson}}'
+  all: '{{.all}}'
 ```
 
 

executors/http/README.md

@@ -56,6 +56,9 @@ testcases:
       file: '@./venom.gif'
     assertions:
     - result.statuscode ShouldEqual 401
+    vars:
+      statuscode:
+        from: result.statuscode
 
 - name: post http enhanced assertions
   steps:
@@ -74,7 +77,7 @@ testcases:
       - result.bodyjson.bodyjson0 ShouldContainKey prefix
       - result.bodyjson.bodyjson0 ShouldContainKey examples
       - result.bodyjson.bodyjson0 ShouldNotContainKey lol
-      - result.statuscode ShouldNotEqual {{.post-http-multipart.result.statuscode}}
+      - result.statuscode ShouldNotEqual {{.post-http-multipart.statuscode}}
 
 - name: get http (with options)
   steps:

process_testcase.go

@@ -313,7 +313,7 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase, tsIn *TestStepRe
 			} else {
 				tsResult.Start = time.Now()
 				tsResult.Status = StatusRun
-				result := v.RunTestStep(ctx, e, tc, tsResult, stepNumber, rangedIndex, step)
+				v.RunTestStep(ctx, e, tc, tsResult, stepNumber, rangedIndex, step)
 				if len(tsResult.Errors) > 0 || !tsResult.AssertionsApplied.OK {
 					tsResult.Status = StatusFail
 				} else {
@@ -323,9 +323,6 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase, tsIn *TestStepRe
 				tsResult.End = time.Now()
 				tsResult.Duration = tsResult.End.Sub(tsResult.Start).Seconds()
 
-				mapResult := GetExecutorResult(result)
-				previousStepVars.AddAll(H(mapResult))
-
 				tc.testSteps = append(tc.testSteps, step)
 			}
 
@@ -341,17 +338,16 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase, tsIn *TestStepRe
 				if isRequired {
 					failure := newFailure(ctx, *tc, stepNumber, rangedIndex, "", fmt.Errorf("At least one required assertion failed, skipping remaining steps"))
 					tsResult.appendFailure(*failure)
-					v.printTestStepResult(tc, tsResult, tsIn, ranged, stepNumber, true)
+					v.printTestStepResult(tc, tsResult, tsIn, stepNumber, true)
 					return
 				}
-				v.printTestStepResult(tc, tsResult, tsIn, ranged, stepNumber, false)
+				v.printTestStepResult(tc, tsResult, tsIn, stepNumber, false)
 				continue
 			}
-			v.printTestStepResult(tc, tsResult, tsIn, ranged, stepNumber, false)
+			v.printTestStepResult(tc, tsResult, tsIn, stepNumber, false)
 
 			allVars := tc.Vars.Clone()
-			allVars.AddAll(tc.computedVars.Clone())
-			tsResult.ComputedVars = tc.computedVars.Clone()
+			allVars.AddAll(tsResult.ComputedVars.Clone())
 
 			assign, _, err := processVariableAssignments(ctx, tc.Name, allVars, rawStep)
 			if err != nil {
@@ -376,48 +372,47 @@ func (v *Venom) setTestStepName(ts *TestStepResult, e ExecutorRunner, step TestS
 		}
 	}
 	if ranged.Enabled {
-		if rangedIndex == 0 {
-			v.Print("\n")
-		}
 		name = fmt.Sprintf("%s (range=%s)", name, rangedData.Key)
 	}
 	ts.Name = name
 
-	if print || ranged.Enabled {
+	if print {
 		v.Print(" \t\t• %s", ts.Name)
 	}
 }
 
 // Print a single step result (if verbosity is enabled)
-func (v *Venom) printTestStepResult(tc *TestCase, ts *TestStepResult, tsIn *TestStepResult, ranged Range, stepNumber int, mustAssertionFailed bool) {
-	fromUserExecutor := tsIn != nil
-	if fromUserExecutor {
+func (v *Venom) printTestStepResult(tc *TestCase, ts *TestStepResult, tsIn *TestStepResult, stepNumber int, mustAssertionFailed bool) {
+	if tsIn != nil {
 		tsIn.appendFailure(ts.Errors...)
-	}
-	if ranged.Enabled || v.Verbose >= 1 {
-		if !fromUserExecutor { //Else print step status
-			if len(ts.Errors) > 0 {
-				v.Println(" %s", Red(StatusFail))
-				for _, f := range ts.Errors {
-					v.Println(" \t\t  %s", Yellow(f.Value))
-				}
-				if mustAssertionFailed {
-					skipped := len(tc.RawTestSteps) - stepNumber - 1
-					if skipped == 1 {
-						v.Println(" \t\t  %s", Gray(fmt.Sprintf("%d other step was skipped", skipped)))
-					} else {
-						v.Println(" \t\t  %s", Gray(fmt.Sprintf("%d other steps were skipped", skipped)))
-					}
-				}
-			} else if ts.Status == StatusSkip {
-				v.Println(" %s", Gray(StatusSkip))
-			} else {
-				if ts.Retries == 0 {
-					v.Println(" %s", Green(StatusPass))
+	} else if v.Verbose >= 1 {
+		if len(ts.Errors) > 0 {
+			v.Println(" %s", Red(StatusFail))
+			for _, i := range ts.ComputedInfo {
+				v.Println(" \t\t  %s %s", Cyan("[info]"), Cyan(i))
+			}
+			for _, f := range ts.Errors {
+				v.Println(" \t\t  %s", Yellow(f.Value))
+			}
+			if mustAssertionFailed {
+				skipped := len(tc.RawTestSteps) - stepNumber - 1
+				if skipped == 1 {
+					v.Println(" \t\t  %s", Gray(fmt.Sprintf("%d other step was skipped", skipped)))
 				} else {
-					v.Println(" %s (after %d attempts)", Green(StatusPass), ts.Retries)
+					v.Println(" \t\t  %s", Gray(fmt.Sprintf("%d other steps were skipped", skipped)))
 				}
 			}
+		} else if ts.Status == StatusSkip {
+			v.Println(" %s", Gray(StatusSkip))
+		} else {
+			if ts.Retries == 0 {
+				v.Println(" %s", Green(StatusPass))
+			} else {
+				v.Println(" %s (after %d attempts)", Green(StatusPass), ts.Retries)
+			}
+			for _, i := range ts.ComputedInfo {
+				v.Println(" \t\t  %s %s", Cyan("[info]"), Cyan(i))
+			}
 		}
 	}
 }

process_teststep.go

@@ -19,7 +19,7 @@ type dumpFile struct {
 }
 
 // RunTestStep executes a venom testcase is a venom context
-func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase, tsResult *TestStepResult, stepNumber int, rangedIndex int, step TestStep) interface{} {
+func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase, tsResult *TestStepResult, stepNumber int, rangedIndex int, step TestStep) {
 	ctx = context.WithValue(ctx, ContextKey("executor"), e.Name())
 
 	var assertRes AssertionsApplied
@@ -64,7 +64,8 @@ func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase,
 			filename := path.Join(oDir, fmt.Sprintf("%s.%s.step.%d.%d.dump.json", slug.Make(StringVarFromCtx(ctx, "venom.testsuite.shortName")), slug.Make(tc.Name), stepNumber, rangedIndex))
 
 			if err := os.WriteFile(filename, []byte(output), 0644); err != nil {
-				return fmt.Errorf("Error while creating file %s: %v", filename, err)
+				Error(ctx, "Error while creating file %s: %v", filename, err)
+				return
 			}
 			tc.computedVerbose = append(tc.computedVerbose, fmt.Sprintf("writing %s", filename))
 		}
@@ -91,7 +92,6 @@ func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase,
 				}
 			}
 			Info(ctx, info)
-			tc.computedInfo = append(tc.computedInfo, info)
 			tsResult.ComputedInfo = append(tsResult.ComputedInfo, info)
 		}
 
@@ -107,12 +107,12 @@ func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase,
 		}
 
 		tsResult.AssertionsApplied = assertRes
-		tc.computedVars.AddAll(H(mapResult))
+		tsResult.ComputedVars.AddAll(H(mapResult))
 
 		if assertRes.OK {
 			break
 		}
-		failures, err := testConditionalStatement(ctx, tc, e.RetryIf(), tc.computedVars, "")
+		failures, err := testConditionalStatement(ctx, tc, e.RetryIf(), tsResult.ComputedVars, "")
 		if err != nil {
 			tsResult.appendError(fmt.Errorf("Error while evaluating retry condition: %v", err))
 			break
@@ -134,8 +134,6 @@ func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase,
 
 	tsResult.Systemerr += assertRes.systemerr + "\n"
 	tsResult.Systemout += assertRes.systemout + "\n"
-
-	return result
 }
 
 func (v *Venom) runTestStepExecutor(ctx context.Context, e ExecutorRunner, tc *TestCase, ts *TestStepResult, step TestStep) (interface{}, error) {

process_testsuite.go

@@ -144,7 +144,7 @@ func (v *Venom) runTestCases(ctx context.Context, ts *TestSuite) {
 
 		// Verbose mode already reported tests status, so just print them when non-verbose
 		indent := ""
-		if hasRanged || verboseReport {
+		if verboseReport {
 			indent = "\t  "
 			// If the testcase was entirely skipped, then the verbose mode will not have any output
 			// Print something to inform that the testcase was indeed processed although skipped
@@ -163,19 +163,21 @@ func (v *Venom) runTestCases(ctx context.Context, ts *TestSuite) {
 			}
 		}
 
-		for _, i := range tc.computedInfo {
-			v.Println("\t  %s%s %s", indent, Cyan("[info]"), Cyan(i))
-		}
-
 		for _, i := range tc.computedVerbose {
 			v.PrintlnIndentedTrace(i, indent)
 		}
 
 		// Verbose mode already reported failures, so just print them when non-verbose
-		if !hasRanged && !verboseReport && hasFailure {
+		if !verboseReport && hasFailure {
 			for _, testStepResult := range tc.TestStepResults {
-				for _, f := range testStepResult.Errors {
-					v.Println("%s", Yellow(f.Value))
+				if len(testStepResult.ComputedInfo) > 0 || len(testStepResult.Errors) > 0 {
+					v.Println(" \t\t• %s", testStepResult.Name)
+					for _, f := range testStepResult.ComputedInfo {
+						v.Println(" \t\t  %s", Cyan(f))
+					}
+					for _, f := range testStepResult.Errors {
+						v.Println(" \t\t  %s", Yellow(f.Value))
+					}
 				}
 			}
 		}

tests/http.yml

@@ -49,6 +49,11 @@ testcases:
     - or:
       - result.statuscode ShouldEqual 401
       - result.statuscode ShouldEqual 415
+    vars:
+      statuscode:
+        from: result.statuscode
+      body:
+        from: result.bodyjson
 
 - name: post http enhanced assertions
   steps:
@@ -67,16 +72,16 @@ testcases:
       - result.bodyjson.bodyjson0 ShouldContainKey prefix
       - result.bodyjson.bodyjson0 ShouldContainKey examples
       - result.bodyjson.bodyjson0 ShouldNotContainKey lol
-      - result.statuscode ShouldNotEqual {{.post-http-multipart.result.statuscode}}
+      - result.statuscode ShouldNotEqual {{.post-http-multipart.statuscode}}
 
 - name: get http with templated variables
   steps:
   - type: http
     method: POST
-    url: https://eu.api.ovh.com/1.0/{{.post-http-multipart.result.bodyjson.bodyjson0.fieldName}}
+    url: https://eu.api.ovh.com/1.0/{{.post-http-multipart.body.body0.fieldName}}
     assertions:
       - result.statuscode ShouldEqual 404
-      - result.statuscode ShouldNotEqual {{.post-http-multipart.result.statuscode}}
+      - result.statuscode ShouldNotEqual {{.post-http-multipart.statuscode}}
 
 - name: get http (with skip body)
   steps:

tests/interpolate_once.yml

@@ -7,11 +7,14 @@ testcases:
   - type: exec
     script: "echo myvar {{.randomvar}}"
     info: "{{.result.systemout}}"
+    vars:
+      systemout:
+        from: result.systemout
 
 - name: myvar_second
   steps:
   - type: exec
     script: "echo myvar {{.randomvar}}"
     info: "{{.result.systemout}}"
     assertions:
-    - result.systemout ShouldContainSubstring "{{.myvar_first.result.systemout}}"
+    - result.systemout ShouldContainSubstring "{{.myvar_first.systemout}}"

tests/lib/executor_http_get.yml

@@ -5,6 +5,11 @@ steps:
 - type: http
   method: GET
   url: "{{.input.url}}"
+  vars:
+    statuscode:
+      from: result.statuscode
+    body:
+      from: result.body
 output:
-  statuscode: "{{.result.statuscode}}"
-  body: "{{.result.body}}"
+  statuscode: "{{.statuscode}}"
+  body: "{{.body}}"

tests/lib/foobar.yml

@@ -5,5 +5,8 @@ steps:
 - script: echo "foo-{{.input.bar}}"
   assertions:
   - result.code ShouldEqual 0
+  vars:
+    systemout:
+      from: result.systemout
 output:
-  foobar: "{{.result.systemout}}"
+  foobar: "{{.systemout}}"

tests/lib/hello.yml

@@ -6,7 +6,12 @@ steps:
   assertions:
   - result.code ShouldEqual 0
   info: "{{.result.systemoutjson.hello}}"
+  vars:
+    hello:
+      from: result.systemoutjson.hello
+    systemout:
+      from: result.systemout
 output:
   display:
-    hello: "{{.result.systemoutjson.hello}}"
-  therawout: '{{.result.systemout}}'
+    hello: "{{.hello}}"
+  therawout: '{{.systemout}}'

tests/lib/multilines.yml

@@ -6,5 +6,8 @@ steps:
   script: {{ .input.script | nindent 4 }}
   assertions:
   - result.code ShouldEqual 0
+  vars:
+    systemoutjson:
+      from: result.systemout
 output:
-  all: '{{.result.systemoutjson}}'
+  all: '{{.systemoutjson}}'

tests/lib/withArray.yml

@@ -5,6 +5,8 @@ input:
 steps:
 - script: echo '{{.input.thearray}}'
   info: {{.input.thearray}}
-
+  vars:
+    systemout:
+      from: result.systemout
 output:
-  foobar: "{{.result.systemout}}"
+  foobar: "{{.systemout}}"

tests/lib_custom/foobar_custom.yml

@@ -5,5 +5,8 @@ steps:
 - script: echo "custom-{{.input.bar}}"
   assertions:
   - result.code ShouldEqual 0
+  vars:
+    systemout:
+      from: result.systemout
 output:
-  foobar: "{{.result.systemout}}"
+  foobar: "{{.systemout}}"

tests/lib_custom/foobar_custom_multisteps.yml

@@ -6,5 +6,8 @@ steps:
     content:
       from: result.systemout
 - script: echo "{{.content}} world"
+  vars:
+    systemout:
+      from: result.systemout
 output:
-  foobar: "{{.result.systemout}}"
+  foobar: "{{.systemout}}"