feat: skip testcase (#328)

Signed-off-by: francois  samin <[email protected]>

README.md

@@ -376,7 +376,8 @@ Available format: jUnit (xml), json, yaml, tap reports
 Most assertion keywords documentation can be found on https://pkg.go.dev/github.com/ovh/venom/assertions.
 
 
-# Debug your testsuites
+# Advanced usage
+## Debug your testsuites
 
 There is two ways to debug a testsuite:
  - use `-v` flag on venom binary.
@@ -417,6 +418,42 @@ $ venom run test.yml
 	  [info] the value of result.systemoutjson is map[foo:bar] (exec.yml:34)
 ```
 
+## Skip testcase
+
+It is possible to skip `testcase` according to some `assertions`. For instance, the following exampl will skip the last testcase.
+
+```yaml
+name: "Skip testsuite"
+vars:
+  foo: bar
+
+testcases:
+- name: init
+  steps:
+  - type: exec
+    script: echo {{.foo}}
+    assertions:
+    - result.code ShouldEqual 0
+    - result.systemout ShouldContainSubstring bar
+
+- name: do-not-skip-this
+  skip: 
+  - foo ShouldNotBeEmpty
+  steps:
+  - type: exec
+    script: exit 0
+
+- name: skip-this
+  skip: 
+    - foo ShouldBeEmpty
+  steps:
+  - type: exec
+    script: command_not_found
+    assertions:
+    - result.code ShouldEqual 0
+
+```
+
 # Use venom in CI
 
 Venom can be use on dev environement or your CI server.

assertion.go

@@ -2,6 +2,7 @@ package venom
 
 import (
 	"bufio"
+	"context"
 	"errors"
 	"fmt"
 	"os"
@@ -13,6 +14,7 @@ import (
 
 	"github.com/mitchellh/mapstructure"
 	"github.com/ovh/venom/assertions"
+	"github.com/ovh/venom/executors"
 )
 
 type assertionsApplied struct {
@@ -47,7 +49,7 @@ func applyAssertions(r interface{}, tc TestCase, stepNumber int, step TestStep,
 
 	isOK := true
 	for _, assertion := range sa.Assertions {
-		errs, fails := check(tc, stepNumber, assertion, executorResult)
+		errs, fails := check(tc, stepNumber, assertion, r)
 		if errs != nil {
 			errors = append(errors, *errs)
 			isOK = false
@@ -69,30 +71,50 @@ func applyAssertions(r interface{}, tc TestCase, stepNumber int, step TestStep,
 	return assertionsApplied{isOK, errors, failures, systemout, systemerr}
 }
 
-func check(tc TestCase, stepNumber int, assertion string, r interface{}) (*Failure, *Failure) {
-	executorResult := GetExecutorResult(r)
+type assertion struct {
+	Actual interface{}
+	Func   assertions.AssertFunc
+	Args   []interface{}
+}
 
-	assert := splitAssertion(assertion)
+func parseAssertions(ctx context.Context, s string, input interface{}) (*assertion, error) {
+	dump, err := executors.Dump(input)
+	if err != nil {
+		return nil, errors.New("assertion syntax error")
+	}
+	assert := splitAssertion(s)
 	if len(assert) < 2 {
-		return nil, newFailure(tc, stepNumber, assertion, errors.New("syntax error"))
+		return nil, errors.New("assertion syntax error")
 	}
+	actual := dump[assert[0]]
 
-	actual := executorResult[assert[0]]
 	f, ok := assertions.Get(assert[1])
 	if !ok {
-		return nil, newFailure(tc, stepNumber, assertion, errors.New("assertion not supported"))
+		return nil, errors.New("assertion not supported")
 	}
 
 	args := make([]interface{}, len(assert[2:]))
 	for i, v := range assert[2:] {
 		var err error
 		args[i], err = stringToType(v, actual)
 		if err != nil {
-			return nil, newFailure(tc, stepNumber, assertion, fmt.Errorf("mismatched type between '%v' and '%v': %v", assert[0], v, err))
+			return nil, fmt.Errorf("mismatched type between '%v' and '%v': %v", assert[0], v, err)
 		}
 	}
+	return &assertion{
+		Actual: actual,
+		Func:   f,
+		Args:   args,
+	}, nil
+}
+
+func check(tc TestCase, stepNumber int, assertion string, r interface{}) (*Failure, *Failure) {
+	assert, err := parseAssertions(context.Background(), assertion, r)
+	if err != nil {
+		return nil, newFailure(tc, stepNumber, assertion, err)
+	}
 
-	if err := f(actual, args...); err != nil {
+	if err := assert.Func(assert.Actual, assert.Args...); err != nil {
 		return nil, newFailure(tc, stepNumber, assertion, err)
 	}
 	return nil, nil

process_testcase.go

@@ -122,6 +122,25 @@ func (v *Venom) runTestCase(ctx context.Context, ts *TestSuite, tc *TestCase) {
 
 	defer Info(ctx, "Ending testcase")
 
+	for _, skipAssertion := range tc.Skip {
+		Debug(ctx, "evaluating %s", skipAssertion)
+		assert, err := parseAssertions(ctx, skipAssertion, tc.Vars)
+		if err != nil {
+			Error(ctx, "unable to parse skip assertion: %v", err)
+			tc.AppendError(err)
+			return
+		}
+		if err := assert.Func(assert.Actual, assert.Args...); err != nil {
+			s := fmt.Sprintf("skipping testcase %q: %v", tc.originalName, err)
+			tc.Skipped = append(tc.Skipped, Skipped{Value: s})
+			Warn(ctx, s)
+		}
+	}
+
+	if len(tc.Skipped) > 0 {
+		return
+	}
+
 	var knowExecutors = map[string]struct{}{}
 
 	for stepNumber, rawStep := range tc.RawTestSteps {

process_testsuite.go

@@ -65,6 +65,7 @@ func (v *Venom) runTestCases(ctx context.Context, ts *TestSuite) {
 	var red = color.New(color.FgRed).SprintFunc()
 	var green = color.New(color.FgGreen).SprintFunc()
 	var cyan = color.New(color.FgCyan).SprintFunc()
+	var gray = color.New(color.Attribute(90)).SprintFunc()
 
 	v.Println(" • %s (%s)", ts.Name, ts.Package)
 
@@ -73,7 +74,8 @@ func (v *Venom) runTestCases(ctx context.Context, ts *TestSuite) {
 		v.Print(" \t• %s", tc.Name)
 		tc.Classname = ts.Filename
 		var hasFailure bool
-		if len(tc.Skipped) == 0 {
+		var hasSkipped = len(tc.Skipped) > 0
+		if !hasSkipped {
 			v.runTestCase(ctx, ts, tc)
 		}
 
@@ -87,11 +89,16 @@ func (v *Venom) runTestCases(ctx context.Context, ts *TestSuite) {
 		}
 		if len(tc.Skipped) > 0 {
 			ts.Skipped += len(tc.Skipped)
+			hasSkipped = true
+		}
+
+		if hasSkipped {
+			v.Println(" %s", gray("SKIPPED"))
+			continue
 		}
 
 		if hasFailure {
 			v.Println(" %s", red("FAILURE"))
-
 		} else {
 			v.Println(" %s", green("SUCCESS"))
 		}

tests/skip.yml

@@ -0,0 +1,28 @@
+name: "Skip testsuite"
+vars:
+  foo: bar
+
+testcases:
+- name: init
+  steps:
+  - type: exec
+    script: echo {{.foo}}
+    assertions:
+    - result.code ShouldEqual 0
+    - result.systemout ShouldContainSubstring bar
+
+- name: do-not-skip-this
+  skip: 
+  - foo ShouldNotBeEmpty
+  steps:
+  - type: exec
+    script: exit 0
+
+- name: skip-this
+  skip: 
+    - foo ShouldBeEmpty
+  steps:
+  - type: exec
+    script: command_not_found
+    assertions:
+    - result.code ShouldEqual 0

types.go

@@ -233,11 +233,11 @@ type TestCase struct {
 	Time            string            `xml:"time,attr,omitempty" json:"time" yaml:"time,omitempty"`
 	RawTestSteps    []json.RawMessage `xml:"-" hcl:"step" json:"steps" yaml:"steps"`
 	testSteps       []TestStep
-	Context         map[string]interface{} `xml:"-" json:"-" yaml:"context,omitempty"`
-	Vars            H                      `xml:"-" json:"-" yaml:"vars"`
+	Vars            H `xml:"-" json:"-" yaml:"vars"`
 	computedVars    H
 	computedInfo    []string
 	computedVerbose []string
+	Skip            []string `xml:"-" json:"skip" yaml:"skip"`
 }
 
 // TestStep represents a testStep