diff --git a/process_testcase.go b/process_testcase.go index 9b510e5d..e59d61d1 100644 --- a/process_testcase.go +++ b/process_testcase.go @@ -312,7 +312,7 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase, tsIn *TestStepRe } else { tsResult.Start = time.Now() tsResult.Status = StatusRun - v.RunTestStep(ctx, e, tc, tsResult, stepNumber, rangedIndex, step) + result := v.RunTestStep(ctx, e, tc, tsResult, stepNumber, rangedIndex, step) if len(tsResult.Errors) > 0 || !tsResult.AssertionsApplied.OK { tsResult.Status = StatusFail } else { @@ -322,6 +322,9 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase, tsIn *TestStepRe tsResult.End = time.Now() tsResult.Duration = tsResult.End.Sub(tsResult.Start).Seconds() + mapResult := GetExecutorResult(result) + previousStepVars.AddAll(H(mapResult)) + tc.testSteps = append(tc.testSteps, step) } @@ -337,16 +340,17 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase, tsIn *TestStepRe if isRequired { failure := newFailure(ctx, *tc, stepNumber, rangedIndex, "", fmt.Errorf("At least one required assertion failed, skipping remaining steps")) tsResult.appendFailure(*failure) - v.printTestStepResult(tc, tsResult, tsIn, stepNumber, true) + v.printTestStepResult(tc, tsResult, tsIn, ranged, stepNumber, true) return } - v.printTestStepResult(tc, tsResult, tsIn, stepNumber, false) + v.printTestStepResult(tc, tsResult, tsIn, ranged, stepNumber, false) continue } - v.printTestStepResult(tc, tsResult, tsIn, stepNumber, false) + v.printTestStepResult(tc, tsResult, tsIn, ranged, stepNumber, false) allVars := tc.Vars.Clone() - allVars.AddAll(tsResult.ComputedVars.Clone()) + allVars.AddAll(tc.computedVars.Clone()) + tsResult.ComputedVars = tc.computedVars.Clone() assign, _, err := processVariableAssignments(ctx, tc.Name, allVars, rawStep) if err != nil { @@ -371,42 +375,43 @@ func (v *Venom) setTestStepName(ts *TestStepResult, e ExecutorRunner, step TestS } } if ranged.Enabled { + if rangedIndex == 0 { + v.Print("\n") + } name = fmt.Sprintf("%s (range=%s)", name, rangedData.Key) } ts.Name = name - if print { + if print || ranged.Enabled { v.Print(" \t\t• %s", ts.Name) } } // Print a single step result (if verbosity is enabled) -func (v *Venom) printTestStepResult(tc *TestCase, ts *TestStepResult, tsIn *TestStepResult, stepNumber int, mustAssertionFailed bool) { - if tsIn != nil { +func (v *Venom) printTestStepResult(tc *TestCase, ts *TestStepResult, tsIn *TestStepResult, ranged Range, stepNumber int, mustAssertionFailed bool) { + fromUserExecutor := tsIn != nil + if fromUserExecutor { tsIn.appendFailure(ts.Errors...) - } else if v.Verbose >= 1 { - if len(ts.Errors) > 0 { - v.Println(" %s", Red(StatusFail)) - for _, i := range ts.ComputedInfo { - v.Println(" \t\t %s %s", Cyan("[info]"), Cyan(i)) - } - for _, f := range ts.Errors { - v.Println(" \t\t %s", Yellow(f.Value)) - } - if mustAssertionFailed { - skipped := len(tc.RawTestSteps) - stepNumber - 1 - if skipped == 1 { - v.Println(" \t\t %s", Gray(fmt.Sprintf("%d other step was skipped", skipped))) - } else { - v.Println(" \t\t %s", Gray(fmt.Sprintf("%d other steps were skipped", skipped))) + } + if ranged.Enabled || v.Verbose >= 1 { + if !fromUserExecutor { //Else print step status + if len(ts.Errors) > 0 { + v.Println(" %s", Red(StatusFail)) + for _, f := range ts.Errors { + v.Println(" \t\t %s", Yellow(f.Value)) } - } - } else if ts.Status == StatusSkip { - v.Println(" %s", Gray(StatusSkip)) - } else { - v.Println(" %s", Green(StatusPass)) - for _, i := range ts.ComputedInfo { - v.Println(" \t\t %s %s", Cyan("[info]"), Cyan(i)) + if mustAssertionFailed { + skipped := len(tc.RawTestSteps) - stepNumber - 1 + if skipped == 1 { + v.Println(" \t\t %s", Gray(fmt.Sprintf("%d other step was skipped", skipped))) + } else { + v.Println(" \t\t %s", Gray(fmt.Sprintf("%d other steps were skipped", skipped))) + } + } + } else if ts.Status == StatusSkip { + v.Println(" %s", Gray(StatusSkip)) + } else { + v.Println(" %s", Green(StatusPass)) } } } diff --git a/process_teststep.go b/process_teststep.go index 76012248..22ddf73e 100644 --- a/process_teststep.go +++ b/process_teststep.go @@ -19,7 +19,7 @@ type dumpFile struct { } // RunTestStep executes a venom testcase is a venom context -func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase, tsResult *TestStepResult, stepNumber int, rangedIndex int, step TestStep) { +func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase, tsResult *TestStepResult, stepNumber int, rangedIndex int, step TestStep) interface{} { ctx = context.WithValue(ctx, ContextKey("executor"), e.Name()) var assertRes AssertionsApplied @@ -65,8 +65,7 @@ func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase, filename := path.Join(oDir, fmt.Sprintf("%s.%s.step.%d.%d.dump.json", slug.Make(StringVarFromCtx(ctx, "venom.testsuite.shortName")), slug.Make(tc.Name), stepNumber, rangedIndex)) if err := os.WriteFile(filename, []byte(output), 0644); err != nil { - Error(ctx, "Error while creating file %s: %v", filename, err) - return + return fmt.Errorf("Error while creating file %s: %v", filename, err) } tc.computedVerbose = append(tc.computedVerbose, fmt.Sprintf("writing %s", filename)) } @@ -93,6 +92,7 @@ func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase, } } Info(ctx, info) + tc.computedInfo = append(tc.computedInfo, info) tsResult.ComputedInfo = append(tsResult.ComputedInfo, info) } @@ -108,12 +108,12 @@ func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase, } tsResult.AssertionsApplied = assertRes - tsResult.ComputedVars.AddAll(H(mapResult)) + tc.computedVars.AddAll(H(mapResult)) if assertRes.OK { break } - failures, err := testConditionalStatement(ctx, tc, e.RetryIf(), tsResult.ComputedVars, "") + failures, err := testConditionalStatement(ctx, tc, e.RetryIf(), tc.computedVars, "") if err != nil { tsResult.appendError(fmt.Errorf("Error while evaluating retry condition: %v", err)) break @@ -133,6 +133,8 @@ func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase, tsResult.Systemerr += assertRes.systemerr + "\n" tsResult.Systemout += assertRes.systemout + "\n" + + return result } func (v *Venom) runTestStepExecutor(ctx context.Context, e ExecutorRunner, tc *TestCase, ts *TestStepResult, step TestStep) (interface{}, error) { diff --git a/process_testsuite.go b/process_testsuite.go index c070bd2f..d8debe9e 100644 --- a/process_testsuite.go +++ b/process_testsuite.go @@ -144,7 +144,7 @@ func (v *Venom) runTestCases(ctx context.Context, ts *TestSuite) { // Verbose mode already reported tests status, so just print them when non-verbose indent := "" - if verboseReport { + if hasRanged || verboseReport { indent = "\t " // If the testcase was entirely skipped, then the verbose mode will not have any output // Print something to inform that the testcase was indeed processed although skipped @@ -163,21 +163,19 @@ func (v *Venom) runTestCases(ctx context.Context, ts *TestSuite) { } } + for _, i := range tc.computedInfo { + v.Println("\t %s%s %s", indent, Cyan("[info]"), Cyan(i)) + } + for _, i := range tc.computedVerbose { v.PrintlnIndentedTrace(i, indent) } // Verbose mode already reported failures, so just print them when non-verbose - if !verboseReport && hasFailure { + if !hasRanged && !verboseReport && hasFailure { for _, testStepResult := range tc.TestStepResults { - if len(testStepResult.ComputedInfo) > 0 || len(testStepResult.Errors) > 0 { - v.Println(" \t\t• %s", testStepResult.Name) - for _, f := range testStepResult.ComputedInfo { - v.Println(" \t\t %s", Cyan(f)) - } - for _, f := range testStepResult.Errors { - v.Println(" \t\t %s", Yellow(f.Value)) - } + for _, f := range testStepResult.Errors { + v.Println("%s", Yellow(f.Value)) } } } diff --git a/tests/interpolate_once.yml b/tests/interpolate_once.yml index d499c30f..97e90c65 100644 --- a/tests/interpolate_once.yml +++ b/tests/interpolate_once.yml @@ -14,4 +14,4 @@ testcases: script: "echo myvar {{.randomvar}}" info: "{{.result.systemout}}" assertions: - - result.systemout ShouldContainSubstring "{{.myvar_first.result.systemout}}" \ No newline at end of file + - result.systemout ShouldContainSubstring "{{.myvar_first.result.systemout}}" diff --git a/types.go b/types.go index dd5d727e..252d5f84 100644 --- a/types.go +++ b/types.go @@ -166,6 +166,7 @@ type TestCase struct { TestSuiteVars H `json:"-" yaml:"-"` computedVars H `json:"-" yaml:"-"` + computedInfo []string `json:"-" yaml:"-"` computedVerbose []string `json:"-" yaml:"-"` IsExecutor bool `json:"-" yaml:"-"` IsEvaluated bool `json:"-" yaml:"-"` diff --git a/types_executor.go b/types_executor.go index 87cddb3c..3b8363d5 100644 --- a/types_executor.go +++ b/types_executor.go @@ -276,6 +276,10 @@ func (v *Venom) RunUserExecutor(ctx context.Context, runner ExecutorRunner, tcIn return nil, err } + // re-inject info into executorRunner + b := runner.(*executor) + b.info = append(b.info, tc.computedInfo...) + var outputResult interface{} if err := yaml.Unmarshal([]byte(outputS), &outputResult); err != nil { return nil, errors.Wrapf(err, "unable to unmarshal")