From dac39283a84386a0ccba8d75ae99e1f12fd8a339 Mon Sep 17 00:00:00 2001 From: kilianpaquier Date: Wed, 31 May 2023 15:06:04 +0200 Subject: [PATCH] feat: removed result from testcase variables and removed errors warning and info from non verbose runs (#670) * feat: remove tc computedInfo, fix info not showing right variable value in teststep and show info before errors * feat: removed ranged stdout on non-verbose runs * fix: update failing tests following breaking change * chore: review README according to latest change Signed-off-by: kpaquier --- README.md | 14 +++- executors/http/README.md | 5 +- process_testcase.go | 71 +++++++++---------- process_teststep.go | 12 ++-- process_testsuite.go | 18 ++--- tests/http.yml | 11 ++- tests/interpolate_once.yml | 5 +- tests/lib/executor_http_get.yml | 9 ++- tests/lib/foobar.yml | 5 +- tests/lib/hello.yml | 9 ++- tests/lib/multilines.yml | 5 +- tests/lib/withArray.yml | 6 +- tests/lib_custom/foobar_custom.yml | 5 +- tests/lib_custom/foobar_custom_multisteps.yml | 5 +- tests/lib_custom/foobar_http.yml | 9 ++- tests/lib_custom/foobar_http_get.yml | 9 ++- tests/lib_custom/foobar_http_get_user.yml | 9 ++- tests/tmpl.yml | 5 +- tests/user_executor.yml | 13 ++-- tests/verbose_output.yml | 11 +-- types.go | 1 - types_executor.go | 4 -- 22 files changed, 150 insertions(+), 91 deletions(-) diff --git a/README.md b/README.md index 97e5b168..3cbf1389 100644 --- a/README.md +++ b/README.md @@ -377,10 +377,15 @@ steps: - script: echo "{\"hello\":\"{{.input.myarg}}\"}" assertions: - result.code ShouldEqual 0 + vars: + hello: + from: result.systemoutjson.hello + all: + from: result.systemoutjson output: display: - hello: "{{.result.systemoutjson.hello}}" - all: "{{.result.systemoutjson}}" + hello: "{{.hello}}" + all: "{{.all}}" ``` file `testsuite.yml`: @@ -430,8 +435,11 @@ steps: script: {{ .input.script | nindent 4 }} assertions: - result.code ShouldEqual 0 + vars: + all: + from: result.systemoutjson output: - all: '{{.result.systemoutjson}}' + all: '{{.all}}' ``` diff --git a/executors/http/README.md b/executors/http/README.md index 7b9e86a8..fd4bc913 100644 --- a/executors/http/README.md +++ b/executors/http/README.md @@ -56,6 +56,9 @@ testcases: file: '@./venom.gif' assertions: - result.statuscode ShouldEqual 401 + vars: + statuscode: + from: result.statuscode - name: post http enhanced assertions steps: @@ -74,7 +77,7 @@ testcases: - result.bodyjson.bodyjson0 ShouldContainKey prefix - result.bodyjson.bodyjson0 ShouldContainKey examples - result.bodyjson.bodyjson0 ShouldNotContainKey lol - - result.statuscode ShouldNotEqual {{.post-http-multipart.result.statuscode}} + - result.statuscode ShouldNotEqual {{.post-http-multipart.statuscode}} - name: get http (with options) steps: diff --git a/process_testcase.go b/process_testcase.go index 24ca58bf..70d20b39 100644 --- a/process_testcase.go +++ b/process_testcase.go @@ -313,7 +313,7 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase, tsIn *TestStepRe } else { tsResult.Start = time.Now() tsResult.Status = StatusRun - result := v.RunTestStep(ctx, e, tc, tsResult, stepNumber, rangedIndex, step) + v.RunTestStep(ctx, e, tc, tsResult, stepNumber, rangedIndex, step) if len(tsResult.Errors) > 0 || !tsResult.AssertionsApplied.OK { tsResult.Status = StatusFail } else { @@ -323,9 +323,6 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase, tsIn *TestStepRe tsResult.End = time.Now() tsResult.Duration = tsResult.End.Sub(tsResult.Start).Seconds() - mapResult := GetExecutorResult(result) - previousStepVars.AddAll(H(mapResult)) - tc.testSteps = append(tc.testSteps, step) } @@ -341,17 +338,16 @@ func (v *Venom) runTestSteps(ctx context.Context, tc *TestCase, tsIn *TestStepRe if isRequired { failure := newFailure(ctx, *tc, stepNumber, rangedIndex, "", fmt.Errorf("At least one required assertion failed, skipping remaining steps")) tsResult.appendFailure(*failure) - v.printTestStepResult(tc, tsResult, tsIn, ranged, stepNumber, true) + v.printTestStepResult(tc, tsResult, tsIn, stepNumber, true) return } - v.printTestStepResult(tc, tsResult, tsIn, ranged, stepNumber, false) + v.printTestStepResult(tc, tsResult, tsIn, stepNumber, false) continue } - v.printTestStepResult(tc, tsResult, tsIn, ranged, stepNumber, false) + v.printTestStepResult(tc, tsResult, tsIn, stepNumber, false) allVars := tc.Vars.Clone() - allVars.AddAll(tc.computedVars.Clone()) - tsResult.ComputedVars = tc.computedVars.Clone() + allVars.AddAll(tsResult.ComputedVars.Clone()) assign, _, err := processVariableAssignments(ctx, tc.Name, allVars, rawStep) if err != nil { @@ -376,48 +372,47 @@ func (v *Venom) setTestStepName(ts *TestStepResult, e ExecutorRunner, step TestS } } if ranged.Enabled { - if rangedIndex == 0 { - v.Print("\n") - } name = fmt.Sprintf("%s (range=%s)", name, rangedData.Key) } ts.Name = name - if print || ranged.Enabled { + if print { v.Print(" \t\t• %s", ts.Name) } } // Print a single step result (if verbosity is enabled) -func (v *Venom) printTestStepResult(tc *TestCase, ts *TestStepResult, tsIn *TestStepResult, ranged Range, stepNumber int, mustAssertionFailed bool) { - fromUserExecutor := tsIn != nil - if fromUserExecutor { +func (v *Venom) printTestStepResult(tc *TestCase, ts *TestStepResult, tsIn *TestStepResult, stepNumber int, mustAssertionFailed bool) { + if tsIn != nil { tsIn.appendFailure(ts.Errors...) - } - if ranged.Enabled || v.Verbose >= 1 { - if !fromUserExecutor { //Else print step status - if len(ts.Errors) > 0 { - v.Println(" %s", Red(StatusFail)) - for _, f := range ts.Errors { - v.Println(" \t\t %s", Yellow(f.Value)) - } - if mustAssertionFailed { - skipped := len(tc.RawTestSteps) - stepNumber - 1 - if skipped == 1 { - v.Println(" \t\t %s", Gray(fmt.Sprintf("%d other step was skipped", skipped))) - } else { - v.Println(" \t\t %s", Gray(fmt.Sprintf("%d other steps were skipped", skipped))) - } - } - } else if ts.Status == StatusSkip { - v.Println(" %s", Gray(StatusSkip)) - } else { - if ts.Retries == 0 { - v.Println(" %s", Green(StatusPass)) + } else if v.Verbose >= 1 { + if len(ts.Errors) > 0 { + v.Println(" %s", Red(StatusFail)) + for _, i := range ts.ComputedInfo { + v.Println(" \t\t %s %s", Cyan("[info]"), Cyan(i)) + } + for _, f := range ts.Errors { + v.Println(" \t\t %s", Yellow(f.Value)) + } + if mustAssertionFailed { + skipped := len(tc.RawTestSteps) - stepNumber - 1 + if skipped == 1 { + v.Println(" \t\t %s", Gray(fmt.Sprintf("%d other step was skipped", skipped))) } else { - v.Println(" %s (after %d attempts)", Green(StatusPass), ts.Retries) + v.Println(" \t\t %s", Gray(fmt.Sprintf("%d other steps were skipped", skipped))) } } + } else if ts.Status == StatusSkip { + v.Println(" %s", Gray(StatusSkip)) + } else { + if ts.Retries == 0 { + v.Println(" %s", Green(StatusPass)) + } else { + v.Println(" %s (after %d attempts)", Green(StatusPass), ts.Retries) + } + for _, i := range ts.ComputedInfo { + v.Println(" \t\t %s %s", Cyan("[info]"), Cyan(i)) + } } } } diff --git a/process_teststep.go b/process_teststep.go index 8878087f..fbef5d36 100644 --- a/process_teststep.go +++ b/process_teststep.go @@ -19,7 +19,7 @@ type dumpFile struct { } // RunTestStep executes a venom testcase is a venom context -func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase, tsResult *TestStepResult, stepNumber int, rangedIndex int, step TestStep) interface{} { +func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase, tsResult *TestStepResult, stepNumber int, rangedIndex int, step TestStep) { ctx = context.WithValue(ctx, ContextKey("executor"), e.Name()) var assertRes AssertionsApplied @@ -64,7 +64,8 @@ func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase, filename := path.Join(oDir, fmt.Sprintf("%s.%s.step.%d.%d.dump.json", slug.Make(StringVarFromCtx(ctx, "venom.testsuite.shortName")), slug.Make(tc.Name), stepNumber, rangedIndex)) if err := os.WriteFile(filename, []byte(output), 0644); err != nil { - return fmt.Errorf("Error while creating file %s: %v", filename, err) + Error(ctx, "Error while creating file %s: %v", filename, err) + return } tc.computedVerbose = append(tc.computedVerbose, fmt.Sprintf("writing %s", filename)) } @@ -91,7 +92,6 @@ func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase, } } Info(ctx, info) - tc.computedInfo = append(tc.computedInfo, info) tsResult.ComputedInfo = append(tsResult.ComputedInfo, info) } @@ -107,12 +107,12 @@ func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase, } tsResult.AssertionsApplied = assertRes - tc.computedVars.AddAll(H(mapResult)) + tsResult.ComputedVars.AddAll(H(mapResult)) if assertRes.OK { break } - failures, err := testConditionalStatement(ctx, tc, e.RetryIf(), tc.computedVars, "") + failures, err := testConditionalStatement(ctx, tc, e.RetryIf(), tsResult.ComputedVars, "") if err != nil { tsResult.appendError(fmt.Errorf("Error while evaluating retry condition: %v", err)) break @@ -134,8 +134,6 @@ func (v *Venom) RunTestStep(ctx context.Context, e ExecutorRunner, tc *TestCase, tsResult.Systemerr += assertRes.systemerr + "\n" tsResult.Systemout += assertRes.systemout + "\n" - - return result } func (v *Venom) runTestStepExecutor(ctx context.Context, e ExecutorRunner, tc *TestCase, ts *TestStepResult, step TestStep) (interface{}, error) { diff --git a/process_testsuite.go b/process_testsuite.go index d8debe9e..c070bd2f 100644 --- a/process_testsuite.go +++ b/process_testsuite.go @@ -144,7 +144,7 @@ func (v *Venom) runTestCases(ctx context.Context, ts *TestSuite) { // Verbose mode already reported tests status, so just print them when non-verbose indent := "" - if hasRanged || verboseReport { + if verboseReport { indent = "\t " // If the testcase was entirely skipped, then the verbose mode will not have any output // Print something to inform that the testcase was indeed processed although skipped @@ -163,19 +163,21 @@ func (v *Venom) runTestCases(ctx context.Context, ts *TestSuite) { } } - for _, i := range tc.computedInfo { - v.Println("\t %s%s %s", indent, Cyan("[info]"), Cyan(i)) - } - for _, i := range tc.computedVerbose { v.PrintlnIndentedTrace(i, indent) } // Verbose mode already reported failures, so just print them when non-verbose - if !hasRanged && !verboseReport && hasFailure { + if !verboseReport && hasFailure { for _, testStepResult := range tc.TestStepResults { - for _, f := range testStepResult.Errors { - v.Println("%s", Yellow(f.Value)) + if len(testStepResult.ComputedInfo) > 0 || len(testStepResult.Errors) > 0 { + v.Println(" \t\t• %s", testStepResult.Name) + for _, f := range testStepResult.ComputedInfo { + v.Println(" \t\t %s", Cyan(f)) + } + for _, f := range testStepResult.Errors { + v.Println(" \t\t %s", Yellow(f.Value)) + } } } } diff --git a/tests/http.yml b/tests/http.yml index cf95a892..859cd2c2 100644 --- a/tests/http.yml +++ b/tests/http.yml @@ -49,6 +49,11 @@ testcases: - or: - result.statuscode ShouldEqual 401 - result.statuscode ShouldEqual 415 + vars: + statuscode: + from: result.statuscode + body: + from: result.bodyjson - name: post http enhanced assertions steps: @@ -67,16 +72,16 @@ testcases: - result.bodyjson.bodyjson0 ShouldContainKey prefix - result.bodyjson.bodyjson0 ShouldContainKey examples - result.bodyjson.bodyjson0 ShouldNotContainKey lol - - result.statuscode ShouldNotEqual {{.post-http-multipart.result.statuscode}} + - result.statuscode ShouldNotEqual {{.post-http-multipart.statuscode}} - name: get http with templated variables steps: - type: http method: POST - url: https://eu.api.ovh.com/1.0/{{.post-http-multipart.result.bodyjson.bodyjson0.fieldName}} + url: https://eu.api.ovh.com/1.0/{{.post-http-multipart.body.body0.fieldName}} assertions: - result.statuscode ShouldEqual 404 - - result.statuscode ShouldNotEqual {{.post-http-multipart.result.statuscode}} + - result.statuscode ShouldNotEqual {{.post-http-multipart.statuscode}} - name: get http (with skip body) steps: diff --git a/tests/interpolate_once.yml b/tests/interpolate_once.yml index 97e90c65..22089a34 100644 --- a/tests/interpolate_once.yml +++ b/tests/interpolate_once.yml @@ -7,6 +7,9 @@ testcases: - type: exec script: "echo myvar {{.randomvar}}" info: "{{.result.systemout}}" + vars: + systemout: + from: result.systemout - name: myvar_second steps: @@ -14,4 +17,4 @@ testcases: script: "echo myvar {{.randomvar}}" info: "{{.result.systemout}}" assertions: - - result.systemout ShouldContainSubstring "{{.myvar_first.result.systemout}}" + - result.systemout ShouldContainSubstring "{{.myvar_first.systemout}}" diff --git a/tests/lib/executor_http_get.yml b/tests/lib/executor_http_get.yml index b78b3952..64b6fa13 100644 --- a/tests/lib/executor_http_get.yml +++ b/tests/lib/executor_http_get.yml @@ -5,6 +5,11 @@ steps: - type: http method: GET url: "{{.input.url}}" + vars: + statuscode: + from: result.statuscode + body: + from: result.body output: - statuscode: "{{.result.statuscode}}" - body: "{{.result.body}}" \ No newline at end of file + statuscode: "{{.statuscode}}" + body: "{{.body}}" \ No newline at end of file diff --git a/tests/lib/foobar.yml b/tests/lib/foobar.yml index 17cdd11b..960218fa 100644 --- a/tests/lib/foobar.yml +++ b/tests/lib/foobar.yml @@ -5,5 +5,8 @@ steps: - script: echo "foo-{{.input.bar}}" assertions: - result.code ShouldEqual 0 + vars: + systemout: + from: result.systemout output: - foobar: "{{.result.systemout}}" + foobar: "{{.systemout}}" diff --git a/tests/lib/hello.yml b/tests/lib/hello.yml index bdbdfde8..a8cf39c4 100644 --- a/tests/lib/hello.yml +++ b/tests/lib/hello.yml @@ -6,7 +6,12 @@ steps: assertions: - result.code ShouldEqual 0 info: "{{.result.systemoutjson.hello}}" + vars: + hello: + from: result.systemoutjson.hello + systemout: + from: result.systemout output: display: - hello: "{{.result.systemoutjson.hello}}" - therawout: '{{.result.systemout}}' \ No newline at end of file + hello: "{{.hello}}" + therawout: '{{.systemout}}' \ No newline at end of file diff --git a/tests/lib/multilines.yml b/tests/lib/multilines.yml index 184458de..20dab8ee 100644 --- a/tests/lib/multilines.yml +++ b/tests/lib/multilines.yml @@ -6,5 +6,8 @@ steps: script: {{ .input.script | nindent 4 }} assertions: - result.code ShouldEqual 0 + vars: + systemoutjson: + from: result.systemout output: - all: '{{.result.systemoutjson}}' + all: '{{.systemoutjson}}' diff --git a/tests/lib/withArray.yml b/tests/lib/withArray.yml index a7a31e2e..01eb8683 100644 --- a/tests/lib/withArray.yml +++ b/tests/lib/withArray.yml @@ -5,6 +5,8 @@ input: steps: - script: echo '{{.input.thearray}}' info: {{.input.thearray}} - + vars: + systemout: + from: result.systemout output: - foobar: "{{.result.systemout}}" + foobar: "{{.systemout}}" diff --git a/tests/lib_custom/foobar_custom.yml b/tests/lib_custom/foobar_custom.yml index bc1f9723..8d2207a5 100644 --- a/tests/lib_custom/foobar_custom.yml +++ b/tests/lib_custom/foobar_custom.yml @@ -5,5 +5,8 @@ steps: - script: echo "custom-{{.input.bar}}" assertions: - result.code ShouldEqual 0 + vars: + systemout: + from: result.systemout output: - foobar: "{{.result.systemout}}" + foobar: "{{.systemout}}" diff --git a/tests/lib_custom/foobar_custom_multisteps.yml b/tests/lib_custom/foobar_custom_multisteps.yml index 9afe0080..1bd6373e 100644 --- a/tests/lib_custom/foobar_custom_multisteps.yml +++ b/tests/lib_custom/foobar_custom_multisteps.yml @@ -6,5 +6,8 @@ steps: content: from: result.systemout - script: echo "{{.content}} world" + vars: + systemout: + from: result.systemout output: - foobar: "{{.result.systemout}}" \ No newline at end of file + foobar: "{{.systemout}}" \ No newline at end of file diff --git a/tests/lib_custom/foobar_http.yml b/tests/lib_custom/foobar_http.yml index 0ffde184..0db0d664 100644 --- a/tests/lib_custom/foobar_http.yml +++ b/tests/lib_custom/foobar_http.yml @@ -11,6 +11,11 @@ steps: body: "{{.input.body}}" assertions: - result.statuscode ShouldEqual 200 + vars: + statuscode: + from: result.statuscode + bodyjson: + from: result.bodyjson output: - statuscode: "{{.result.statuscode}}" - body: "{{.result.bodyjson}}" \ No newline at end of file + statuscode: "{{.statuscode}}" + body: "{{.bodyjson}}" \ No newline at end of file diff --git a/tests/lib_custom/foobar_http_get.yml b/tests/lib_custom/foobar_http_get.yml index ffd43ed1..e788a567 100644 --- a/tests/lib_custom/foobar_http_get.yml +++ b/tests/lib_custom/foobar_http_get.yml @@ -9,6 +9,11 @@ steps: method: "GET" assertions: - result.statuscode ShouldEqual 200 + vars: + statuscode: + from: result.statuscode + bodyjson: + from: result.bodyjson output: - statuscode: "{{.result.statuscode}}" - body: "{{.result.bodyjson}}" \ No newline at end of file + statuscode: "{{.statuscode}}" + body: "{{.bodyjson}}" \ No newline at end of file diff --git a/tests/lib_custom/foobar_http_get_user.yml b/tests/lib_custom/foobar_http_get_user.yml index 36243b37..0584d5a2 100644 --- a/tests/lib_custom/foobar_http_get_user.yml +++ b/tests/lib_custom/foobar_http_get_user.yml @@ -7,6 +7,11 @@ steps: res: users assertions: - result.statuscode ShouldEqual 200 + vars: + statuscode: + from: result.statuscode + bodyjson: + from: result.bodyjson output: - statuscode: "{{.result.statuscode}}" - body: "{{.result.bodyjson}}" \ No newline at end of file + statuscode: "{{.statuscode}}" + body: "{{.bodyjson}}" \ No newline at end of file diff --git a/tests/tmpl.yml b/tests/tmpl.yml index 3c334317..ee871ae8 100644 --- a/tests/tmpl.yml +++ b/tests/tmpl.yml @@ -11,11 +11,14 @@ testcases: assertions: - result.code ShouldEqual 0 - result.systemout ShouldEqual http://api/foo + vars: + systemout: + from: result.systemout - name: testB steps: - type: exec - script: echo 'XXX{{.testA.result.systemout}}YYY' + script: echo 'XXX{{.testA.systemout}}YYY' assertions: - result.code ShouldEqual 0 - result.systemout ShouldEqual XXXhttp://api/fooYYY diff --git a/tests/user_executor.yml b/tests/user_executor.yml index b4fdb11c..39fc5575 100644 --- a/tests/user_executor.yml +++ b/tests/user_executor.yml @@ -15,19 +15,24 @@ testcases: myarg: World assertions: - result.display.hello ShouldEqual World + vars: + therawout: + from: result.therawout + hello: + from: result.display.hello - name: testAResultDisplay steps: - - script: echo '{{.testA.result.therawout}}' - info: value is "{{.testA.result.therawout}}" + - script: echo '{{.testA.therawout}}' + info: value is "{{.testA.therawout}}" assertions: - result.systemoutjson.hello ShouldEqual World - name: testB steps: - - script: echo "{{.testA.result.display.hello}}" + - script: echo "{{.testA.hello}}" assertions: - - result.systemout ShouldEqual {{.testA.result.display.hello}} + - result.systemout ShouldEqual World - name: testfoobar steps: diff --git a/tests/verbose_output.yml b/tests/verbose_output.yml index e8595601..2ba26337 100644 --- a/tests/verbose_output.yml +++ b/tests/verbose_output.yml @@ -8,9 +8,12 @@ testcases: script: NO_COLOR=1 ./venom run failing/verbose_output.yml {{.value.opt}} info: NO_COLOR=1 ./venom run failing/verbose_output.yml {{.value.opt}} range: - verbose: + verbose-level2: opt: "-vv" op: Should + verbose-level1: + opt: "-v" + op: Should default: opt: "" op: ShouldNot @@ -25,9 +28,9 @@ testcases: - result.systemout {{.value.op}}ContainSubstring 'step1 PASS' - result.systemout {{.value.op}}ContainSubstring 'step2 FAIL' # ranged steps - - result.systemout ShouldContainSubstring 'exec (range=0) PASS' - - result.systemout ShouldContainSubstring 'exec (range=1) FAIL' - - result.systemout ShouldContainSubstring 'exec (range=2) PASS' + - result.systemout {{.value.op}}ContainSubstring 'exec (range=0) PASS' + - result.systemout {{.value.op}}ContainSubstring 'exec (range=1) FAIL' + - result.systemout {{.value.op}}ContainSubstring 'exec (range=2) PASS' # must assertions - result.systemout {{.value.op}}ContainSubstring 'must1 FAIL' - result.systemout ShouldContainSubstring 'At least one required assertion failed, skipping remaining steps' diff --git a/types.go b/types.go index 4c84cae6..ca7b26a9 100644 --- a/types.go +++ b/types.go @@ -168,7 +168,6 @@ type TestCase struct { TestSuiteVars H `json:"-" yaml:"-"` computedVars H `json:"-" yaml:"-"` - computedInfo []string `json:"-" yaml:"-"` computedVerbose []string `json:"-" yaml:"-"` IsExecutor bool `json:"-" yaml:"-"` IsEvaluated bool `json:"-" yaml:"-"` diff --git a/types_executor.go b/types_executor.go index 3b8363d5..87cddb3c 100644 --- a/types_executor.go +++ b/types_executor.go @@ -276,10 +276,6 @@ func (v *Venom) RunUserExecutor(ctx context.Context, runner ExecutorRunner, tcIn return nil, err } - // re-inject info into executorRunner - b := runner.(*executor) - b.info = append(b.info, tc.computedInfo...) - var outputResult interface{} if err := yaml.Unmarshal([]byte(outputS), &outputResult); err != nil { return nil, errors.Wrapf(err, "unable to unmarshal")