diff --git a/Makefile b/Makefile index 8e7f1fb3..9e43f1d4 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ GO_FILES?=$$(find ./ -name '*.go' | grep -v /vendor | grep -v /template/ | grep -v /build/ | grep -v swagger-client) -test: +test: goimportscheck GOFLAGS='-mod=vendor' go test ./... -v -race -failfast -p 1 require-travis-env: diff --git a/internal/run/result.go b/internal/run/result.go index 4521c830..ac3451a1 100644 --- a/internal/run/result.go +++ b/internal/run/result.go @@ -18,6 +18,8 @@ type RunResult struct { SuccessfulIterationDurations DurationPercentileMap FailedIterationCount uint64 FailedIterationDurations DurationPercentileMap + MaxFailedIterations int + MaxFailedIterationsRate int startTime time.Time TestDuration time.Duration LogFile string @@ -173,7 +175,9 @@ func (r *RunResult) String() string { func (r *RunResult) Failed() bool { r.mu.RLock() defer r.mu.RUnlock() - return r.Error() != nil || r.FailedIterationCount > 0 || (!r.IgnoreDropped && r.DroppedIterationCount > 0) + fmt.Printf("r.FailedIterationCount > uint64(r.MaxFailedIterations) %d, %d\n\n", r.FailedIterationCount, uint64(r.MaxFailedIterations)) + + return r.Error() != nil || r.FailedIterationCount > uint64(r.MaxFailedIterations) || (!r.IgnoreDropped && r.DroppedIterationCount > 0) } func (r *RunResult) Progress() string { diff --git a/internal/run/run_cmd_test.go b/internal/run/run_cmd_test.go index becca96d..1e19c955 100644 --- a/internal/run/run_cmd_test.go +++ b/internal/run/run_cmd_test.go @@ -602,3 +602,57 @@ func TestFailureCounts(t *testing.T) { the_iteration_metric_has_n_results(5, "success").and(). the_iteration_metric_has_n_results(5, "fail") } + +func TestMaxFailureCount(t *testing.T) { + given, when, then := NewRunTestStage(t) + + given. + a_rate_of("10/s").and(). + a_max_failures_of(5).and(). + a_duration_of(500 * time.Millisecond).and(). + a_test_scenario_that_fails_intermittently().and(). + a_distribution_type("none") + + when.i_execute_the_run_command() + + then. + the_iteration_metric_has_n_results(5, "success").and(). + the_iteration_metric_has_n_results(5, "fail").and(). + the_command_should_succeeded() +} + +func TestInferiorMaxFailureCount(t *testing.T) { + given, when, then := NewRunTestStage(t) + + given. + a_rate_of("10/s").and(). + a_max_failures_of(2).and(). + a_duration_of(500 * time.Millisecond).and(). + a_test_scenario_that_fails_intermittently().and(). + a_distribution_type("none") + + when.i_execute_the_run_command() + + then. + the_iteration_metric_has_n_results(5, "success").and(). + the_iteration_metric_has_n_results(5, "fail").and(). + the_command_should_fail() +} + +func TestSuperiorMaxFailureCount(t *testing.T) { + given, when, then := NewRunTestStage(t) + + given. + a_rate_of("10/s").and(). + a_max_failures_of(6).and(). + a_duration_of(500 * time.Millisecond).and(). + a_test_scenario_that_fails_intermittently().and(). + a_distribution_type("none") + + when.i_execute_the_run_command() + + then. + the_iteration_metric_has_n_results(5, "success").and(). + the_iteration_metric_has_n_results(5, "fail").and(). + the_command_should_succeeded() +} diff --git a/internal/run/run_stage_test.go b/internal/run/run_stage_test.go index b01d9096..d3c702e5 100644 --- a/internal/run/run_stage_test.go +++ b/internal/run/run_stage_test.go @@ -50,8 +50,8 @@ type RunTestStage struct { assert *assert.Assertions rate string maxIterations int32 - maxFailures int32 - maxFailuresRate int32 + maxFailures int + maxFailuresRate int triggerType TriggerType stages string frequency string @@ -100,6 +100,11 @@ func (s *RunTestStage) a_concurrency_of(concurrency int) *RunTestStage { return s } +func (s *RunTestStage) a_max_failures_of(maxFailures int) *RunTestStage { + s.maxFailures = maxFailures + return s +} + func (s *RunTestStage) a_config_file_location_of(commandsFile string) *RunTestStage { s.configFile = commandsFile return s @@ -127,6 +132,8 @@ func (s *RunTestStage) i_execute_the_run_command() *RunTestStage { MaxDuration: s.duration, Concurrency: s.concurrency, MaxIterations: s.maxIterations, + MaxFailures: s.maxFailures, + MaxFailuresRate: s.maxFailuresRate, RegisterLogHookFunc: fluentd_hook.AddFluentdLoggingHook, }, s.build_trigger()) @@ -174,6 +181,12 @@ func (s *RunTestStage) the_command_should_fail() *RunTestStage { return s } +func (s *RunTestStage) the_command_should_succeeded() *RunTestStage { + s.assert.NotNil(s.runResult) + s.assert.Equal(false, s.runResult.Failed()) + return s +} + func (s *RunTestStage) a_test_scenario_that_always_fails() *RunTestStage { s.scenario = uuid.New().String() s.f1.Add(s.scenario, func(t *f1_testing.T) (fn f1_testing.RunFn) { diff --git a/internal/run/test_runner.go b/internal/run/test_runner.go index 49ead700..43b1f48d 100644 --- a/internal/run/test_runner.go +++ b/internal/run/test_runner.go @@ -48,6 +48,8 @@ func NewRun(options options.RunOptions, t *api.Trigger) (*Run, error) { run.Options.RegisterLogHookFunc = logging.NoneRegisterLogHookFunc } run.result.IgnoreDropped = options.IgnoreDropped + run.result.MaxFailedIterations = options.MaxFailures + run.result.MaxFailedIterationsRate = options.MaxFailuresRate progressRunner, _ := raterun.New(func(rate time.Duration, t time.Time) { run.gatherProgressMetrics(rate) diff --git a/internal/trigger/file/file_parser_test.go b/internal/trigger/file/file_parser_test.go index 03677c5f..76ed665c 100644 --- a/internal/trigger/file/file_parser_test.go +++ b/internal/trigger/file/file_parser_test.go @@ -347,6 +347,37 @@ stages: expectedRates: []int{2, 2, 2, 2, 2}, expectedParameters: map[string]string{"FOO": "bar"}, }, + { + testName: "Include max failures", + fileContent: `scenario: template +limits: + max-duration: 1m + concurrency: 50 + max-iterations: 100 + max-failures: 10 + max-failures-rate: 5 + ignore-dropped: true +stages: +- duration: 5s + mode: constant + rate: 6/s + jitter: 0 + distribution: none + parameters: + FOO: bar +`, + expectedScenario: "template", + expectedMaxDuration: 1 * time.Minute, + expectedConcurrency: 50, + expectedMaxIterations: 100, + expectedMaxFailures: 10, + expectedMaxFailuresRate: 5, + expectedIgnoreDropped: true, + expectedTotalDuration: 5 * time.Second, + expectedIterationDuration: 1 * time.Second, + expectedRates: []int{6, 6, 6, 6, 6, 6}, + expectedParameters: map[string]string{"FOO": "bar"}, + }, } { t.Run(test.testName, func(t *testing.T) { now, _ := time.Parse(time.RFC3339, "2020-12-10T10:00:00+00:00") @@ -631,6 +662,8 @@ type testData struct { expectedMaxDuration time.Duration expectedIgnoreDropped bool expectedMaxIterations int32 + expectedMaxFailures int + expectedMaxFailuresRate int expectedConcurrency int expectedUsersConcurrency int expectedRates []int