Add tests

internal/run/result.go

@@ -18,6 +18,8 @@ type RunResult struct {
 	SuccessfulIterationDurations DurationPercentileMap
 	FailedIterationCount         uint64
 	FailedIterationDurations     DurationPercentileMap
+	MaxFailedIterations          int
+	MaxFailedIterationsRate      int
 	startTime                    time.Time
 	TestDuration                 time.Duration
 	LogFile                      string
@@ -173,7 +175,9 @@ func (r *RunResult) String() string {
 func (r *RunResult) Failed() bool {
 	r.mu.RLock()
 	defer r.mu.RUnlock()
-	return r.Error() != nil || r.FailedIterationCount > 0 || (!r.IgnoreDropped && r.DroppedIterationCount > 0)
+	fmt.Printf("r.FailedIterationCount > uint64(r.MaxFailedIterations) %d, %d\n\n", r.FailedIterationCount, uint64(r.MaxFailedIterations))
+
+	return r.Error() != nil || r.FailedIterationCount > uint64(r.MaxFailedIterations) || (!r.IgnoreDropped && r.DroppedIterationCount > 0)
 }
 
 func (r *RunResult) Progress() string {

internal/run/run_cmd_test.go

@@ -602,3 +602,57 @@ func TestFailureCounts(t *testing.T) {
 		the_iteration_metric_has_n_results(5, "success").and().
 		the_iteration_metric_has_n_results(5, "fail")
 }
+
+func TestMaxFailureCount(t *testing.T) {
+	given, when, then := NewRunTestStage(t)
+
+	given.
+		a_rate_of("10/s").and().
+		a_max_failures_of(5).and().
+		a_duration_of(500 * time.Millisecond).and().
+		a_test_scenario_that_fails_intermittently().and().
+		a_distribution_type("none")
+
+	when.i_execute_the_run_command()
+
+	then.
+		the_iteration_metric_has_n_results(5, "success").and().
+		the_iteration_metric_has_n_results(5, "fail").and().
+		the_command_should_succeeded()
+}
+
+func TestInferiorMaxFailureCount(t *testing.T) {
+	given, when, then := NewRunTestStage(t)
+
+	given.
+		a_rate_of("10/s").and().
+		a_max_failures_of(2).and().
+		a_duration_of(500 * time.Millisecond).and().
+		a_test_scenario_that_fails_intermittently().and().
+		a_distribution_type("none")
+
+	when.i_execute_the_run_command()
+
+	then.
+		the_iteration_metric_has_n_results(5, "success").and().
+		the_iteration_metric_has_n_results(5, "fail").and().
+		the_command_should_fail()
+}
+
+func TestSuperiorMaxFailureCount(t *testing.T) {
+	given, when, then := NewRunTestStage(t)
+
+	given.
+		a_rate_of("10/s").and().
+		a_max_failures_of(6).and().
+		a_duration_of(500 * time.Millisecond).and().
+		a_test_scenario_that_fails_intermittently().and().
+		a_distribution_type("none")
+
+	when.i_execute_the_run_command()
+
+	then.
+		the_iteration_metric_has_n_results(5, "success").and().
+		the_iteration_metric_has_n_results(5, "fail").and().
+		the_command_should_succeeded()
+}

internal/run/run_stage_test.go

@@ -50,8 +50,8 @@ type RunTestStage struct {
 	assert                 *assert.Assertions
 	rate                   string
 	maxIterations          int32
-	maxFailures            int32
-	maxFailuresRate        int32
+	maxFailures            int
+	maxFailuresRate        int
 	triggerType            TriggerType
 	stages                 string
 	frequency              string
@@ -100,6 +100,11 @@ func (s *RunTestStage) a_concurrency_of(concurrency int) *RunTestStage {
 	return s
 }
 
+func (s *RunTestStage) a_max_failures_of(maxFailures int) *RunTestStage {
+	s.maxFailures = maxFailures
+	return s
+}
+
 func (s *RunTestStage) a_config_file_location_of(commandsFile string) *RunTestStage {
 	s.configFile = commandsFile
 	return s
@@ -127,6 +132,8 @@ func (s *RunTestStage) i_execute_the_run_command() *RunTestStage {
 			MaxDuration:         s.duration,
 			Concurrency:         s.concurrency,
 			MaxIterations:       s.maxIterations,
+			MaxFailures:         s.maxFailures,
+			MaxFailuresRate:     s.maxFailuresRate,
 			RegisterLogHookFunc: fluentd_hook.AddFluentdLoggingHook,
 		},
 		s.build_trigger())
@@ -174,6 +181,12 @@ func (s *RunTestStage) the_command_should_fail() *RunTestStage {
 	return s
 }
 
+func (s *RunTestStage) the_command_should_succeeded() *RunTestStage {
+	s.assert.NotNil(s.runResult)
+	s.assert.Equal(false, s.runResult.Failed())
+	return s
+}
+
 func (s *RunTestStage) a_test_scenario_that_always_fails() *RunTestStage {
 	s.scenario = uuid.New().String()
 	s.f1.Add(s.scenario, func(t *f1_testing.T) (fn f1_testing.RunFn) {

internal/run/test_runner.go

@@ -48,6 +48,8 @@ func NewRun(options options.RunOptions, t *api.Trigger) (*Run, error) {
 		run.Options.RegisterLogHookFunc = logging.NoneRegisterLogHookFunc
 	}
 	run.result.IgnoreDropped = options.IgnoreDropped
+	run.result.MaxFailedIterations = options.MaxFailures
+	run.result.MaxFailedIterationsRate = options.MaxFailuresRate
 
 	progressRunner, _ := raterun.New(func(rate time.Duration, t time.Time) {
 		run.gatherProgressMetrics(rate)

internal/trigger/file/file_parser_test.go

@@ -347,6 +347,37 @@ stages:
 			expectedRates:             []int{2, 2, 2, 2, 2},
 			expectedParameters:        map[string]string{"FOO": "bar"},
 		},
+		{
+			testName: "Include max failures",
+			fileContent: `scenario: template
+limits:
+  max-duration: 1m
+  concurrency: 50
+  max-iterations: 100
+  max-failures: 10
+  max-failures-rate: 5
+  ignore-dropped: true
+stages:
+- duration: 5s
+  mode: constant
+  rate: 6/s
+  jitter: 0
+  distribution: none
+  parameters:
+    FOO: bar
+`,
+			expectedScenario:          "template",
+			expectedMaxDuration:       1 * time.Minute,
+			expectedConcurrency:       50,
+			expectedMaxIterations:     100,
+			expectedMaxFailures:       10,
+			expectedMaxFailuresRate:   5,
+			expectedIgnoreDropped:     true,
+			expectedTotalDuration:     5 * time.Second,
+			expectedIterationDuration: 1 * time.Second,
+			expectedRates:             []int{6, 6, 6, 6, 6, 6},
+			expectedParameters:        map[string]string{"FOO": "bar"},
+		},
 	} {
 		t.Run(test.testName, func(t *testing.T) {
 			now, _ := time.Parse(time.RFC3339, "2020-12-10T10:00:00+00:00")
@@ -631,6 +662,8 @@ type testData struct {
 	expectedMaxDuration       time.Duration
 	expectedIgnoreDropped     bool
 	expectedMaxIterations     int32
+	expectedMaxFailures       int
+	expectedMaxFailuresRate   int
 	expectedConcurrency       int
 	expectedUsersConcurrency  int
 	expectedRates             []int