diff --git a/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.cc b/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.cc index 580258dee..da47985e7 100644 --- a/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.cc +++ b/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.cc @@ -15,9 +15,10 @@ absl::Status StatusFromProtoRpcStatus(const google::rpc::Status& status_proto) { } // namespace FakeStepController::FakeStepController( - const nighthawk::adaptive_load::FakeStepControllerConfig& config, + nighthawk::adaptive_load::FakeStepControllerConfig config, nighthawk::client::CommandLineOptions command_line_options_template) - : is_converged_{false}, is_doomed_{false}, fixed_rps_value_{config.fixed_rps_value()}, + : input_setting_failure_countdown_{config.artificial_input_setting_failure_countdown()}, + config_{std::move(config)}, is_converged_{false}, is_doomed_{false}, command_line_options_template_{std::move(command_line_options_template)} {} bool FakeStepController::IsConverged() const { return is_converged_; } @@ -33,20 +34,31 @@ bool FakeStepController::IsDoomed(std::string& doomed_reason) const { absl::StatusOr FakeStepController::GetCurrentCommandLineOptions() const { + if (config_.has_artificial_input_setting_failure() && input_setting_failure_countdown_ <= 0) { + return StatusFromProtoRpcStatus(config_.artificial_input_setting_failure()); + } nighthawk::client::CommandLineOptions options = command_line_options_template_; - options.mutable_requests_per_second()->set_value(fixed_rps_value_); + options.mutable_requests_per_second()->set_value(config_.fixed_rps_value()); return options; } void FakeStepController::UpdateAndRecompute( const nighthawk::adaptive_load::BenchmarkResult& benchmark_result) { + if (input_setting_failure_countdown_ > 0) { + --input_setting_failure_countdown_; + } // "Convergence" is defined as the latest benchmark reporting any score > 0.0. + // "Doom" is defined as any score < 0.0. Neutral is all scores equal to 0.0. is_converged_ = false; + is_doomed_ = false; + doomed_reason_ = ""; for (const nighthawk::adaptive_load::MetricEvaluation& metric_evaluation : benchmark_result.metric_evaluations()) { - if (metric_evaluation.threshold_score() > 0.0) { + if (metric_evaluation.threshold_score() < 0.0) { + is_doomed_ = true; + doomed_reason_ = "artificial doom triggered by negative score"; + } else if (metric_evaluation.threshold_score() > 0.0) { is_converged_ = true; - break; } } } @@ -110,4 +122,20 @@ envoy::config::core::v3::TypedExtensionConfig MakeFakeStepControllerPluginConfig return outer_config; } +envoy::config::core::v3::TypedExtensionConfig +MakeFakeStepControllerPluginConfigWithInputSettingError( + int fixed_rps_value, const absl::Status& artificial_input_setting_failure, int countdown) { + envoy::config::core::v3::TypedExtensionConfig outer_config; + outer_config.set_name("nighthawk.fake_step_controller"); + nighthawk::adaptive_load::FakeStepControllerConfig config; + config.set_fixed_rps_value(fixed_rps_value); + config.mutable_artificial_input_setting_failure()->set_code( + static_cast(artificial_input_setting_failure.code())); + config.mutable_artificial_input_setting_failure()->set_message( + std::string(artificial_input_setting_failure.message())); + config.set_artificial_input_setting_failure_countdown(countdown); + outer_config.mutable_typed_config()->PackFrom(config); + return outer_config; +} + } // namespace Nighthawk diff --git a/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.h b/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.h index a039eba8c..0c462929a 100644 --- a/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.h +++ b/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.h @@ -13,6 +13,8 @@ namespace Nighthawk { /** * StepController for testing: Configurable convergence and doom countdowns, fixed RPS value. + * + * This class is not thread-safe. */ class FakeStepController : public StepController { public: @@ -22,7 +24,7 @@ class FakeStepController : public StepController { * @param config FakeStepControllerConfig proto for setting the fixed RPS value. * @param command_line_options_template A template for producing Nighthawk input. */ - FakeStepController(const nighthawk::adaptive_load::FakeStepControllerConfig& config, + FakeStepController(nighthawk::adaptive_load::FakeStepControllerConfig config, nighthawk::client::CommandLineOptions command_line_options_template); /** * @return bool The current value of |is_converged_|. @@ -42,9 +44,9 @@ class FakeStepController : public StepController { absl::StatusOr GetCurrentCommandLineOptions() const override; /** - * Updates |is_converged_| to reflect whether |benchmark_result| contains any score >0. Sets - * |is_doomed_| based whether the status in |benchmark_result| is OK; copies the status message - * into |doomed_reason_| only when the status is not OK. + * Updates |is_converged_| to reflect whether |benchmark_result| contains any score >0. Updates + * |is_doomed_| to reflect whether |benchmark_result| contains any score <0. A non-converged, + * non-doomed input has scores all equal to 0. * * @param benchmark_result A Nighthawk benchmark result proto. */ @@ -52,10 +54,13 @@ class FakeStepController : public StepController { UpdateAndRecompute(const nighthawk::adaptive_load::BenchmarkResult& benchmark_result) override; private: + // Counts down UpdateAndRecompute() calls. When this reaches zero, GetCurrentCommandLineOptions() + // starts to return an artificial input value setting failure if one is specified in the config. + int input_setting_failure_countdown_; + const nighthawk::adaptive_load::FakeStepControllerConfig config_; bool is_converged_; bool is_doomed_; std::string doomed_reason_; - const int fixed_rps_value_; const nighthawk::client::CommandLineOptions command_line_options_template_; }; @@ -91,7 +96,8 @@ MakeFakeStepControllerPluginConfig(int fixed_rps_value); * Creates a valid TypedExtensionConfig proto that activates a FakeStepController with a * FakeInputVariableSetterConfig that fails validation. * - * @param artificial_validation_error An error status. + * @param artificial_validation_error An artificial error status to be returned by + * FakeStepControllerConfigFactory::ValidateConfig() when attempting LoadStepControllerPlugin(). * * @return TypedExtensionConfig A proto that activates FakeStepController by name and includes * a FakeStepControllerConfig proto wrapped in an Any. This proto will fail validation when @@ -100,4 +106,21 @@ MakeFakeStepControllerPluginConfig(int fixed_rps_value); envoy::config::core::v3::TypedExtensionConfig MakeFakeStepControllerPluginConfigWithValidationError( const absl::Status& artificial_validation_error); +/** + * Creates a valid TypedExtensionConfig proto that activates a FakeStepController with a + * FakeInputVariableSetterConfig that returns an error from GetCurrentCommandLineOptions(). + * + * @param fixed_rps_value Value for RPS to set in the FakeStepControllerConfig proto until the + * countdown reaches zero. + * @param artificial_input_setting_failure An error status. + * @param countdown Number of times UpdateAndRecompute() must be called before + * GetCurrentCommandLineOptions() starts to return the input error status. + * + * @return TypedExtensionConfig A proto that activates FakeStepController by name and includes + * a FakeStepControllerConfig proto wrapped in an Any. + */ +envoy::config::core::v3::TypedExtensionConfig +MakeFakeStepControllerPluginConfigWithInputSettingError( + int fixed_rps_value, const absl::Status& artificial_input_setting_failure, int countdown); + } // namespace Nighthawk diff --git a/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.proto b/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.proto index 8dfe0ff76..6bf17e229 100644 --- a/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.proto +++ b/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.proto @@ -5,12 +5,25 @@ package nighthawk.adaptive_load; import "envoy/config/core/v3/extension.proto"; import "google/rpc/status.proto"; -// Configuration for FakeStepController (plugin name: "nighthawk.fake_step_controller") that always -// returns a fixed RPS value and changes converged and doomed states based on the latest reported -// BenchmarkResult. +// Configuration for FakeStepController (plugin name: "nighthawk.fake_step_controller") that returns +// a fixed RPS value and changes converged and doomed states based on the latest reported +// BenchmarkResult. Can also be programmed to return a proto validation failure, return an error +// from input value setting every time, or return an error after some number of UpdateAndRecompute() +// iterations. message FakeStepControllerConfig { - // RPS that should always be returned. Optional, default 0. + // RPS that should always be returned, except when artificial errors are configured. Optional, + // default 0. int32 fixed_rps_value = 1; // Artificial error that the plugin factory should return during validation. Optional. google.rpc.Status artificial_validation_failure = 2; + // Artificial error that should be returned from GetCurrentCommandLineOptions(). Optional. May be + // used in conjunction with |artificial_input_setting_failure_countdown| to activate error + // behavior after a delay. + google.rpc.Status artificial_input_setting_failure = 3; + // Relevant only when |artificial_input_setting_failure| is set. Number of calls to + // UpdateAndRecompute() the controller must receive before it starts to return + // |artificial_input_setting_failure|. Before this total is reached, |fixed_rps_value| is + // returned. Optional, default 0, meaning the failure is returned regardless of calls to + // UpdateAndRecompute(). + int32 artificial_input_setting_failure_countdown = 4; } diff --git a/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller_test.cc b/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller_test.cc index bdf29987f..c6952c743 100644 --- a/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller_test.cc +++ b/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller_test.cc @@ -111,19 +111,66 @@ TEST(FakeStepController, GetCurrentCommandLineOptionsReturnsRpsFromConfig) { kExpectedValue); } +TEST(FakeStepController, GetCurrentCommandLineOptionsReturnsArtificialErrorImmediately) { + FakeStepControllerConfig config; + const int kExpectedCode = ::grpc::DEADLINE_EXCEEDED; + const std::string kExpectedMessage = "artificial input setting error"; + config.mutable_artificial_input_setting_failure()->set_code(kExpectedCode); + config.mutable_artificial_input_setting_failure()->set_message(kExpectedMessage); + // Not setting countdown. + + FakeStepController step_controller(config, CommandLineOptions()); + absl::StatusOr command_line_options_or = + step_controller.GetCurrentCommandLineOptions(); + ASSERT_FALSE(command_line_options_or.ok()); + EXPECT_EQ(static_cast(command_line_options_or.status().code()), kExpectedCode); + EXPECT_EQ(command_line_options_or.status().message(), kExpectedMessage); +} + +TEST(FakeStepController, GetCurrentCommandLineOptionsReturnsArtificialErrorAfterCountdown) { + FakeStepControllerConfig config; + const int kExpectedCode = ::grpc::DEADLINE_EXCEEDED; + const std::string kExpectedMessage = "artificial input setting error"; + config.mutable_artificial_input_setting_failure()->set_code(kExpectedCode); + config.mutable_artificial_input_setting_failure()->set_message(kExpectedMessage); + config.set_artificial_input_setting_failure_countdown(2); + + FakeStepController step_controller(config, CommandLineOptions()); + absl::StatusOr command_line_options_or1 = + step_controller.GetCurrentCommandLineOptions(); + EXPECT_TRUE(command_line_options_or1.ok()); + + step_controller.UpdateAndRecompute(nighthawk::adaptive_load::BenchmarkResult()); + // Countdown should now be 1. + + absl::StatusOr command_line_options_or2 = + step_controller.GetCurrentCommandLineOptions(); + EXPECT_TRUE(command_line_options_or2.ok()); + + step_controller.UpdateAndRecompute(nighthawk::adaptive_load::BenchmarkResult()); + // Countdown should now have reached 0. + + // This should now return the artificial input setting failure: + absl::StatusOr command_line_options_or3 = + step_controller.GetCurrentCommandLineOptions(); + ASSERT_FALSE(command_line_options_or3.ok()); + EXPECT_EQ(static_cast(command_line_options_or3.status().code()), kExpectedCode); + EXPECT_EQ(command_line_options_or3.status().message(), kExpectedMessage); +} + TEST(FakeStepController, IsConvergedInitiallyReturnsFalse) { FakeStepController step_controller(FakeStepControllerConfig{}, CommandLineOptions{}); EXPECT_FALSE(step_controller.IsConverged()); } -TEST(FakeStepController, IsConvergedReturnsFalseAfterBenchmarkResultWithoutPositiveScore) { +TEST(FakeStepController, IsConvergedReturnsFalseAfterNeutralBenchmarkResult) { FakeStepController step_controller(FakeStepControllerConfig{}, CommandLineOptions{}); BenchmarkResult benchmark_result; step_controller.UpdateAndRecompute(benchmark_result); EXPECT_FALSE(step_controller.IsConverged()); } -TEST(FakeStepController, IsConvergedReturnsTrueAfterBenchmarkResultWithPositiveScore) { +TEST(FakeStepController, IsConvergedReturnsTrueAfterPositiveBenchmarkResultScore) { FakeStepController step_controller(FakeStepControllerConfig{}, CommandLineOptions{}); BenchmarkResult benchmark_result; MetricEvaluation* evaluation = benchmark_result.mutable_metric_evaluations()->Add(); @@ -132,6 +179,35 @@ TEST(FakeStepController, IsConvergedReturnsTrueAfterBenchmarkResultWithPositiveS EXPECT_TRUE(step_controller.IsConverged()); } +TEST(FakeStepController, IsDoomedReturnsFalseAfterNeutralBenchmarkResult) { + FakeStepController step_controller(FakeStepControllerConfig{}, CommandLineOptions{}); + BenchmarkResult benchmark_result; + step_controller.UpdateAndRecompute(benchmark_result); + std::string doomed_reason; + EXPECT_FALSE(step_controller.IsDoomed(doomed_reason)); +} + +TEST(FakeStepController, + IsDoomedReturnsFalseAndLeavesDoomedReasonUntouchedAfterNeutralBenchmarkResult) { + FakeStepController step_controller(FakeStepControllerConfig{}, CommandLineOptions{}); + BenchmarkResult benchmark_result; + step_controller.UpdateAndRecompute(benchmark_result); + std::string variable_that_should_not_be_written = "original value"; + EXPECT_FALSE(step_controller.IsDoomed(variable_that_should_not_be_written)); + EXPECT_EQ(variable_that_should_not_be_written, "original value"); +} + +TEST(FakeStepController, IsDoomedReturnsTrueAndSetsDoomedReasonAfterNegativeBenchmarkResultScore) { + FakeStepController step_controller(FakeStepControllerConfig{}, CommandLineOptions{}); + BenchmarkResult benchmark_result; + MetricEvaluation* evaluation = benchmark_result.mutable_metric_evaluations()->Add(); + evaluation->set_threshold_score(-1.0); + step_controller.UpdateAndRecompute(benchmark_result); + std::string doomed_reason; + EXPECT_TRUE(step_controller.IsDoomed(doomed_reason)); + EXPECT_EQ(doomed_reason, "artificial doom triggered by negative score"); +} + TEST(MakeFakeStepControllerPluginConfig, ActivatesFakeStepControllerPlugin) { absl::StatusOr plugin_or = LoadStepControllerPlugin( MakeFakeStepControllerPluginConfig(0), nighthawk::client::CommandLineOptions{}); @@ -153,7 +229,7 @@ TEST(MakeFakeStepControllerPluginConfig, ProducesFakeStepControllerPluginWithCon } TEST(MakeFakeStepControllerPluginConfigWithValidationError, - ProducesFakeStepControllerPluginWithConfiguredValue) { + ProducesFakeStepControllerPluginWithConfiguredError) { std::string kValidationErrorMessage = "artificial validation error"; absl::StatusOr plugin_or = LoadStepControllerPlugin(MakeFakeStepControllerPluginConfigWithValidationError( @@ -163,5 +239,29 @@ TEST(MakeFakeStepControllerPluginConfigWithValidationError, EXPECT_EQ(plugin_or.status().message(), kValidationErrorMessage); } +TEST(MakeFakeStepControllerPluginConfigWithInputSettingError, + ProducesFakeStepControllerPluginWithConfiguredErrorAndCountdown) { + const int kExpectedRpsValue = 123; + const std::string kInputSettingErrorMessage = "artificial input setting error"; + absl::StatusOr plugin_or = LoadStepControllerPlugin( + MakeFakeStepControllerPluginConfigWithInputSettingError( + kExpectedRpsValue, absl::DeadlineExceededError(kInputSettingErrorMessage), + /*countdown=*/1), + nighthawk::client::CommandLineOptions{}); + ASSERT_TRUE(plugin_or.ok()); + auto* plugin = dynamic_cast(plugin_or.value().get()); + ASSERT_NE(plugin, nullptr); + absl::StatusOr command_line_options_or1 = + plugin->GetCurrentCommandLineOptions(); + ASSERT_TRUE(command_line_options_or1.ok()); + EXPECT_EQ(command_line_options_or1.value().requests_per_second().value(), kExpectedRpsValue); + plugin->UpdateAndRecompute(BenchmarkResult()); + absl::StatusOr command_line_options_or2 = + plugin->GetCurrentCommandLineOptions(); + ASSERT_FALSE(command_line_options_or2.ok()); + EXPECT_EQ(command_line_options_or2.status().code(), absl::StatusCode::kDeadlineExceeded); + EXPECT_EQ(command_line_options_or2.status().message(), kInputSettingErrorMessage); +} + } // namespace } // namespace Nighthawk