diff --git a/include/nighthawk/adaptive_load/adaptive_load_controller.h b/include/nighthawk/adaptive_load/adaptive_load_controller.h index 512e8cd5f..e510ee3b6 100644 --- a/include/nighthawk/adaptive_load/adaptive_load_controller.h +++ b/include/nighthawk/adaptive_load/adaptive_load_controller.h @@ -1,6 +1,6 @@ #pragma once -#include "envoy/common/time.h" +#include "envoy/common/pure.h" #include "external/envoy/source/common/common/statusor.h" @@ -10,26 +10,35 @@ namespace Nighthawk { /** - * Performs an adaptive load session consisting of the Adjusting Stage and the - * Testing Stage. Adjusting Stage: Runs a series of short benchmarks, checks metrics according to - * MetricSpecs, and adjusts load up or down based on the result; returns an error if convergence is - * not detected before the deadline in the spec. Load adjustments and convergence detection are - * computed by a StepController plugin. Metric values are obtained through MetricsPlugins. Testing - * Stage: When the optimal load is found, runs one long benchmark to validate it. - * - * @param nighthawk_service_stub A Nighthawk Service gRPC stub. - * @param spec A proto that defines all aspects of the adaptive load session, including metrics, - * threshold, duration of adjusting stage benchmarks, and underlying Nighthawk traffic parameters. - * @param time_source An abstraction of the system clock. Normally, just construct an - * Envoy::Event::RealTimeSystem and pass it. NO_CHECK_FORMAT(real_time). If calling from an - * Envoy-based process, there may be an existing TimeSource or TimeSystem to use. If calling - * from a test, pass a fake TimeSource. - * - * @return StatusOr A proto logging the result of all traffic attempted - * and all corresponding metric values and scores, or an overall error status if the session failed. + * Contains the main loop of the adaptive load controller. Consults a StepController for load + * decisions, interacts with Nighthawk Service and MetricsPlugins. */ -absl::StatusOr PerformAdaptiveLoadSession( - nighthawk::client::NighthawkService::StubInterface* nighthawk_service_stub, - const nighthawk::adaptive_load::AdaptiveLoadSessionSpec& spec, Envoy::TimeSource& time_source); +class AdaptiveLoadController { +public: + virtual ~AdaptiveLoadController() = default; + /** + * Performs an adaptive load session consisting of the Adjusting Stage and the + * Testing Stage. + * + * Adjusting Stage: Runs a series of short benchmarks, checks metrics according to MetricSpecs, + * and adjusts load up or down based on the result. Returns an error if convergence is not + * detected before the deadline in the spec. Load adjustments and convergence detection are + * computed by a StepController plugin. Metric values are obtained through MetricsPlugins. + * + * Testing Stage: When the optimal load is found, runs one long benchmark to validate it. + * + * @param nighthawk_service_stub A Nighthawk Service gRPC stub. + * @param spec A proto that defines all aspects of the adaptive load session, including metrics, + * threshold, duration of adjusting stage benchmarks, and underlying Nighthawk traffic parameters. + * + * @return StatusOr A proto logging the result of all traffic attempted + * and all corresponding metric values and scores, or an overall error status if the session + * failed. + */ + virtual absl::StatusOr + PerformAdaptiveLoadSession( + nighthawk::client::NighthawkService::StubInterface* nighthawk_service_stub, + const nighthawk::adaptive_load::AdaptiveLoadSessionSpec& spec) PURE; +}; } // namespace Nighthawk diff --git a/source/adaptive_load/BUILD b/source/adaptive_load/BUILD index 3bc034c93..f6d8ccd8c 100644 --- a/source/adaptive_load/BUILD +++ b/source/adaptive_load/BUILD @@ -8,6 +8,30 @@ licenses(["notice"]) # Apache 2 envoy_package() +envoy_cc_library( + name = "adaptive_load_controller_impl", + srcs = [ + "adaptive_load_controller_impl.cc", + ], + hdrs = [ + "adaptive_load_controller_impl.h", + ], + repository = "@envoy", + visibility = ["//visibility:public"], + deps = [ + ":metrics_plugin_impl", + ":plugin_loader", + "//api/adaptive_load:adaptive_load_proto_cc_proto", + "//api/client:base_cc_proto", + "//include/nighthawk/adaptive_load:adaptive_load_controller", + "//include/nighthawk/adaptive_load:metrics_evaluator", + "//include/nighthawk/adaptive_load:scoring_function", + "//include/nighthawk/adaptive_load:session_spec_proto_helper", + "//include/nighthawk/adaptive_load:step_controller", + "//include/nighthawk/common:nighthawk_service_client", + ], +) + envoy_cc_library( name = "config_validator_impl", srcs = [ diff --git a/source/adaptive_load/adaptive_load_controller_impl.cc b/source/adaptive_load/adaptive_load_controller_impl.cc new file mode 100644 index 000000000..39d29c915 --- /dev/null +++ b/source/adaptive_load/adaptive_load_controller_impl.cc @@ -0,0 +1,205 @@ +#include "adaptive_load/adaptive_load_controller_impl.h" + +#include + +#include "envoy/common/exception.h" +#include "envoy/config/core/v3/base.pb.h" + +#include "nighthawk/adaptive_load/adaptive_load_controller.h" +#include "nighthawk/adaptive_load/metrics_plugin.h" +#include "nighthawk/adaptive_load/scoring_function.h" +#include "nighthawk/adaptive_load/step_controller.h" + +#include "external/envoy/source/common/common/logger.h" +#include "external/envoy/source/common/common/statusor.h" +#include "external/envoy/source/common/protobuf/protobuf.h" + +#include "api/adaptive_load/adaptive_load.pb.h" +#include "api/adaptive_load/benchmark_result.pb.h" +#include "api/adaptive_load/metric_spec.pb.h" +#include "api/client/options.pb.h" +#include "api/client/output.pb.h" +#include "api/client/service.grpc.pb.h" + +#include "absl/container/flat_hash_map.h" +#include "absl/status/status.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" +#include "adaptive_load/metrics_plugin_impl.h" +#include "adaptive_load/plugin_loader.h" + +namespace Nighthawk { + +namespace { + +using nighthawk::adaptive_load::AdaptiveLoadSessionOutput; +using nighthawk::adaptive_load::AdaptiveLoadSessionSpec; +using nighthawk::adaptive_load::BenchmarkResult; +using nighthawk::adaptive_load::MetricEvaluation; +using nighthawk::adaptive_load::MetricSpec; +using nighthawk::adaptive_load::MetricSpecWithThreshold; +using nighthawk::adaptive_load::ThresholdSpec; + +/** + * Loads and initializes MetricsPlugins requested in the session spec. Assumes the spec has already + * been validated; crashes the process otherwise. + * + * @param spec Adaptive load session spec proto that has already been validated. + * + * @return Map from MetricsPlugin names to initialized plugins, to be used in the course of a single + * adaptive load session based on the session spec. + */ +absl::flat_hash_map +LoadMetricsPlugins(const AdaptiveLoadSessionSpec& spec) { + absl::flat_hash_map name_to_custom_metrics_plugin_map; + for (const envoy::config::core::v3::TypedExtensionConfig& config : + spec.metrics_plugin_configs()) { + absl::StatusOr metrics_plugin_or = LoadMetricsPlugin(config); + RELEASE_ASSERT( + metrics_plugin_or.ok(), + absl::StrCat( + "MetricsPlugin loading error should have been caught during input validation: ", + metrics_plugin_or.status().message())); + name_to_custom_metrics_plugin_map[config.name()] = std::move(metrics_plugin_or.value()); + } + return name_to_custom_metrics_plugin_map; +} + +/** + * Loads and initializes a StepController plugin requested in the session spec. Assumes + * the spec has already been validated; crashes the process otherwise. + * + * @param spec Adaptive load session spec proto that has already been validated. + * + * @return unique_ptr Initialized StepController plugin. + */ +StepControllerPtr LoadStepControllerPluginFromSpec(const AdaptiveLoadSessionSpec& spec) { + absl::StatusOr step_controller_or = + LoadStepControllerPlugin(spec.step_controller_config(), spec.nighthawk_traffic_template()); + RELEASE_ASSERT( + step_controller_or.ok(), + absl::StrCat( + "StepController plugin loading error should have been caught during input validation: ", + step_controller_or.status().message())); + return std::move(step_controller_or.value()); +} + +} // namespace + +AdaptiveLoadControllerImpl::AdaptiveLoadControllerImpl( + const NighthawkServiceClient& nighthawk_service_client, + const MetricsEvaluator& metrics_evaluator, + const AdaptiveLoadSessionSpecProtoHelper& session_spec_proto_helper, + Envoy::TimeSource& time_source) + : nighthawk_service_client_{nighthawk_service_client}, metrics_evaluator_{metrics_evaluator}, + session_spec_proto_helper_{session_spec_proto_helper}, time_source_{time_source} {} + +absl::StatusOr AdaptiveLoadControllerImpl::PerformAndAnalyzeNighthawkBenchmark( + nighthawk::client::NighthawkService::StubInterface* nighthawk_service_stub, + const AdaptiveLoadSessionSpec& spec, + const absl::flat_hash_map& name_to_custom_plugin_map, + StepController& step_controller, Envoy::ProtobufWkt::Duration duration) { + absl::StatusOr command_line_options_or = + step_controller.GetCurrentCommandLineOptions(); + if (!command_line_options_or.ok()) { + ENVOY_LOG_MISC(error, "Error constructing Nighthawk input: {}: {}", + command_line_options_or.status().code(), + command_line_options_or.status().message()); + return command_line_options_or.status(); + } + nighthawk::client::CommandLineOptions command_line_options = command_line_options_or.value(); + // Overwrite the duration in the traffic template with the specified duration of the adjusting + // or testing stage. + *command_line_options.mutable_duration() = std::move(duration); + + ENVOY_LOG_MISC(info, "Sending load: {}", command_line_options.DebugString()); + absl::StatusOr nighthawk_response_or = + nighthawk_service_client_.PerformNighthawkBenchmark(nighthawk_service_stub, + command_line_options); + if (!nighthawk_response_or.ok()) { + ENVOY_LOG_MISC(error, "Nighthawk Service error: {}: {}", nighthawk_response_or.status().code(), + nighthawk_response_or.status().message()); + return nighthawk_response_or.status(); + } + nighthawk::client::ExecutionResponse nighthawk_response = nighthawk_response_or.value(); + + absl::StatusOr benchmark_result_or = + metrics_evaluator_.AnalyzeNighthawkBenchmark(nighthawk_response, spec, + name_to_custom_plugin_map); + if (!benchmark_result_or.ok()) { + ENVOY_LOG_MISC(error, "Benchmark scoring error: {}: {}", benchmark_result_or.status().code(), + benchmark_result_or.status().message()); + return benchmark_result_or.status(); + } + BenchmarkResult benchmark_result = benchmark_result_or.value(); + for (const MetricEvaluation& evaluation : benchmark_result.metric_evaluations()) { + ENVOY_LOG_MISC(info, "Evaluation: {}", evaluation.DebugString()); + } + step_controller.UpdateAndRecompute(benchmark_result); + return benchmark_result; +} + +absl::StatusOr AdaptiveLoadControllerImpl::PerformAdaptiveLoadSession( + nighthawk::client::NighthawkService::StubInterface* nighthawk_service_stub, + const AdaptiveLoadSessionSpec& input_spec) { + AdaptiveLoadSessionSpec spec = session_spec_proto_helper_.SetSessionSpecDefaults(input_spec); + absl::Status validation_status = session_spec_proto_helper_.CheckSessionSpec(spec); + if (!validation_status.ok()) { + ENVOY_LOG_MISC(error, "Validation failed: {}", validation_status.message()); + return validation_status; + } + absl::flat_hash_map name_to_custom_metrics_plugin_map = + LoadMetricsPlugins(spec); + StepControllerPtr step_controller = LoadStepControllerPluginFromSpec(spec); + AdaptiveLoadSessionOutput output; + + // Threshold specs are reproduced in the output proto for convenience. + for (const nighthawk::adaptive_load::MetricSpecWithThreshold& threshold : + spec.metric_thresholds()) { + *output.mutable_metric_thresholds()->Add() = threshold; + } + + // Perform adjusting stage: + Envoy::MonotonicTime start_time = time_source_.monotonicTime(); + std::string doom_reason; + do { + absl::StatusOr result_or = PerformAndAnalyzeNighthawkBenchmark( + nighthawk_service_stub, spec, name_to_custom_metrics_plugin_map, *step_controller, + spec.measuring_period()); + if (!result_or.ok()) { + return result_or.status(); + } + BenchmarkResult result = result_or.value(); + *output.mutable_adjusting_stage_results()->Add() = result; + + const std::chrono::nanoseconds time_limit_ns( + Envoy::Protobuf::util::TimeUtil::DurationToNanoseconds(spec.convergence_deadline())); + const auto elapsed_time_ns = std::chrono::duration_cast( + time_source_.monotonicTime() - start_time); + if (elapsed_time_ns > time_limit_ns) { + std::string message = absl::StrFormat("Failed to converge before deadline of %.2f seconds.", + time_limit_ns.count() / 1e9); + ENVOY_LOG_MISC(error, message); + return absl::DeadlineExceededError(message); + } + } while (!step_controller->IsConverged() && !step_controller->IsDoomed(doom_reason)); + + if (step_controller->IsDoomed(doom_reason)) { + std::string message = + absl::StrCat("Step controller determined that it can never converge: ", doom_reason); + ENVOY_LOG_MISC(error, message); + return absl::AbortedError(message); + } + + // Perform testing stage: + absl::StatusOr result_or = PerformAndAnalyzeNighthawkBenchmark( + nighthawk_service_stub, spec, name_to_custom_metrics_plugin_map, *step_controller, + spec.testing_stage_duration()); + if (!result_or.ok()) { + return result_or.status(); + } + *output.mutable_testing_stage_result() = result_or.value(); + return output; +} + +} // namespace Nighthawk diff --git a/source/adaptive_load/adaptive_load_controller_impl.h b/source/adaptive_load/adaptive_load_controller_impl.h new file mode 100644 index 000000000..4dc073892 --- /dev/null +++ b/source/adaptive_load/adaptive_load_controller_impl.h @@ -0,0 +1,83 @@ +#include "envoy/common/time.h" + +#include "nighthawk/adaptive_load/adaptive_load_controller.h" +#include "nighthawk/adaptive_load/metrics_evaluator.h" +#include "nighthawk/adaptive_load/session_spec_proto_helper.h" +#include "nighthawk/adaptive_load/step_controller.h" +#include "nighthawk/common/nighthawk_service_client.h" + +namespace Nighthawk { + +class AdaptiveLoadControllerImpl : public AdaptiveLoadController { +public: + /** + * Constructs an implementation of the adaptive load controller main loop that relies on logic in + * several helper objects. Through helpers, it performs Nighthawk Service benchmarks, obtains + * metrics from MetricsPlugins, scores the results, and consults a StepController plugin to + * determine the next load and detect convergence. All plugins are specified through the + * AdaptiveLoadSessionSpec proto. + * + * This class is thread-safe, but Nighthawk Service itself does not support multiple simultaneous + * benchmarks. + * + * Usage: + * + * AdaptiveLoadControllerImpl controller( + * NighthawkServiceClientImpl(), + * MetricsEvaluatorImpl(), + * AdaptiveLoadSessionSpecProtoHelperImpl(), + * Envoy::Event::RealTimeSystem()); // NO_CHECK_FORMAT(real_time)) + * AdaptiveLoadSessionSpec spec; + * // (set spec fields here) + * StatusOr output = + * controller.PerformAdaptiveLoadSession(&nighthawk_service_stub, spec); + * + * @param nighthawk_service_client A helper that executes Nighthawk Service benchmarks given a + * gRPC stub. + * @param metrics_evaluator A helper that obtains metrics from MetricsPlugins and Nighthawk + * Service responses, then scores them. + * @param session_spec_proto_helper A helper that sets default values and performs validation in + * an AdaptiveLoadSessionSpec proto. + * @param time_source An abstraction of the system clock. Normally, just construct an + * Envoy::Event::RealTimeSystem and pass it. NO_CHECK_FORMAT(real_time). If calling from an + * Envoy-based process, there may be an existing TimeSource or TimeSystem to use. If calling + * from a test, pass a fake TimeSource. + */ + AdaptiveLoadControllerImpl(const NighthawkServiceClient& nighthawk_service_client, + const MetricsEvaluator& metrics_evaluator, + const AdaptiveLoadSessionSpecProtoHelper& session_spec_proto_helper, + Envoy::TimeSource& time_source); + + absl::StatusOr PerformAdaptiveLoadSession( + nighthawk::client::NighthawkService::StubInterface* nighthawk_service_stub, + const nighthawk::adaptive_load::AdaptiveLoadSessionSpec& spec) override; + +private: + /** + * Gets the current load from the StepController, performs a benchmark via a Nighthawk Service, + * hands the result off for analysis, and reports the scores back to the StepController. + * + * @param nighthawk_service_stub Nighthawk Service gRPC stub. + * @param spec Proto describing the overall adaptive load session. + * @param name_to_custom_plugin_map Common map from plugin names to MetricsPlugins loaded and + * initialized once at the beginning of the session and passed to all calls of this function. + * @param step_controller The active StepController specified in the session spec proto. + * @param duration The duration of the benchmark to insert into the traffic template, different + * between adjusting and testing stages. + * + * @return BenchmarkResult Proto containing either an error status or raw Nighthawk Service + * results, metric values, and metric scores. + */ + absl::StatusOr PerformAndAnalyzeNighthawkBenchmark( + nighthawk::client::NighthawkService::StubInterface* nighthawk_service_stub, + const nighthawk::adaptive_load::AdaptiveLoadSessionSpec& spec, + const absl::flat_hash_map& name_to_custom_plugin_map, + StepController& step_controller, Envoy::ProtobufWkt::Duration duration); + + const NighthawkServiceClient& nighthawk_service_client_; + const MetricsEvaluator& metrics_evaluator_; + const AdaptiveLoadSessionSpecProtoHelper& session_spec_proto_helper_; + Envoy::TimeSource& time_source_; +}; + +} // namespace Nighthawk diff --git a/test/adaptive_load/BUILD b/test/adaptive_load/BUILD index 78518f6cb..b831726cc 100644 --- a/test/adaptive_load/BUILD +++ b/test/adaptive_load/BUILD @@ -21,6 +21,30 @@ envoy_cc_test_library( ], ) +envoy_cc_test( + name = "adaptive_load_controller_test", + srcs = ["adaptive_load_controller_test.cc"], + repository = "@envoy", + deps = [ + ":minimal_output", + "//api/client:grpc_service_lib", + "//include/nighthawk/adaptive_load:input_variable_setter", + "//include/nighthawk/adaptive_load:step_controller", + "//source/adaptive_load:adaptive_load_controller_impl", + "//source/adaptive_load:scoring_function_impl", + "//source/adaptive_load:session_spec_proto_helper_impl", + "//test/adaptive_load/fake_plugins/fake_step_controller", + "//test/common:fake_time_source", + "//test/mocks/adaptive_load:mock_metrics_evaluator", + "//test/mocks/adaptive_load:mock_session_spec_proto_helper", + "//test/mocks/common:mock_nighthawk_service_client", + "@envoy//source/common/common:assert_lib_with_external_headers", + "@envoy//source/common/common:statusor_lib_with_external_headers", + "@envoy//source/common/config:utility_lib_with_external_headers", + "@envoy//source/common/protobuf:protobuf_with_external_headers", + ], +) + envoy_cc_test( name = "input_variable_setter_test", srcs = ["input_variable_setter_test.cc"], diff --git a/test/adaptive_load/adaptive_load_controller_test.cc b/test/adaptive_load/adaptive_load_controller_test.cc new file mode 100644 index 000000000..122a7f5e6 --- /dev/null +++ b/test/adaptive_load/adaptive_load_controller_test.cc @@ -0,0 +1,303 @@ +#include + +#include "envoy/config/core/v3/base.pb.h" +#include "envoy/registry/registry.h" + +#include "nighthawk/adaptive_load/adaptive_load_controller.h" +#include "nighthawk/adaptive_load/input_variable_setter.h" +#include "nighthawk/adaptive_load/metrics_evaluator.h" +#include "nighthawk/adaptive_load/metrics_plugin.h" +#include "nighthawk/adaptive_load/scoring_function.h" +#include "nighthawk/adaptive_load/step_controller.h" + +#include "external/envoy/source/common/common/statusor.h" +#include "external/envoy/source/common/config/utility.h" +#include "external/envoy/source/common/protobuf/protobuf.h" + +#include "api/adaptive_load/adaptive_load.pb.h" +#include "api/adaptive_load/benchmark_result.pb.h" +#include "api/adaptive_load/input_variable_setter_impl.pb.h" +#include "api/adaptive_load/metric_spec.pb.h" +#include "api/adaptive_load/metrics_plugin_impl.pb.h" +#include "api/adaptive_load/scoring_function_impl.pb.h" +#include "api/adaptive_load/step_controller_impl.pb.h" +#include "api/client/options.pb.h" +#include "api/client/output.pb.h" +#include "api/client/service.grpc.pb.h" +#include "api/client/service.pb.h" +#include "api/client/service_mock.grpc.pb.h" + +#include "test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.h" +#include "test/common/fake_time_source.h" +#include "test/mocks/adaptive_load/mock_metrics_evaluator.h" +#include "test/mocks/adaptive_load/mock_session_spec_proto_helper.h" +#include "test/mocks/common/mock_nighthawk_service_client.h" + +#include "absl/container/flat_hash_map.h" +#include "absl/status/status.h" +#include "absl/strings/str_join.h" +#include "adaptive_load/adaptive_load_controller_impl.h" +#include "adaptive_load/metrics_plugin_impl.h" +#include "adaptive_load/plugin_loader.h" +#include "adaptive_load/scoring_function_impl.h" +#include "adaptive_load/session_spec_proto_helper_impl.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Nighthawk { + +namespace { + +using ::Envoy::Protobuf::util::MessageDifferencer; +using ::nighthawk::adaptive_load::AdaptiveLoadSessionOutput; +using ::nighthawk::adaptive_load::AdaptiveLoadSessionSpec; +using ::nighthawk::adaptive_load::BenchmarkResult; +using ::nighthawk::adaptive_load::MetricEvaluation; +using ::nighthawk::adaptive_load::MetricSpec; +using ::nighthawk::adaptive_load::MetricSpecWithThreshold; +using ::nighthawk::adaptive_load::ThresholdSpec; +using ::nighthawk::client::MockNighthawkServiceStub; +using ::testing::_; +using ::testing::DoAll; +using ::testing::Eq; +using ::testing::HasSubstr; +using ::testing::NiceMock; +using ::testing::Return; +using ::testing::SaveArg; +using ::testing::SetArgPointee; + +/** + * Creates a valid BenchmarkResult proto with only the score set. Useful for controlling the + * FakeStepController, which returns convergence for score > 0 and doom for a score < 0. + * + * @param score Positive number for a converging BenchmarkResult, negative number for doomed, zero + * for neither. + * + * @return BenchmarkResult An incomplete BenchmarkResult useful only for determining + * FakeStepController convergence and doom. + */ +BenchmarkResult MakeBenchmarkResultWithScore(double score) { + BenchmarkResult benchmark_result; + MetricEvaluation* evaluation = benchmark_result.mutable_metric_evaluations()->Add(); + evaluation->set_threshold_score(score); + return benchmark_result; +} + +/** + * Creates a minimal AdaptiveLoadSessionSpec with a FakeStepController. + * + * @return AdaptiveLoadSessionSpec with a FakeStepController and enough fields set to pass + * validation. + */ +AdaptiveLoadSessionSpec MakeValidAdaptiveLoadSessionSpec() { + AdaptiveLoadSessionSpec spec; + spec.mutable_convergence_deadline()->set_seconds(100); + *spec.mutable_step_controller_config() = MakeFakeStepControllerPluginConfigWithRps(10); + MetricSpecWithThreshold* expected_spec_with_threshold = spec.mutable_metric_thresholds()->Add(); + expected_spec_with_threshold->mutable_metric_spec()->set_metric_name("success-rate"); + expected_spec_with_threshold->mutable_threshold_spec()->mutable_scoring_function()->set_name( + "nighthawk.binary_scoring"); + expected_spec_with_threshold->mutable_threshold_spec() + ->mutable_scoring_function() + ->mutable_typed_config() + ->PackFrom(nighthawk::adaptive_load::BinaryScoringFunctionConfig()); + return spec; +} + +class AdaptiveLoadControllerImplFixture : public testing::Test { +public: + void SetUp() override { + ON_CALL(mock_nighthawk_service_client_, PerformNighthawkBenchmark) + .WillByDefault(Return(nighthawk::client::ExecutionResponse())); + } + +protected: + NiceMock mock_nighthawk_service_client_; + NiceMock mock_metrics_evaluator_; + FakeIncrementingMonotonicTimeSource fake_time_source_; + MockNighthawkServiceStub mock_nighthawk_service_stub_; + // Real spec helper is simpler to use because SetSessionSpecDefaults preserves values a test + // sets in the spec; the mock inconveniently discards the input and returns an empty spec. + AdaptiveLoadSessionSpecProtoHelperImpl real_spec_proto_helper_; +}; + +TEST_F(AdaptiveLoadControllerImplFixture, SetsSpecDefaults) { + NiceMock mock_spec_proto_helper; + AdaptiveLoadSessionSpec spec = MakeValidAdaptiveLoadSessionSpec(); + EXPECT_CALL(mock_spec_proto_helper, SetSessionSpecDefaults(_)).WillOnce(Return(spec)); + + AdaptiveLoadControllerImpl controller(mock_nighthawk_service_client_, mock_metrics_evaluator_, + mock_spec_proto_helper, fake_time_source_); + + (void)controller.PerformAdaptiveLoadSession(&mock_nighthawk_service_stub_, spec); +} + +TEST_F(AdaptiveLoadControllerImplFixture, PropagatesSpecValidationError) { + NiceMock mock_spec_proto_helper; + EXPECT_CALL(mock_spec_proto_helper, CheckSessionSpec(_)) + .WillOnce(Return(absl::DataLossError("artificial spec error"))); + + AdaptiveLoadControllerImpl controller(mock_nighthawk_service_client_, mock_metrics_evaluator_, + mock_spec_proto_helper, fake_time_source_); + + absl::StatusOr output_or = controller.PerformAdaptiveLoadSession( + &mock_nighthawk_service_stub_, MakeValidAdaptiveLoadSessionSpec()); + ASSERT_FALSE(output_or.ok()); + EXPECT_EQ(output_or.status().code(), absl::StatusCode::kDataLoss); + EXPECT_EQ(output_or.status().message(), "artificial spec error"); +} + +TEST_F(AdaptiveLoadControllerImplFixture, CopiesThresholdSpecsIntoOutput) { + EXPECT_CALL(mock_metrics_evaluator_, AnalyzeNighthawkBenchmark(_, _, _)) + .WillRepeatedly(Return(MakeBenchmarkResultWithScore(1.0))); + + AdaptiveLoadSessionSpecProtoHelperImpl spec_helper; + AdaptiveLoadControllerImpl controller(mock_nighthawk_service_client_, mock_metrics_evaluator_, + spec_helper, fake_time_source_); + + AdaptiveLoadSessionSpec spec = + spec_helper.SetSessionSpecDefaults(MakeValidAdaptiveLoadSessionSpec()); + absl::StatusOr output_or = + controller.PerformAdaptiveLoadSession(&mock_nighthawk_service_stub_, spec); + ASSERT_TRUE(output_or.ok()); + ASSERT_GT(output_or.value().metric_thresholds_size(), 0); + MetricSpecWithThreshold actual_spec_with_threshold = output_or.value().metric_thresholds(0); + EXPECT_TRUE( + MessageDifferencer::Equivalent(actual_spec_with_threshold, spec.metric_thresholds(0))); + EXPECT_EQ(actual_spec_with_threshold.DebugString(), spec.metric_thresholds(0).DebugString()); +} + +TEST_F(AdaptiveLoadControllerImplFixture, TimesOutIfNeverConverged) { + EXPECT_CALL(mock_metrics_evaluator_, AnalyzeNighthawkBenchmark(_, _, _)) + .WillRepeatedly(Return(MakeBenchmarkResultWithScore(0.0))); + + AdaptiveLoadControllerImpl controller(mock_nighthawk_service_client_, mock_metrics_evaluator_, + real_spec_proto_helper_, fake_time_source_); + + AdaptiveLoadSessionSpec spec = MakeValidAdaptiveLoadSessionSpec(); + absl::StatusOr output_or = + controller.PerformAdaptiveLoadSession(&mock_nighthawk_service_stub_, spec); + ASSERT_FALSE(output_or.ok()); + EXPECT_EQ(output_or.status().code(), absl::StatusCode::kDeadlineExceeded); + EXPECT_THAT(output_or.status().message(), HasSubstr("Failed to converge")); +} + +TEST_F(AdaptiveLoadControllerImplFixture, ReturnsErrorWhenDoomed) { + EXPECT_CALL(mock_metrics_evaluator_, AnalyzeNighthawkBenchmark(_, _, _)) + .WillOnce(Return(MakeBenchmarkResultWithScore(-1.0))); + + AdaptiveLoadControllerImpl controller(mock_nighthawk_service_client_, mock_metrics_evaluator_, + real_spec_proto_helper_, fake_time_source_); + + absl::StatusOr output_or = controller.PerformAdaptiveLoadSession( + &mock_nighthawk_service_stub_, MakeValidAdaptiveLoadSessionSpec()); + ASSERT_FALSE(output_or.ok()); + EXPECT_EQ(output_or.status().code(), absl::StatusCode::kAborted); + EXPECT_THAT(output_or.status().message(), HasSubstr("can never converge")); +} + +TEST_F(AdaptiveLoadControllerImplFixture, + PropagatesErrorWhenInputValueSettingFailsInAdjustingStage) { + const std::string kExpectedErrorMessage = "artificial input setting error"; + EXPECT_CALL(mock_metrics_evaluator_, AnalyzeNighthawkBenchmark(_, _, _)) + .WillRepeatedly(Return(MakeBenchmarkResultWithScore(-1.0))); + + AdaptiveLoadControllerImpl controller(mock_nighthawk_service_client_, mock_metrics_evaluator_, + real_spec_proto_helper_, fake_time_source_); + + AdaptiveLoadSessionSpec spec = MakeValidAdaptiveLoadSessionSpec(); + *spec.mutable_step_controller_config() = MakeFakeStepControllerPluginConfigWithInputSettingError( + 10, absl::DataLossError(kExpectedErrorMessage), /*countdown=*/0); + absl::StatusOr output_or = + controller.PerformAdaptiveLoadSession(&mock_nighthawk_service_stub_, spec); + ASSERT_FALSE(output_or.ok()); + EXPECT_EQ(output_or.status().code(), absl::StatusCode::kDataLoss); + EXPECT_THAT(output_or.status().message(), HasSubstr(kExpectedErrorMessage)); +} + +TEST_F(AdaptiveLoadControllerImplFixture, PropagatesErrorWhenInputValueSettingFailsInTestingStage) { + const std::string kExpectedErrorMessage = "artificial input setting error"; + EXPECT_CALL(mock_metrics_evaluator_, AnalyzeNighthawkBenchmark(_, _, _)) + .WillRepeatedly(Return(MakeBenchmarkResultWithScore(1.0))); + + AdaptiveLoadControllerImpl controller(mock_nighthawk_service_client_, mock_metrics_evaluator_, + real_spec_proto_helper_, fake_time_source_); + + AdaptiveLoadSessionSpec spec = MakeValidAdaptiveLoadSessionSpec(); + *spec.mutable_step_controller_config() = MakeFakeStepControllerPluginConfigWithInputSettingError( + 10, absl::DataLossError(kExpectedErrorMessage), /*countdown=*/1); + absl::StatusOr output_or = + controller.PerformAdaptiveLoadSession(&mock_nighthawk_service_stub_, spec); + ASSERT_FALSE(output_or.ok()); + EXPECT_EQ(output_or.status().code(), absl::StatusCode::kDataLoss); + EXPECT_THAT(output_or.status().message(), HasSubstr(kExpectedErrorMessage)); +} + +TEST_F(AdaptiveLoadControllerImplFixture, PropagatesErrorFromNighthawkService) { + const std::string kExpectedErrorMessage = "artificial nighthawk service error"; + EXPECT_CALL(mock_nighthawk_service_client_, PerformNighthawkBenchmark(_, _)) + .WillOnce(Return(absl::DataLossError(kExpectedErrorMessage))); + + AdaptiveLoadControllerImpl controller(mock_nighthawk_service_client_, mock_metrics_evaluator_, + real_spec_proto_helper_, fake_time_source_); + + absl::StatusOr output_or = controller.PerformAdaptiveLoadSession( + &mock_nighthawk_service_stub_, MakeValidAdaptiveLoadSessionSpec()); + ASSERT_FALSE(output_or.ok()); + EXPECT_EQ(output_or.status().code(), absl::StatusCode::kDataLoss); + EXPECT_THAT(output_or.status().message(), HasSubstr(kExpectedErrorMessage)); +} + +TEST_F(AdaptiveLoadControllerImplFixture, PropagatesErrorFromMetricsEvaluator) { + const std::string kExpectedErrorMessage = "artificial metrics evaluator error"; + EXPECT_CALL(mock_metrics_evaluator_, AnalyzeNighthawkBenchmark(_, _, _)) + .WillOnce(Return(absl::DataLossError(kExpectedErrorMessage))); + + AdaptiveLoadControllerImpl controller(mock_nighthawk_service_client_, mock_metrics_evaluator_, + real_spec_proto_helper_, fake_time_source_); + + absl::StatusOr output_or = controller.PerformAdaptiveLoadSession( + &mock_nighthawk_service_stub_, MakeValidAdaptiveLoadSessionSpec()); + ASSERT_FALSE(output_or.ok()); + EXPECT_EQ(output_or.status().code(), absl::StatusCode::kDataLoss); + EXPECT_THAT(output_or.status().message(), HasSubstr(kExpectedErrorMessage)); +} + +TEST_F(AdaptiveLoadControllerImplFixture, StoresAdjustingStageResult) { + BenchmarkResult expected_benchmark_result = MakeBenchmarkResultWithScore(1.0); + EXPECT_CALL(mock_metrics_evaluator_, AnalyzeNighthawkBenchmark(_, _, _)) + .WillRepeatedly(Return(expected_benchmark_result)); + + AdaptiveLoadControllerImpl controller(mock_nighthawk_service_client_, mock_metrics_evaluator_, + real_spec_proto_helper_, fake_time_source_); + + AdaptiveLoadSessionSpec spec = MakeValidAdaptiveLoadSessionSpec(); + absl::StatusOr output_or = + controller.PerformAdaptiveLoadSession(&mock_nighthawk_service_stub_, spec); + ASSERT_TRUE(output_or.ok()); + ASSERT_EQ(output_or.value().adjusting_stage_results_size(), 1); + const BenchmarkResult& actual_benchmark_result = output_or.value().adjusting_stage_results(0); + EXPECT_TRUE(MessageDifferencer::Equivalent(actual_benchmark_result, expected_benchmark_result)); + EXPECT_EQ(actual_benchmark_result.DebugString(), expected_benchmark_result.DebugString()); +} + +TEST_F(AdaptiveLoadControllerImplFixture, StoresTestingStageResult) { + BenchmarkResult expected_benchmark_result = MakeBenchmarkResultWithScore(1.0); + EXPECT_CALL(mock_metrics_evaluator_, AnalyzeNighthawkBenchmark(_, _, _)) + .WillRepeatedly(Return(expected_benchmark_result)); + + AdaptiveLoadControllerImpl controller(mock_nighthawk_service_client_, mock_metrics_evaluator_, + real_spec_proto_helper_, fake_time_source_); + + AdaptiveLoadSessionSpec spec = MakeValidAdaptiveLoadSessionSpec(); + absl::StatusOr output_or = + controller.PerformAdaptiveLoadSession(&mock_nighthawk_service_stub_, spec); + ASSERT_TRUE(output_or.ok()); + const BenchmarkResult& actual_benchmark_result = output_or.value().testing_stage_result(); + EXPECT_TRUE(MessageDifferencer::Equivalent(actual_benchmark_result, expected_benchmark_result)); + EXPECT_EQ(actual_benchmark_result.DebugString(), expected_benchmark_result.DebugString()); +} + +} // namespace + +} // namespace Nighthawk diff --git a/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.cc b/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.cc index da47985e7..6bb0e0ba1 100644 --- a/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.cc +++ b/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.cc @@ -98,7 +98,7 @@ FakeStepControllerConfigFactory::ValidateConfig(const Envoy::Protobuf::Message& REGISTER_FACTORY(FakeStepControllerConfigFactory, StepControllerConfigFactory); envoy::config::core::v3::TypedExtensionConfig -MakeFakeStepControllerPluginConfig(int fixed_rps_value) { +MakeFakeStepControllerPluginConfigWithRps(int fixed_rps_value) { envoy::config::core::v3::TypedExtensionConfig outer_config; outer_config.set_name("nighthawk.fake_step_controller"); nighthawk::adaptive_load::FakeStepControllerConfig config; diff --git a/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.h b/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.h index 0c462929a..c22f72826 100644 --- a/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.h +++ b/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.h @@ -90,7 +90,7 @@ DECLARE_FACTORY(FakeStepControllerConfigFactory); * FakeStepControllerConfig proto wrapped in an Any. */ envoy::config::core::v3::TypedExtensionConfig -MakeFakeStepControllerPluginConfig(int fixed_rps_value); +MakeFakeStepControllerPluginConfigWithRps(int fixed_rps_value); /** * Creates a valid TypedExtensionConfig proto that activates a FakeStepController with a diff --git a/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller_test.cc b/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller_test.cc index c6952c743..905c9fa02 100644 --- a/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller_test.cc +++ b/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller_test.cc @@ -210,15 +210,16 @@ TEST(FakeStepController, IsDoomedReturnsTrueAndSetsDoomedReasonAfterNegativeBenc TEST(MakeFakeStepControllerPluginConfig, ActivatesFakeStepControllerPlugin) { absl::StatusOr plugin_or = LoadStepControllerPlugin( - MakeFakeStepControllerPluginConfig(0), nighthawk::client::CommandLineOptions{}); + MakeFakeStepControllerPluginConfigWithRps(0), nighthawk::client::CommandLineOptions{}); ASSERT_TRUE(plugin_or.ok()); EXPECT_NE(dynamic_cast(plugin_or.value().get()), nullptr); } TEST(MakeFakeStepControllerPluginConfig, ProducesFakeStepControllerPluginWithConfiguredValue) { const int kExpectedRps = 5; - absl::StatusOr plugin_or = LoadStepControllerPlugin( - MakeFakeStepControllerPluginConfig(kExpectedRps), nighthawk::client::CommandLineOptions{}); + absl::StatusOr plugin_or = + LoadStepControllerPlugin(MakeFakeStepControllerPluginConfigWithRps(kExpectedRps), + nighthawk::client::CommandLineOptions{}); ASSERT_TRUE(plugin_or.ok()); auto* plugin = dynamic_cast(plugin_or.value().get()); ASSERT_NE(plugin, nullptr);