diff --git a/api/adaptive_load/adaptive_load.proto b/api/adaptive_load/adaptive_load.proto index 8b0d2724e..474d0c433 100644 --- a/api/adaptive_load/adaptive_load.proto +++ b/api/adaptive_load/adaptive_load.proto @@ -33,11 +33,17 @@ message AdaptiveLoadSessionSpec { // A proto describing Nighthawk Service traffic. See // https://github.com/envoyproxy/nighthawk/blob/master/api/client/options.proto // - // The adaptive load controller will return an error if the |duration| or - // |open_loop| fields are set within |nighthawk_traffic_options|. The - // controller will also be configured to overwrite at least one of the - // numerical fields during the search, such as requests_per_second, so any - // value of those fields specified here will be ignored. + // The adaptive load controller will return an error if the |duration| field is set within + // |nighthawk_traffic_options|. + // + // If |open_loop| is unset, it will be overridden to true by the adaptive load controller. This is + // to support the typical case where the controller needs full control over attempted requests per + // second, which could conflict with the backpressure mechanism of closed-loop mode. Note that in + // standalone Nighthawk clients, closed-loop mode is the default so |open_loop| defaults to false. + // + // The controller will override at least one field in this proto to vary the load, such as the + // requests_per_second field or headers. Any existing value for such a field in the template will + // be ignored. // // All other fields in |nighthawk_traffic_options| are passed through to the // Nighthawk Service. @@ -65,10 +71,6 @@ message AdaptiveLoadSessionSpec { // Complete description of an adaptive load session, including metric scores // for every degree of load attempted during the adjusting stage. message AdaptiveLoadSessionOutput { - // Overall status of the session with error detail. INVALID_ARGUMENT if the input spec contained - // errors, DEADLINE_EXCEEDED if convergence did not occur before the deadline, ABORTED if the step - // controller determined it can never converge. - google.rpc.Status session_status = 1; // Results of each short benchmark performed during the adjusting stage. repeated BenchmarkResult adjusting_stage_results = 2; // Result of the single benchmark of the testing stage. diff --git a/api/adaptive_load/benchmark_result.proto b/api/adaptive_load/benchmark_result.proto index 13e2c0ab5..4989a33b9 100644 --- a/api/adaptive_load/benchmark_result.proto +++ b/api/adaptive_load/benchmark_result.proto @@ -30,9 +30,6 @@ message BenchmarkResult { // Raw Nighthawk Service output. Includes start/end times and full Nighthawk // Service input spec. nighthawk.client.Output nighthawk_service_output = 1; - // Execution status of this call to the Nighthawk Service. This will record errors connecting to - // the Nighthawk Service and internal errors returned from the Nighthawk Service. - google.rpc.Status status = 2; // Status of all declared metrics during this benchmark session. Not present // in the event of Nighthawk Service errors. repeated MetricEvaluation metric_evaluations = 3; diff --git a/api/client/BUILD b/api/client/BUILD index 3befdf496..0eb4c6037 100644 --- a/api/client/BUILD +++ b/api/client/BUILD @@ -23,6 +23,7 @@ cc_grpc_library( srcs = [ ":base", ], + generate_mocks = True, grpc_only = True, proto_only = False, use_external = False, diff --git a/include/nighthawk/adaptive_load/BUILD b/include/nighthawk/adaptive_load/BUILD index 3607226d1..07e6522c7 100644 --- a/include/nighthawk/adaptive_load/BUILD +++ b/include/nighthawk/adaptive_load/BUILD @@ -21,6 +21,7 @@ envoy_basic_cc_library( "@envoy//include/envoy/common:base_includes", "@envoy//include/envoy/common:time_interface", "@envoy//include/envoy/config:typed_config_interface", + "@envoy//source/common/common:statusor_lib_with_external_headers", ], ) diff --git a/include/nighthawk/adaptive_load/adaptive_load_controller.h b/include/nighthawk/adaptive_load/adaptive_load_controller.h index 6ede4e315..512e8cd5f 100644 --- a/include/nighthawk/adaptive_load/adaptive_load_controller.h +++ b/include/nighthawk/adaptive_load/adaptive_load_controller.h @@ -2,6 +2,8 @@ #include "envoy/common/time.h" +#include "external/envoy/source/common/common/statusor.h" + #include "api/adaptive_load/adaptive_load.pb.h" #include "api/client/service.grpc.pb.h" @@ -23,11 +25,10 @@ namespace Nighthawk { * Envoy-based process, there may be an existing TimeSource or TimeSystem to use. If calling * from a test, pass a fake TimeSource. * - * @return AdaptiveLoadSessionOutput a proto logging the result of all traffic attempted and all - * corresponding metric values and scores. Any errors that occur will be recorded in the - * |session_status| field. + * @return StatusOr A proto logging the result of all traffic attempted + * and all corresponding metric values and scores, or an overall error status if the session failed. */ -nighthawk::adaptive_load::AdaptiveLoadSessionOutput PerformAdaptiveLoadSession( +absl::StatusOr PerformAdaptiveLoadSession( nighthawk::client::NighthawkService::StubInterface* nighthawk_service_stub, const nighthawk::adaptive_load::AdaptiveLoadSessionSpec& spec, Envoy::TimeSource& time_source); diff --git a/source/adaptive_load/BUILD b/source/adaptive_load/BUILD index 3eca6c792..17d81acef 100644 --- a/source/adaptive_load/BUILD +++ b/source/adaptive_load/BUILD @@ -8,6 +8,40 @@ licenses(["notice"]) # Apache 2 envoy_package() +envoy_cc_library( + name = "adaptive_load", + repository = "@envoy", + visibility = ["//visibility:public"], + deps = [ + ":adaptive_load_controller_impl", + ":input_variable_setter_impl", + ":metrics_plugin_impl", + ":scoring_function_impl", + ":step_controller_impl", + ], +) + +envoy_cc_library( + name = "adaptive_load_controller_impl", + srcs = [ + "adaptive_load_controller_impl.cc", + ], + repository = "@envoy", + visibility = ["//visibility:public"], + deps = [ + ":metrics_plugin_impl", + ":plugin_loader", + "//api/adaptive_load:adaptive_load_proto_cc_proto", + "//api/client:base_cc_proto", + "//api/client:grpc_service_lib", + "//include/nighthawk/adaptive_load:adaptive_load_controller", + "//include/nighthawk/adaptive_load:scoring_function", + "//include/nighthawk/adaptive_load:step_controller", + "@envoy//source/common/common:minimal_logger_lib_with_external_headers", + "@envoy//source/common/event:real_time_system_lib_with_external_headers", + ], +) + envoy_cc_library( name = "config_validator_impl", srcs = [ diff --git a/source/adaptive_load/adaptive_load_controller_impl.cc b/source/adaptive_load/adaptive_load_controller_impl.cc new file mode 100644 index 000000000..62ce16fb2 --- /dev/null +++ b/source/adaptive_load/adaptive_load_controller_impl.cc @@ -0,0 +1,476 @@ +#include + +#include "envoy/common/exception.h" +#include "envoy/config/core/v3/base.pb.h" + +#include "nighthawk/adaptive_load/adaptive_load_controller.h" +#include "nighthawk/adaptive_load/metrics_plugin.h" +#include "nighthawk/adaptive_load/scoring_function.h" +#include "nighthawk/adaptive_load/step_controller.h" + +#include "external/envoy/source/common/common/logger.h" +#include "external/envoy/source/common/common/statusor.h" +#include "external/envoy/source/common/protobuf/protobuf.h" + +#include "api/adaptive_load/adaptive_load.pb.h" +#include "api/adaptive_load/benchmark_result.pb.h" +#include "api/adaptive_load/metric_spec.pb.h" +#include "api/client/options.pb.h" +#include "api/client/output.pb.h" +#include "api/client/service.grpc.pb.h" + +#include "absl/container/flat_hash_map.h" +#include "absl/status/status.h" +#include "absl/strings/str_join.h" +#include "adaptive_load/metrics_plugin_impl.h" +#include "adaptive_load/plugin_loader.h" + +namespace Nighthawk { + +namespace { + +using namespace Envoy; // for Envoy::Logger + +using nighthawk::adaptive_load::AdaptiveLoadSessionOutput; +using nighthawk::adaptive_load::AdaptiveLoadSessionSpec; +using nighthawk::adaptive_load::BenchmarkResult; +using nighthawk::adaptive_load::MetricEvaluation; +using nighthawk::adaptive_load::MetricSpec; +using nighthawk::adaptive_load::MetricSpecWithThreshold; +using nighthawk::adaptive_load::ThresholdSpec; + +/** + * Runs a single benchmark using a Nighthawk Service. + * + * @param nighthawk_service_stub Nighthawk Service gRPC stub. + * @param command_line_options Nighthawk Service benchmark request proto generated by the + * StepController, without the duration set. + * @param duration Duration to insert into the benchmark request. + * + * @return ExecutionResponse If we reached the Nighthawk Service, this is the raw ExecutionResponse + * proto, containing the benchmark data or possibly an error message from Nighthawk Service; if we + * had trouble communicating with the Nighthawk Service, we insert the details into the + * |error_detail| of this proto. + */ +absl::StatusOr PerformNighthawkBenchmark( + nighthawk::client::NighthawkService::StubInterface* nighthawk_service_stub, + const nighthawk::client::CommandLineOptions& command_line_options, + const Envoy::Protobuf::Duration& duration) { + nighthawk::client::CommandLineOptions options = command_line_options; + *options.mutable_duration() = duration; + + nighthawk::client::ExecutionRequest request; + nighthawk::client::ExecutionResponse response; + *request.mutable_start_request()->mutable_options() = options; + + ::grpc::ClientContext context; + std::shared_ptr<::grpc::ClientReaderWriterInterface> + stream(nighthawk_service_stub->ExecutionStream(&context)); + + if (!stream->Write(request)) { + return absl::UnknownError("Failed to write request to the Nighthawk Service gRPC channel."); + } else if (!stream->WritesDone()) { + return absl::UnknownError("WritesDone() failed on the Nighthawk Service gRPC channel."); + } + + bool got_response = false; + while (stream->Read(&response)) { + RELEASE_ASSERT(!got_response, "gRPC server has started sending more than one message."); + got_response = true; + } + if (!got_response) { + return absl::UnknownError("Nighthawk Service did not send a gRPC response."); + } + ::grpc::Status status = stream->Finish(); + if (!status.ok()) { + return absl::Status(static_cast(status.error_code()), status.error_message()); + } + return response; +} + +/** + * Given a MetricSpec, obtains a single metric value from the MetricPlugin and optionally scores it + * according to a threshold and scoring function. + * + * @param metric_spec The metric spec identifying the metric by name and plugin name. + * @param metrics_plugin An already activated MetricsPlugin used by the metric_spec. + * @param threshold_spec Proto describing the threshold and scoring function. Nullptr if the metric + * is informational only. + * @param errors A vector to append error messages to. + * + * @return MetricEvaluation A proto containing the metric value and its score if a threshold was + * specified, or an error mesasge if the metric could not be obtained from the MetricsPlugin. + */ +absl::StatusOr +EvaluateMetric(const MetricSpec& metric_spec, MetricsPlugin& metrics_plugin, + const nighthawk::adaptive_load::ThresholdSpec* threshold_spec) { + MetricEvaluation evaluation; + evaluation.set_metric_id( + absl::StrCat(metric_spec.metrics_plugin_name(), "/", metric_spec.metric_name())); + const absl::StatusOr metric_value_or = + metrics_plugin.GetMetricByName(metric_spec.metric_name()); + if (!metric_value_or.ok()) { + return absl::Status(static_cast(metric_value_or.status().code()), + absl::StrCat("Error calling MetricsPlugin '", + metric_spec.metrics_plugin_name(), ": ", + metric_value_or.status().message())); + } + const double metric_value = metric_value_or.value(); + evaluation.set_metric_value(metric_value); + if (threshold_spec == nullptr) { + // Informational metric. + evaluation.set_weight(0.0); + } else { + evaluation.set_weight(threshold_spec->weight().value()); + absl::StatusOr scoring_function_or = + LoadScoringFunctionPlugin(threshold_spec->scoring_function()); + RELEASE_ASSERT(scoring_function_or.ok(), + absl::StrCat("ScoringFunction plugin loading error should have been caught " + "during input validation: ", + scoring_function_or.status().message())); + ScoringFunctionPtr scoring_function = std::move(scoring_function_or.value()); + evaluation.set_threshold_score(scoring_function->EvaluateMetric(metric_value)); + } + return evaluation; +} + +/** + * Extracts metric descriptors and corresponding thresholds to an ordered list and a map. Allows for + * uniform treatment of scored and informational metrics. + * + * @param spec The adaptive load session spec. + * @param metric_specs A list of MetricSpecs in order of definition. + * @param threshold_spec_from_metric_spec A map from each MetricSpec to its threshold if it had one, + * or nullptr if it was an informational metric. + * + */ +void ExtractMetricSpecs(const AdaptiveLoadSessionSpec& spec, + std::vector& metric_specs, + absl::flat_hash_map& + threshold_spec_from_metric_spec) { + for (const MetricSpecWithThreshold& metric_threshold : spec.metric_thresholds()) { + metric_specs.push_back(&metric_threshold.metric_spec()); + threshold_spec_from_metric_spec[&metric_threshold.metric_spec()] = + &metric_threshold.threshold_spec(); + } + for (const MetricSpec& metric_spec : spec.informational_metric_specs()) { + metric_specs.push_back(&metric_spec); + threshold_spec_from_metric_spec[&metric_spec] = nullptr; + } +} + +/** + * Analyzes a recently completed Nighthawk Service benchmark against configured MetricThresholds. + * Queries outside MetricsPlugins if configured and/or uses "nighthawk.builtin" plugin to exrtact + * stats and counters from the Nighthawk Service output. + * + * @param nighthawk_response Proto returned from Nighthawk Service describing a single benchmark + * session. + * @param spec Top-level proto defining the adaptive load session. + * @param name_to_custom_metrics_plugin_map Common map from plugin names to MetricsPlugins, loaded + * and initialized once at the beginning of the session and passed to all calls of this function. + * + * @return BenchmarkResult Proto containing metric scores for this Nighthawk Service benchmark + * session, or an error propagated from the Nighthawk Service or MetricsPlugins. + */ +absl::StatusOr AnalyzeNighthawkBenchmark( + const nighthawk::client::ExecutionResponse& nighthawk_response, + const AdaptiveLoadSessionSpec& spec, + const absl::flat_hash_map& name_to_custom_metrics_plugin_map) { + if (nighthawk_response.error_detail().code() != ::grpc::OK) { + return absl::Status(static_cast(nighthawk_response.error_detail().code()), + nighthawk_response.error_detail().message()); + } + + BenchmarkResult benchmark_result; + *benchmark_result.mutable_nighthawk_service_output() = nighthawk_response.output(); + + // A map containing all available MetricsPlugins: preloaded custom plugins shared across all + // benchmarks, and a freshly instantiated builtin plugin for this benchmark only. + absl::flat_hash_map name_to_plugin_map; + for (const auto& name_plugin_pair : name_to_custom_metrics_plugin_map) { + name_to_plugin_map[name_plugin_pair.first] = name_plugin_pair.second.get(); + } + auto builtin_plugin = + std::make_unique(nighthawk_response.output()); + name_to_plugin_map["nighthawk.builtin"] = builtin_plugin.get(); + + // MetricSpecs in original order of definition. + std::vector metric_specs; + // Pointer to the corresponding ThresholdSpec, or nullptr for informational metrics. + absl::flat_hash_map + threshold_spec_from_metric_spec; + ExtractMetricSpecs(spec, metric_specs, threshold_spec_from_metric_spec); + + std::vector errors; + for (const nighthawk::adaptive_load::MetricSpec* metric_spec : metric_specs) { + absl::StatusOr evaluation_or = + EvaluateMetric(*metric_spec, *name_to_plugin_map[metric_spec->metrics_plugin_name()], + threshold_spec_from_metric_spec[metric_spec]); + if (!evaluation_or.ok()) { + errors.emplace_back(absl::StrCat("Error evaluating metric: ", evaluation_or.status().code(), + ": ", evaluation_or.status().message())); + continue; + } + *benchmark_result.mutable_metric_evaluations()->Add() = evaluation_or.value(); + } + if (!errors.empty()) { + return absl::InternalError(absl::StrJoin(errors, "\n")); + } + return benchmark_result; +} + +/** + * Performs a benchmark via a Nighthawk Service, then hands the result off for analysis. + * + * @param nighthawk_service_stub Nighthawk Service gRPC stub. + * @param spec Proto describing the overall adaptive load session. + * @param name_to_custom_plugin_map Common map from plugin names to MetricsPlugins loaded and + * initialized once at the beginning of the session and passed to all calls of this function. + * @param command_line_options Full input for the Nighthawk Service, except for the duration. + * @param duration Duration to insert into the benchmark request. + * + * @return BenchmarkResult Proto containing either an error status or raw Nighthawk Service results, + * metric values, and metric scores. + */ +absl::StatusOr PerformAndAnalyzeNighthawkBenchmark( + nighthawk::client::NighthawkService::StubInterface* nighthawk_service_stub, + const AdaptiveLoadSessionSpec& spec, + const absl::flat_hash_map& name_to_custom_plugin_map, + const nighthawk::client::CommandLineOptions& command_line_options, + const Envoy::Protobuf::Duration& duration) { + absl::StatusOr nighthawk_response_or = + PerformNighthawkBenchmark(nighthawk_service_stub, command_line_options, duration); + if (!nighthawk_response_or.ok()) { + ENVOY_LOG_MISC(error, "Nighthawk Service error: {}", nighthawk_response_or.status().message()); + return nighthawk_response_or.status(); + } + nighthawk::client::ExecutionResponse nighthawk_response = nighthawk_response_or.value(); + + absl::StatusOr benchmark_result_or = + AnalyzeNighthawkBenchmark(nighthawk_response, spec, name_to_custom_plugin_map); + if (!benchmark_result_or.ok()) { + ENVOY_LOG_MISC(error, "Benchmark scoring error: {}", benchmark_result_or.status().message()); + return benchmark_result_or.status(); + } + BenchmarkResult benchmark_result = benchmark_result_or.value(); + for (const MetricEvaluation& evaluation : benchmark_result.metric_evaluations()) { + ENVOY_LOG_MISC(info, "Evaluation: {}", evaluation.DebugString()); + } + return benchmark_result; +} + +/** + * Returns a copy of the input spec with default values inserted. Avoids overriding pre-set values + * in the original spec. + * + * @param original_spec Valid adaptive load session spec. + * + * @return Adaptive load session spec with default values inserted. + */ +AdaptiveLoadSessionSpec SetDefaults(const AdaptiveLoadSessionSpec& original_spec) { + AdaptiveLoadSessionSpec spec = original_spec; + if (!spec.nighthawk_traffic_template().has_open_loop()) { + spec.mutable_nighthawk_traffic_template()->mutable_open_loop()->set_value(true); + } + if (!spec.has_measuring_period()) { + spec.mutable_measuring_period()->set_seconds(10); + } + if (!spec.has_convergence_deadline()) { + spec.mutable_convergence_deadline()->set_seconds(300); + } + if (!spec.has_testing_stage_duration()) { + spec.mutable_testing_stage_duration()->set_seconds(30); + } + for (nighthawk::adaptive_load::MetricSpecWithThreshold& threshold : + *spec.mutable_metric_thresholds()) { + if (threshold.metric_spec().metrics_plugin_name().empty()) { + threshold.mutable_metric_spec()->set_metrics_plugin_name("nighthawk.builtin"); + } + if (!threshold.threshold_spec().has_weight()) { + threshold.mutable_threshold_spec()->mutable_weight()->set_value(1.0); + } + } + for (nighthawk::adaptive_load::MetricSpec& metric_spec : + *spec.mutable_informational_metric_specs()) { + if (metric_spec.metrics_plugin_name().empty()) { + metric_spec.set_metrics_plugin_name("nighthawk.builtin"); + } + } + return spec; +} + +/** + * Checks whether a session spec is valid: No forbidden fields in Nighthawk traffic spec; no bad + * plugin references or bad plugin configurations (step controller, metric, scoring function); no + * nonexistent metric names. Reports all errors in one pass. + * + * @param spec A potentially invalid adaptive load session spec. + * + * @return Status OK if no problems were found, or InvalidArgument with all errors. + */ +absl::Status CheckSessionSpec(const nighthawk::adaptive_load::AdaptiveLoadSessionSpec& spec) { + std::vector errors; + if (spec.nighthawk_traffic_template().has_duration()) { + errors.emplace_back( + "nighthawk_traffic_template should not have |duration| set. Set |measuring_period| " + "and |testing_stage_duration| in the AdaptiveLoadSessionSpec proto instead."); + } + absl::flat_hash_map plugin_from_name; + std::vector plugin_names = {"nighthawk.builtin"}; + plugin_from_name["nighthawk.builtin"] = + std::make_unique(nighthawk::client::Output()); + for (const envoy::config::core::v3::TypedExtensionConfig& config : + spec.metrics_plugin_configs()) { + plugin_names.push_back(config.name()); + absl::StatusOr metrics_plugin_or = LoadMetricsPlugin(config); + if (!metrics_plugin_or.ok()) { + errors.emplace_back( + absl::StrCat("Failed to load MetricsPlugin: ", metrics_plugin_or.status().message())); + continue; + } + plugin_from_name[config.name()] = std::move(metrics_plugin_or.value()); + } + absl::StatusOr step_controller_or = + LoadStepControllerPlugin(spec.step_controller_config(), spec.nighthawk_traffic_template()); + if (!step_controller_or.ok()) { + errors.emplace_back(absl::StrCat("Failed to load StepController plugin: ", + step_controller_or.status().message())); + } + std::vector all_metric_specs; + for (const MetricSpecWithThreshold& metric_threshold : spec.metric_thresholds()) { + all_metric_specs.push_back(metric_threshold.metric_spec()); + absl::StatusOr scoring_function_or = + LoadScoringFunctionPlugin(metric_threshold.threshold_spec().scoring_function()); + if (!scoring_function_or.ok()) { + errors.emplace_back(absl::StrCat("Failed to load ScoringFunction plugin: ", + scoring_function_or.status().message())); + } + } + for (const MetricSpec& metric_spec : spec.informational_metric_specs()) { + all_metric_specs.push_back(metric_spec); + } + for (const MetricSpec& metric_spec : all_metric_specs) { + if (plugin_from_name.contains(metric_spec.metrics_plugin_name())) { + std::vector supported_metrics = + plugin_from_name[metric_spec.metrics_plugin_name()]->GetAllSupportedMetricNames(); + if (std::find(supported_metrics.begin(), supported_metrics.end(), + metric_spec.metric_name()) == supported_metrics.end()) { + errors.emplace_back( + absl::StrCat("Metric named '", metric_spec.metric_name(), + "' not implemented by plugin '", metric_spec.metrics_plugin_name(), + "'. Metrics implemented: ", absl::StrJoin(supported_metrics, ", "), ".")); + } + } else { + errors.emplace_back(absl::StrCat( + "MetricSpec referred to nonexistent metrics_plugin_name '", + metric_spec.metrics_plugin_name(), + "'. You must declare the plugin in metrics_plugin_configs or use plugin ", + "'nighthawk.builtin'. Available plugins: ", absl::StrJoin(plugin_names, ", "), ".")); + } + } + if (errors.size() > 0) { + return absl::InvalidArgumentError(absl::StrJoin(errors, "\n")); + } + return absl::OkStatus(); +} + +} // namespace + +absl::StatusOr PerformAdaptiveLoadSession( + nighthawk::client::NighthawkService::StubInterface* nighthawk_service_stub, + const AdaptiveLoadSessionSpec& input_spec, Envoy::TimeSource& time_source) { + AdaptiveLoadSessionOutput output; + + AdaptiveLoadSessionSpec spec = SetDefaults(input_spec); + absl::Status validation_status = CheckSessionSpec(spec); + if (!validation_status.ok()) { + return validation_status; + } + + absl::flat_hash_map name_to_custom_metrics_plugin_map; + for (const envoy::config::core::v3::TypedExtensionConfig& config : + spec.metrics_plugin_configs()) { + absl::StatusOr metrics_plugin_or = LoadMetricsPlugin(config); + RELEASE_ASSERT( + metrics_plugin_or.ok(), + absl::StrCat( + "MetricsPlugin loading error should have been caught during input validation: ", + metrics_plugin_or.status().message())); + name_to_custom_metrics_plugin_map[config.name()] = std::move(metrics_plugin_or.value()); + } + + absl::StatusOr step_controller_or = + LoadStepControllerPlugin(spec.step_controller_config(), spec.nighthawk_traffic_template()); + RELEASE_ASSERT( + step_controller_or.ok(), + absl::StrCat( + "StepController plugin loading error should have been caught during input validation: ", + step_controller_or.status().message())); + StepControllerPtr step_controller = std::move(step_controller_or.value()); + + for (const nighthawk::adaptive_load::MetricSpecWithThreshold& threshold : + spec.metric_thresholds()) { + *output.mutable_metric_thresholds()->Add() = threshold; + } + + Envoy::MonotonicTime start_time = time_source.monotonicTime(); + std::string doom_reason; + do { + if (std::chrono::duration_cast(time_source.monotonicTime() - start_time) + .count() > spec.convergence_deadline().seconds()) { + return absl::DeadlineExceededError(absl::StrCat("Failed to converge before deadline of ", + spec.convergence_deadline().seconds(), + " seconds.")); + } + + absl::StatusOr command_line_options_or = + step_controller->GetCurrentCommandLineOptions(); + if (!command_line_options_or.ok()) { + ENVOY_LOG_MISC(error, command_line_options_or.status().message()); + return absl::AbortedError(absl::StrCat("Error constructing Nighthawk input: ", + command_line_options_or.status().message())); + } + nighthawk::client::CommandLineOptions command_line_options = command_line_options_or.value(); + + ENVOY_LOG_MISC(info, "Adjusting Stage: Trying load: {}", command_line_options.DebugString()); + absl::StatusOr result_or = PerformAndAnalyzeNighthawkBenchmark( + nighthawk_service_stub, spec, name_to_custom_metrics_plugin_map, command_line_options, + spec.measuring_period()); + if (!result_or.ok()) { + return result_or.status(); + } + BenchmarkResult result = result_or.value(); + *output.mutable_adjusting_stage_results()->Add() = result; + step_controller->UpdateAndRecompute(result); + } while (!step_controller->IsConverged() && !step_controller->IsDoomed(doom_reason)); + + if (step_controller->IsDoomed(doom_reason)) { + return absl::AbortedError( + absl::StrCat("Step controller determined that it can never converge: ", doom_reason)); + } + + absl::StatusOr command_line_options_or = + step_controller->GetCurrentCommandLineOptions(); + if (!command_line_options_or.ok()) { + ENVOY_LOG_MISC(error, command_line_options_or.status().message()); + return absl::AbortedError(absl::StrCat("Error constructing Nighthawk input: ", + command_line_options_or.status().message())); + } + nighthawk::client::CommandLineOptions command_line_options = command_line_options_or.value(); + + ENVOY_LOG_MISC(info, "Testing Stage with load: {}", command_line_options.DebugString()); + absl::StatusOr result_or = PerformAndAnalyzeNighthawkBenchmark( + nighthawk_service_stub, spec, name_to_custom_metrics_plugin_map, command_line_options, + spec.testing_stage_duration()); + if (!result_or.ok()) { + return result_or.status(); + } + *output.mutable_testing_stage_result() = result_or.value(); + return output; +} + +} // namespace Nighthawk diff --git a/source/adaptive_load/step_controller_impl.cc b/source/adaptive_load/step_controller_impl.cc index 7ade8082a..94ab8bcac 100644 --- a/source/adaptive_load/step_controller_impl.cc +++ b/source/adaptive_load/step_controller_impl.cc @@ -116,10 +116,6 @@ bool ExponentialSearchStepController::IsDoomed(std::string& doom_reason) const { } void ExponentialSearchStepController::UpdateAndRecompute(const BenchmarkResult& benchmark_result) { - if (benchmark_result.status().code()) { - doom_reason_ = "Nighthawk Service returned an error."; - return; - } const double score = TotalScore(benchmark_result); if (is_range_finding_phase_) { IterateRangeFindingPhase(score); diff --git a/test/adaptive_load/BUILD b/test/adaptive_load/BUILD index ff10152fe..f7cdd3544 100644 --- a/test/adaptive_load/BUILD +++ b/test/adaptive_load/BUILD @@ -9,6 +9,17 @@ licenses(["notice"]) # Apache 2 envoy_package() +envoy_cc_test_library( + name = "fake_time_source", + srcs = ["fake_time_source.cc"], + hdrs = ["fake_time_source.h"], + repository = "@envoy", + deps = [ + "//source/client:nighthawk_client_lib", + "@com_google_absl//absl/time", + ], +) + envoy_cc_test_library( name = "minimal_output", srcs = ["minimal_output.cc"], @@ -21,6 +32,22 @@ envoy_cc_test_library( ], ) +envoy_cc_test( + name = "adaptive_load_controller_test", + srcs = ["adaptive_load_controller_test.cc"], + repository = "@envoy", + deps = [ + ":fake_time_source", + ":minimal_output", + "//source/adaptive_load:adaptive_load_controller_impl", + "//source/adaptive_load:scoring_function_impl", + "//source/client:nighthawk_client_lib", + "//test/adaptive_load/fake_plugins/fake_metrics_plugin", + "//test/adaptive_load/fake_plugins/fake_step_controller", + "@com_github_grpc_grpc//:grpc++_test", + ], +) + envoy_cc_test( name = "input_variable_setter_test", srcs = ["input_variable_setter_test.cc"], diff --git a/test/adaptive_load/adaptive_load_controller_test.cc b/test/adaptive_load/adaptive_load_controller_test.cc new file mode 100644 index 000000000..71aadce68 --- /dev/null +++ b/test/adaptive_load/adaptive_load_controller_test.cc @@ -0,0 +1,1039 @@ +#include + +#include "envoy/config/core/v3/base.pb.h" +#include "envoy/registry/registry.h" + +#include "nighthawk/adaptive_load/adaptive_load_controller.h" +#include "nighthawk/adaptive_load/input_variable_setter.h" +#include "nighthawk/adaptive_load/metrics_plugin.h" +#include "nighthawk/adaptive_load/scoring_function.h" +#include "nighthawk/adaptive_load/step_controller.h" + +#include "external/envoy/source/common/common/statusor.h" +#include "external/envoy/source/common/config/utility.h" +#include "external/envoy/source/common/protobuf/protobuf.h" + +#include "api/adaptive_load/adaptive_load.pb.h" +#include "api/adaptive_load/benchmark_result.pb.h" +#include "api/adaptive_load/input_variable_setter_impl.pb.h" +#include "api/adaptive_load/metric_spec.pb.h" +#include "api/adaptive_load/metrics_plugin_impl.pb.h" +#include "api/adaptive_load/scoring_function_impl.pb.h" +#include "api/adaptive_load/step_controller_impl.pb.h" +#include "api/client/options.pb.h" +#include "api/client/output.pb.h" +#include "api/client/service.grpc.pb.h" +#include "api/client/service.pb.h" +#include "api/client/service_mock.grpc.pb.h" + +#include "grpcpp/test/mock_stream.h" + +#include "test/adaptive_load/fake_plugins/fake_metrics_plugin/fake_metrics_plugin.h" +#include "test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.h" +#include "test/adaptive_load/fake_time_source.h" +#include "test/adaptive_load/minimal_output.h" + +#include "absl/container/flat_hash_map.h" +#include "absl/status/status.h" +#include "absl/strings/str_join.h" +#include "adaptive_load/metrics_plugin_impl.h" +#include "adaptive_load/plugin_loader.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Nighthawk { + +namespace { + +using ::nighthawk::adaptive_load::AdaptiveLoadSessionOutput; +using ::testing::_; +using ::testing::DoAll; +using ::testing::Eq; +using ::testing::HasSubstr; +using ::testing::Return; +using ::testing::SetArgPointee; + +/** + * Creates a valid TypedExtensionConfig proto selecting the real BinaryScoringFunction plugin + * and configuring it with a threshold. + * + * @param lower_threshold Threshold value to set within the config proto. + * + * @return TypedExtensionConfig Full scoring function plugin spec that selects + * nighthawk.binary-scoring and provides a config. + */ +envoy::config::core::v3::TypedExtensionConfig +MakeLowerThresholdBinaryScoringFunctionConfig(double lower_threshold) { + envoy::config::core::v3::TypedExtensionConfig config; + config.set_name("nighthawk.binary_scoring"); + nighthawk::adaptive_load::BinaryScoringFunctionConfig inner_config; + inner_config.mutable_lower_threshold()->set_value(lower_threshold); + config.mutable_typed_config()->PackFrom(inner_config); + return config; +} + +/** + * Creates a valid TypedExtensionConfig proto selecting the real LinearScoringFunction plugin + * and configuring it with a threshold. The scaling constant is set so that metric values below the + * threshold produce negative values and values above the threshold produce positive values. + * + * @param threshold Threshold value to set within the config proto. + * + * @return TypedExtensionConfig Full scoring function plugin spec that selects + * nighthawk.linear-scoring and provides a config. + */ +envoy::config::core::v3::TypedExtensionConfig MakeLinearScoringFunctionConfig(double threshold) { + envoy::config::core::v3::TypedExtensionConfig config; + config.set_name("nighthawk.linear_scoring"); + nighthawk::adaptive_load::LinearScoringFunctionConfig inner_config; + inner_config.set_threshold(threshold); + inner_config.set_scaling_constant(-1.0); + config.mutable_typed_config()->PackFrom(inner_config); + return config; +} + +/** + * Creates a session spec with BuiltinMetricsPlugin configured to collect send-rate, a + * LinearScoringFunction with a threshold of 0.5, and a FakeStepController, which + * will report convergence whenever the metric score of a mock Nighthawk Service response receives + * a positive score, doom whenever it receives a negative score. This session spec may be amended or + * overwritten. + * + * @return AdaptiveLoadSessionSpec Adaptive load session spec proto. + */ +nighthawk::adaptive_load::AdaptiveLoadSessionSpec MakeConvergeableDoomableSessionSpec() { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec; + spec.mutable_nighthawk_traffic_template(); + *spec.mutable_step_controller_config() = + MakeFakeStepControllerPluginConfig(/*fixed_rps_value=*/1024); + nighthawk::adaptive_load::MetricSpecWithThreshold* threshold = + spec.mutable_metric_thresholds()->Add(); + threshold->mutable_metric_spec()->set_metric_name("send-rate"); + threshold->mutable_metric_spec()->set_metrics_plugin_name("nighthawk.builtin"); + *threshold->mutable_threshold_spec()->mutable_scoring_function() = + MakeLinearScoringFunctionConfig(0.5); + return spec; +} + +envoy::config::core::v3::TypedExtensionConfig MakeFakeMetricsPluginConfig() { + envoy::config::core::v3::TypedExtensionConfig config; + config.set_name("nighthawk.fake_metrics_plugin"); + nighthawk::adaptive_load::FakeMetricsPluginConfig inner_config; + nighthawk::adaptive_load::FakeMetricsPluginConfig::FakeMetric* fake_metric_good = + inner_config.mutable_fake_metrics()->Add(); + fake_metric_good->set_name("good_metric"); + fake_metric_good->set_value(5.0); + nighthawk::adaptive_load::FakeMetricsPluginConfig::FakeMetric* fake_metric_bad = + inner_config.mutable_fake_metrics()->Add(); + fake_metric_bad->set_name("bad_metric"); + fake_metric_bad->mutable_error_status()->set_code(::grpc::INTERNAL); + fake_metric_bad->mutable_error_status()->set_message("bad_metric simulated error"); + config.mutable_typed_config()->PackFrom(inner_config); + return config; +} + +/** + * Creates a simulated Nighthawk Service response that reflects the specified send rate. Combined + * with BuiltinMetricsPlugin and LinearScoringFunction with a lower threshold, this can be used to + * produce a 'send-rate' metric score that is positive, zero, or negative on demand. This in turn + * can be used to make FakeStepController report convergence or doom for testing purposes, by + * arrranging a positive or negative metric score. + * + * For example, use a LinearScoringFunction threshold of 0.5 and the response + * MakeNighthawkResponseWithSendRate(1.0) to produce a positive score (converging), + * MakeNighthawkResponseWithSendRate(0.25) to produce a negative score (doomed), and + * MakeNighthawkResponseWithSendRate(0.5) to produce a zero score (non-converging, non-doomed). + * + * @return ExecutionResponse A simulated Nighthawk Service response with counters representing the + * specified send rate, along with other dummy counters and stats. + */ +nighthawk::client::ExecutionResponse MakeNighthawkResponseWithSendRate(double send_rate) { + nighthawk::client::ExecutionResponse response; + nighthawk::client::Output output = MakeSimpleNighthawkOutput({ + /*concurrency=*/"auto", + /*requests_per_second=*/1024, + /*actual_duration_seconds=*/10, + /*upstream_rq_total=*/static_cast(10 * 1024 * send_rate), + /*response_count_2xx=*/320, + /*min_ns=*/400, + /*mean_ns=*/500, + /*max_ns=*/600, + /*pstdev_ns=*/11, + }); + *response.mutable_output() = output; + return response; +} + +/** + * Sets up a mock gRPC reader-writer to be returned from the mock Nighthawk Service stub. The mock + * Read() returns a non-converging, non-doomed Nighthawk response. + * + * @return Bare pointer that will be automatically wrapped in a unique_ptr by the caller. + */ +grpc::testing::MockClientReaderWriter* +MakeNonConvergingMockClientReaderWriter() { + auto* mock_reader_writer = + new grpc::testing::MockClientReaderWriter(); + EXPECT_CALL(*mock_reader_writer, Read(_)) + .WillOnce(DoAll(SetArgPointee<0>(MakeNighthawkResponseWithSendRate(0.5)), Return(true))) + .WillRepeatedly(Return(false)); + EXPECT_CALL(*mock_reader_writer, Write(_, _)).WillRepeatedly(Return(true)); + EXPECT_CALL(*mock_reader_writer, WritesDone()).WillRepeatedly(Return(true)); + EXPECT_CALL(*mock_reader_writer, Finish()).WillRepeatedly(Return(::grpc::Status::OK)); + return mock_reader_writer; +} + +/** + * Sets up a mock gRPC reader-writer to be returned from the mock Nighthawk Service stub. The mock + * Read() returns a converging Nighthawk response. + * + * @return Bare pointer that will be automatically wrapped in a unique_ptr by the caller. + */ +grpc::testing::MockClientReaderWriter* +MakeConvergingMockClientReaderWriter() { + auto* mock_reader_writer = + new grpc::testing::MockClientReaderWriter(); + EXPECT_CALL(*mock_reader_writer, Read(_)) + .WillOnce(DoAll(SetArgPointee<0>(MakeNighthawkResponseWithSendRate(1.0)), Return(true))) + .WillRepeatedly(Return(false)); + EXPECT_CALL(*mock_reader_writer, Write(_, _)).WillRepeatedly(Return(true)); + EXPECT_CALL(*mock_reader_writer, WritesDone()).WillRepeatedly(Return(true)); + EXPECT_CALL(*mock_reader_writer, Finish()).WillRepeatedly(Return(::grpc::Status::OK)); + return mock_reader_writer; +} + +/** + * Sets up a mock gRPC reader-writer to be returned from the mock Nighthawk Service stub. The mock + * Read() returns a doomed Nighthawk response. + * + * @return Bare pointer that will be automatically wrapped in a unique_ptr by the caller. + */ +grpc::testing::MockClientReaderWriter* +MakeDoomedMockClientReaderWriter() { + auto* mock_reader_writer = + new grpc::testing::MockClientReaderWriter(); + EXPECT_CALL(*mock_reader_writer, Read(_)) + .WillOnce(DoAll(SetArgPointee<0>(MakeNighthawkResponseWithSendRate(0.25)), Return(true))) + .WillRepeatedly(Return(false)); + EXPECT_CALL(*mock_reader_writer, Write(_, _)).WillRepeatedly(Return(true)); + EXPECT_CALL(*mock_reader_writer, WritesDone()).WillRepeatedly(Return(true)); + EXPECT_CALL(*mock_reader_writer, Finish()).WillRepeatedly(Return(::grpc::Status::OK)); + return mock_reader_writer; +} + +TEST(AdaptiveLoadController, FailsWithTrafficTemplateDurationSet) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec; + spec.mutable_nighthawk_traffic_template()->mutable_duration()->set_seconds(1); + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = PerformAdaptiveLoadSession( + /*nighthawk_service_stub=*/nullptr, spec, time_source); + ASSERT_FALSE(output_or.ok()); + EXPECT_EQ(output_or.status().code(), absl::StatusCode::kInvalidArgument); + EXPECT_THAT(output_or.status().message(), HasSubstr("should not have |duration| set")); +} + +TEST(AdaptiveLoadController, FailsWithNonexistentMetricsPluginName) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec; + envoy::config::core::v3::TypedExtensionConfig* metrics_plugin_config = + spec.mutable_metrics_plugin_configs()->Add(); + metrics_plugin_config->set_name("nonexistent-plugin"); + *metrics_plugin_config->mutable_typed_config() = Envoy::ProtobufWkt::Any(); + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = PerformAdaptiveLoadSession( + /*nighthawk_service_stub=*/nullptr, spec, time_source); + ASSERT_FALSE(output_or.ok()); + EXPECT_EQ(output_or.status().code(), absl::StatusCode::kInvalidArgument); + EXPECT_THAT(output_or.status().message(), HasSubstr("Failed to load MetricsPlugin")); +} + +TEST(AdaptiveLoadController, FailsWithNonexistentStepControllerPluginName) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec; + envoy::config::core::v3::TypedExtensionConfig config; + config.set_name("nonexistent-plugin"); + *config.mutable_typed_config() = Envoy::ProtobufWkt::Any(); + *spec.mutable_step_controller_config() = config; + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = PerformAdaptiveLoadSession( + /*nighthawk_service_stub=*/nullptr, spec, time_source); + ASSERT_FALSE(output_or.ok()); + EXPECT_EQ(output_or.status().code(), absl::StatusCode::kInvalidArgument); + EXPECT_THAT(output_or.status().message(), HasSubstr("Failed to load StepController")); +} + +TEST(AdaptiveLoadController, FailsWithNonexistentScoringFunctionPluginName) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec; + nighthawk::adaptive_load::MetricSpecWithThreshold* threshold = + spec.mutable_metric_thresholds()->Add(); + envoy::config::core::v3::TypedExtensionConfig scoring_function_config; + scoring_function_config.set_name("nonexistent-scoring-function"); + *scoring_function_config.mutable_typed_config() = Envoy::ProtobufWkt::Any(); + *threshold->mutable_threshold_spec()->mutable_scoring_function() = scoring_function_config; + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = PerformAdaptiveLoadSession( + /*nighthawk_service_stub=*/nullptr, spec, time_source); + ASSERT_FALSE(output_or.ok()); + EXPECT_EQ(output_or.status().code(), absl::StatusCode::kInvalidArgument); + EXPECT_THAT(output_or.status().message(), HasSubstr("Failed to load ScoringFunction")); +} + +TEST(AdaptiveLoadController, FailsWithNonexistentMetricsPluginNameInMetricThresholdSpec) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec; + nighthawk::adaptive_load::MetricSpecWithThreshold* threshold = + spec.mutable_metric_thresholds()->Add(); + *threshold->mutable_threshold_spec()->mutable_scoring_function() = + MakeLowerThresholdBinaryScoringFunctionConfig(0.0); + threshold->mutable_metric_spec()->set_metric_name("x"); + threshold->mutable_metric_spec()->set_metrics_plugin_name("nonexistent-metrics-plugin"); + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = PerformAdaptiveLoadSession( + /*nighthawk_service_stub=*/nullptr, spec, time_source); + ASSERT_FALSE(output_or.ok()); + EXPECT_EQ(output_or.status().code(), absl::StatusCode::kInvalidArgument); + EXPECT_THAT(output_or.status().message(), HasSubstr("nonexistent metrics_plugin_name")); +} + +TEST(AdaptiveLoadController, FailsWithUndeclaredMetricsPluginNameInMetricThresholdSpec) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec; + nighthawk::adaptive_load::MetricSpecWithThreshold* threshold = + spec.mutable_metric_thresholds()->Add(); + *threshold->mutable_threshold_spec()->mutable_scoring_function() = + MakeLowerThresholdBinaryScoringFunctionConfig(0.0); + threshold->mutable_metric_spec()->set_metric_name("x"); + // Valid plugin name, but plugin not declared in the spec. + threshold->mutable_metric_spec()->set_metrics_plugin_name("nighthawk.fake_metrics_plugin"); + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = PerformAdaptiveLoadSession( + /*nighthawk_service_stub=*/nullptr, spec, time_source); + ASSERT_FALSE(output_or.ok()); + EXPECT_EQ(output_or.status().code(), absl::StatusCode::kInvalidArgument); + EXPECT_THAT(output_or.status().message(), HasSubstr("nonexistent metrics_plugin_name")); +} + +TEST(AdaptiveLoadController, FailsWithNonexistentMetricsPluginNameInInformationalMetricSpec) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec; + nighthawk::adaptive_load::MetricSpec* metric_spec = + spec.mutable_informational_metric_specs()->Add(); + metric_spec->set_metric_name("x"); + metric_spec->set_metrics_plugin_name("nonexistent-metrics-plugin"); + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = PerformAdaptiveLoadSession( + /*nighthawk_service_stub=*/nullptr, spec, time_source); + ASSERT_FALSE(output_or.ok()); + EXPECT_EQ(output_or.status().code(), absl::StatusCode::kInvalidArgument); + EXPECT_THAT(output_or.status().message(), HasSubstr("nonexistent metrics_plugin_name")); +} + +TEST(AdaptiveLoadController, FailsWithUndeclaredMetricsPluginNameInInformationalMetricSpec) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec; + nighthawk::adaptive_load::MetricSpec* metric_spec = + spec.mutable_informational_metric_specs()->Add(); + metric_spec->set_metric_name("x"); + // Valid plugin name, but plugin not declared in the spec. + metric_spec->set_metrics_plugin_name("nighthawk.fake_metrics_plugin"); + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = PerformAdaptiveLoadSession( + /*nighthawk_service_stub=*/nullptr, spec, time_source); + ASSERT_FALSE(output_or.ok()); + EXPECT_EQ(output_or.status().code(), absl::StatusCode::kInvalidArgument); + EXPECT_THAT(output_or.status().message(), HasSubstr("nonexistent metrics_plugin_name")); +} + +TEST(AdaptiveLoadController, FailsWithNonexistentBuiltinMetricNameInMetricThresholdSpec) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec; + nighthawk::adaptive_load::MetricSpecWithThreshold* threshold = + spec.mutable_metric_thresholds()->Add(); + *threshold->mutable_threshold_spec()->mutable_scoring_function() = + MakeLowerThresholdBinaryScoringFunctionConfig(0.0); + threshold->mutable_metric_spec()->set_metric_name("nonexistent-metric-name"); + threshold->mutable_metric_spec()->set_metrics_plugin_name("nighthawk.builtin"); + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = PerformAdaptiveLoadSession( + /*nighthawk_service_stub=*/nullptr, spec, time_source); + ASSERT_FALSE(output_or.ok()); + EXPECT_EQ(output_or.status().code(), absl::StatusCode::kInvalidArgument); + EXPECT_THAT(output_or.status().message(), HasSubstr("not implemented by plugin")); +} + +TEST(AdaptiveLoadController, FailsWithNonexistentCustomMetricNameInMetricThresholdSpec) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec; + *spec.mutable_metrics_plugin_configs()->Add() = MakeFakeMetricsPluginConfig(); + nighthawk::adaptive_load::MetricSpecWithThreshold* threshold = + spec.mutable_metric_thresholds()->Add(); + *threshold->mutable_threshold_spec()->mutable_scoring_function() = + MakeLowerThresholdBinaryScoringFunctionConfig(0.0); + threshold->mutable_metric_spec()->set_metric_name("nonexistent-metric-name"); + threshold->mutable_metric_spec()->set_metrics_plugin_name("nighthawk.fake_metrics_plugin"); + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = PerformAdaptiveLoadSession( + /*nighthawk_service_stub=*/nullptr, spec, time_source); + ASSERT_FALSE(output_or.ok()); + EXPECT_EQ(output_or.status().code(), absl::StatusCode::kInvalidArgument); + EXPECT_THAT(output_or.status().message(), HasSubstr("not implemented by plugin")); +} + +TEST(AdaptiveLoadController, FailsWithNonexistentBuiltinMetricNameInInformationalMetricSpec) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec; + nighthawk::adaptive_load::MetricSpec* metric_spec = + spec.mutable_informational_metric_specs()->Add(); + metric_spec->set_metric_name("nonexistent-metric-name"); + metric_spec->set_metrics_plugin_name("nighthawk.builtin"); + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = PerformAdaptiveLoadSession( + /*nighthawk_service_stub=*/nullptr, spec, time_source); + ASSERT_FALSE(output_or.ok()); + EXPECT_EQ(output_or.status().code(), absl::StatusCode::kInvalidArgument); + EXPECT_THAT(output_or.status().message(), HasSubstr("not implemented by plugin")); +} + +TEST(AdaptiveLoadController, FailsWithNonexistentCustomMetricNameInInformationalMetricSpec) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec; + *spec.mutable_metrics_plugin_configs()->Add() = MakeFakeMetricsPluginConfig(); + nighthawk::adaptive_load::MetricSpec* metric_spec = + spec.mutable_informational_metric_specs()->Add(); + metric_spec->set_metric_name("nonexistent-metric-name"); + metric_spec->set_metrics_plugin_name("nighthawk.fake_metrics_plugin"); + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = PerformAdaptiveLoadSession( + /*nighthawk_service_stub=*/nullptr, spec, time_source); + ASSERT_FALSE(output_or.ok()); + EXPECT_EQ(output_or.status().code(), absl::StatusCode::kInvalidArgument); + EXPECT_THAT(output_or.status().message(), HasSubstr("not implemented by plugin")); +} + +TEST(AdaptiveLoadController, TimesOutIfNeverConverged) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec = MakeConvergeableDoomableSessionSpec(); + + nighthawk::client::MockNighthawkServiceStub mock_nighthawk_service_stub; + EXPECT_CALL(mock_nighthawk_service_stub, ExecutionStreamRaw) + .WillRepeatedly( + [](grpc_impl::ClientContext*) { return MakeNonConvergingMockClientReaderWriter(); }); + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = + PerformAdaptiveLoadSession(&mock_nighthawk_service_stub, spec, time_source); + ASSERT_FALSE(output_or.ok()); + EXPECT_EQ(output_or.status().code(), absl::StatusCode::kDeadlineExceeded); + EXPECT_THAT(output_or.status().message(), HasSubstr("Failed to converge before deadline")); +} + +TEST(AdaptiveLoadController, UsesDefaultConvergenceDeadline) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec = MakeConvergeableDoomableSessionSpec(); + + // Not setting convergence deadline, should default to 300 seconds. + nighthawk::client::MockNighthawkServiceStub mock_nighthawk_service_stub; + EXPECT_CALL(mock_nighthawk_service_stub, ExecutionStreamRaw) + .WillRepeatedly( + [](grpc_impl::ClientContext*) { return MakeNonConvergingMockClientReaderWriter(); }); + FakeIncrementingMonotonicTimeSource time_source; + Envoy::MonotonicTime start_time = time_source.monotonicTime(); + absl::StatusOr output_or = + PerformAdaptiveLoadSession(&mock_nighthawk_service_stub, spec, time_source); + EXPECT_THAT( + std::chrono::duration_cast(time_source.monotonicTime() - start_time) + .count(), + 303); // 300 ticks plus 2 monotonicTime() calls here plus 1 within the controller when it + // recorded the start time. +} + +TEST(AdaptiveLoadController, UsesOpenLoopModeByDefault) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec = MakeConvergeableDoomableSessionSpec(); + + // Successively overwritten by each adjusting stage request and finally the testing stage request. + nighthawk::client::ExecutionRequest request; + nighthawk::client::MockNighthawkServiceStub mock_nighthawk_service_stub; + EXPECT_CALL(mock_nighthawk_service_stub, ExecutionStreamRaw) + .WillRepeatedly([&request](grpc_impl::ClientContext*) { + auto* mock_reader_writer = MakeConvergingMockClientReaderWriter(); + EXPECT_CALL(*mock_reader_writer, Write(_, _)) + .WillRepeatedly(::testing::DoAll(::testing::SaveArg<0>(&request), Return(true))); + return mock_reader_writer; + }); + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = + PerformAdaptiveLoadSession(&mock_nighthawk_service_stub, spec, time_source); + EXPECT_TRUE(request.start_request().options().open_loop().value()); +} + +TEST(AdaptiveLoadController, UsesExplicitOpenLoopValue) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec = MakeConvergeableDoomableSessionSpec(); + spec.mutable_nighthawk_traffic_template()->mutable_open_loop()->set_value(false); + + // Successively overwritten by each adjusting stage request and finally the testing stage request. + nighthawk::client::ExecutionRequest request; + nighthawk::client::MockNighthawkServiceStub mock_nighthawk_service_stub; + EXPECT_CALL(mock_nighthawk_service_stub, ExecutionStreamRaw) + .WillRepeatedly([&request](grpc_impl::ClientContext*) { + auto* mock_reader_writer = MakeConvergingMockClientReaderWriter(); + EXPECT_CALL(*mock_reader_writer, Write(_, _)) + .WillRepeatedly(::testing::DoAll(::testing::SaveArg<0>(&request), Return(true))); + return mock_reader_writer; + }); + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = + PerformAdaptiveLoadSession(&mock_nighthawk_service_stub, spec, time_source); + EXPECT_FALSE(request.start_request().options().open_loop().value()); +} + +TEST(AdaptiveLoadController, UsesConfiguredConvergenceDeadline) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec = MakeConvergeableDoomableSessionSpec(); + spec.mutable_convergence_deadline()->set_seconds(123); + + nighthawk::client::MockNighthawkServiceStub mock_nighthawk_service_stub; + EXPECT_CALL(mock_nighthawk_service_stub, ExecutionStreamRaw) + .WillRepeatedly( + [](grpc_impl::ClientContext*) { return MakeNonConvergingMockClientReaderWriter(); }); + FakeIncrementingMonotonicTimeSource time_source; + Envoy::MonotonicTime start_time = time_source.monotonicTime(); + absl::StatusOr output_or = + PerformAdaptiveLoadSession(&mock_nighthawk_service_stub, spec, time_source); + EXPECT_THAT( + std::chrono::duration_cast(time_source.monotonicTime() - start_time) + .count(), + 126); // 123 ticks plus 2 monotonicTime() calls here plus 1 within the controller when it + // recorded the start time. +} + +TEST(AdaptiveLoadController, UsesDefaultTestingStageDuration) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec = MakeConvergeableDoomableSessionSpec(); + + // Successively overwritten by each adjusting stage request and finally the testing stage request. + nighthawk::client::ExecutionRequest request; + nighthawk::client::MockNighthawkServiceStub mock_nighthawk_service_stub; + int convergence_countdown = 5; + EXPECT_CALL(mock_nighthawk_service_stub, ExecutionStreamRaw) + .WillRepeatedly([&request, &convergence_countdown](grpc_impl::ClientContext*) { + auto* mock_reader_writer = convergence_countdown > 0 + ? MakeNonConvergingMockClientReaderWriter() + : MakeConvergingMockClientReaderWriter(); + --convergence_countdown; + EXPECT_CALL(*mock_reader_writer, Write(_, _)) + .WillRepeatedly(::testing::DoAll(::testing::SaveArg<0>(&request), Return(true))); + return mock_reader_writer; + }); + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = + PerformAdaptiveLoadSession(&mock_nighthawk_service_stub, spec, time_source); + ASSERT_TRUE(output_or.ok()); + AdaptiveLoadSessionOutput output = output_or.value(); + ASSERT_TRUE(output.has_testing_stage_result()); + EXPECT_EQ(request.start_request().options().duration().seconds(), 30); +} + +TEST(AdaptiveLoadController, UsesDefaultMeasuringPeriod) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec = MakeConvergeableDoomableSessionSpec(); + + nighthawk::client::ExecutionRequest request; + nighthawk::client::MockNighthawkServiceStub mock_nighthawk_service_stub; + int convergence_countdown = 5; + EXPECT_CALL(mock_nighthawk_service_stub, ExecutionStreamRaw) + .WillRepeatedly([&request, &convergence_countdown](grpc_impl::ClientContext*) { + auto* mock_reader_writer = convergence_countdown > 0 + ? MakeNonConvergingMockClientReaderWriter() + : MakeConvergingMockClientReaderWriter(); + --convergence_countdown; + if (convergence_countdown > 0) { + // Ensure we only capture the request if it's the adjusting stage. + EXPECT_CALL(*mock_reader_writer, Write(_, _)) + .WillRepeatedly(::testing::DoAll(::testing::SaveArg<0>(&request), Return(true))); + } + return mock_reader_writer; + }); + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = + PerformAdaptiveLoadSession(&mock_nighthawk_service_stub, spec, time_source); + ASSERT_TRUE(output_or.ok()); + AdaptiveLoadSessionOutput output = output_or.value(); + EXPECT_EQ(request.start_request().options().duration().seconds(), 10); +} + +TEST(AdaptiveLoadController, UsesConfiguredMeasuringPeriod) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec = MakeConvergeableDoomableSessionSpec(); + spec.mutable_measuring_period()->set_seconds(17); + + // Values to trigger convergence are not set up in the mock Nighthawk Service. + + nighthawk::client::ExecutionRequest request; + nighthawk::client::MockNighthawkServiceStub mock_nighthawk_service_stub; + int convergence_countdown = 5; + EXPECT_CALL(mock_nighthawk_service_stub, ExecutionStreamRaw) + .WillRepeatedly([&request, &convergence_countdown](grpc_impl::ClientContext*) { + auto* mock_reader_writer = convergence_countdown > 0 + ? MakeNonConvergingMockClientReaderWriter() + : MakeConvergingMockClientReaderWriter(); + --convergence_countdown; + if (convergence_countdown > 0) { + // Ensure we only capture the request if it's the adjusting stage. + EXPECT_CALL(*mock_reader_writer, Write(_, _)) + .WillRepeatedly(::testing::DoAll(::testing::SaveArg<0>(&request), Return(true))); + } + return mock_reader_writer; + }); + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = + PerformAdaptiveLoadSession(&mock_nighthawk_service_stub, spec, time_source); + ASSERT_TRUE(output_or.ok()); + AdaptiveLoadSessionOutput output = output_or.value(); + EXPECT_EQ(request.start_request().options().duration().seconds(), 17); +} + +TEST(AdaptiveLoadController, UsesCommandLineOptionsFromController) { + // Always sends 1024 rps. + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec = MakeConvergeableDoomableSessionSpec(); + + nighthawk::client::ExecutionRequest request; + nighthawk::client::MockNighthawkServiceStub mock_nighthawk_service_stub; + EXPECT_CALL(mock_nighthawk_service_stub, ExecutionStreamRaw) + .WillRepeatedly([&request](grpc_impl::ClientContext*) { + auto* mock_reader_writer = MakeConvergingMockClientReaderWriter(); + EXPECT_CALL(*mock_reader_writer, Write(_, _)) + .WillRepeatedly(::testing::DoAll(::testing::SaveArg<0>(&request), Return(true))); + return mock_reader_writer; + }); + + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = + PerformAdaptiveLoadSession(&mock_nighthawk_service_stub, spec, time_source); + ASSERT_TRUE(output_or.ok()); + AdaptiveLoadSessionOutput output = output_or.value(); + EXPECT_EQ(request.start_request().options().requests_per_second().value(), 1024); +} + +TEST(AdaptiveLoadController, UsesDefaultMetricWeight) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec = MakeConvergeableDoomableSessionSpec(); + *spec.mutable_metrics_plugin_configs()->Add() = MakeFakeMetricsPluginConfig(); + + nighthawk::adaptive_load::MetricSpecWithThreshold* threshold = + spec.mutable_metric_thresholds()->Add(); + threshold->mutable_metric_spec()->set_metric_name("good_metric"); + threshold->mutable_metric_spec()->set_metrics_plugin_name("nighthawk.fake_metrics_plugin"); + *threshold->mutable_threshold_spec()->mutable_scoring_function() = + MakeLowerThresholdBinaryScoringFunctionConfig(0.0); + + nighthawk::client::MockNighthawkServiceStub mock_nighthawk_service_stub; + EXPECT_CALL(mock_nighthawk_service_stub, ExecutionStreamRaw) + .WillRepeatedly( + [](grpc_impl::ClientContext*) { return MakeConvergingMockClientReaderWriter(); }); + + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = + PerformAdaptiveLoadSession(&mock_nighthawk_service_stub, spec, time_source); + ASSERT_TRUE(output_or.ok()); + AdaptiveLoadSessionOutput output = output_or.value(); + ASSERT_GT(output.adjusting_stage_results_size(), 0); + ASSERT_GT(output.adjusting_stage_results()[0].metric_evaluations_size(), 1); + EXPECT_EQ(output.adjusting_stage_results()[0].metric_evaluations()[1].weight(), 1.0); +} + +TEST(AdaptiveLoadController, UsesCustomMetricWeight) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec = MakeConvergeableDoomableSessionSpec(); + *spec.mutable_metrics_plugin_configs()->Add() = MakeFakeMetricsPluginConfig(); + + nighthawk::adaptive_load::MetricSpecWithThreshold* threshold = + spec.mutable_metric_thresholds()->Add(); + threshold->mutable_metric_spec()->set_metric_name("good_metric"); + threshold->mutable_metric_spec()->set_metrics_plugin_name("nighthawk.fake_metrics_plugin"); + *threshold->mutable_threshold_spec()->mutable_scoring_function() = + MakeLowerThresholdBinaryScoringFunctionConfig(0.0); + threshold->mutable_threshold_spec()->mutable_weight()->set_value(45.0); + + nighthawk::client::MockNighthawkServiceStub mock_nighthawk_service_stub; + EXPECT_CALL(mock_nighthawk_service_stub, ExecutionStreamRaw) + .WillRepeatedly( + [](grpc_impl::ClientContext*) { return MakeConvergingMockClientReaderWriter(); }); + + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = + PerformAdaptiveLoadSession(&mock_nighthawk_service_stub, spec, time_source); + ASSERT_TRUE(output_or.ok()); + AdaptiveLoadSessionOutput output = output_or.value(); + ASSERT_GT(output.adjusting_stage_results_size(), 0); + ASSERT_GT(output.adjusting_stage_results()[0].metric_evaluations_size(), 1); + EXPECT_EQ(output.adjusting_stage_results()[0].metric_evaluations()[1].weight(), 45.0); +} + +TEST(AdaptiveLoadController, ExitsWhenDoomed) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec = MakeConvergeableDoomableSessionSpec(); + + nighthawk::client::MockNighthawkServiceStub mock_nighthawk_service_stub; + int doom_countdown = 3; + EXPECT_CALL(mock_nighthawk_service_stub, ExecutionStreamRaw) + .WillRepeatedly([&doom_countdown](grpc_impl::ClientContext*) { + if (doom_countdown > 0) { + --doom_countdown; + return MakeNonConvergingMockClientReaderWriter(); + } else { + return MakeDoomedMockClientReaderWriter(); + } + }); + + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = + PerformAdaptiveLoadSession(&mock_nighthawk_service_stub, spec, time_source); + ASSERT_FALSE(output_or.ok()); + + EXPECT_THAT(output_or.status().message(), + HasSubstr("Step controller determined that it can never converge")); + EXPECT_THAT(output_or.status().message(), HasSubstr("artificial doom")); +} + +TEST(AdaptiveLoadController, PerformsTestingStageAfterConvergence) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec = MakeConvergeableDoomableSessionSpec(); + + nighthawk::client::MockNighthawkServiceStub mock_nighthawk_service_stub; + EXPECT_CALL(mock_nighthawk_service_stub, ExecutionStreamRaw) + .WillRepeatedly( + [](grpc_impl::ClientContext*) { return MakeConvergingMockClientReaderWriter(); }); + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = + PerformAdaptiveLoadSession(&mock_nighthawk_service_stub, spec, time_source); + ASSERT_TRUE(output_or.ok()); + AdaptiveLoadSessionOutput output = output_or.value(); + EXPECT_TRUE(output.has_testing_stage_result()); +} + +TEST(AdaptiveLoadController, SetsBenchmarkErrorStatusIfNighthawkServiceDoesNotSendResponse) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec = MakeConvergeableDoomableSessionSpec(); + + nighthawk::client::MockNighthawkServiceStub mock_nighthawk_service_stub; + EXPECT_CALL(mock_nighthawk_service_stub, ExecutionStreamRaw) + .WillRepeatedly([](grpc_impl::ClientContext*) { + auto* mock_reader_writer = + new grpc::testing::MockClientReaderWriter(); + // Simulate gRPC Read() failing: + EXPECT_CALL(*mock_reader_writer, Read(_)).WillRepeatedly(Return(false)); + EXPECT_CALL(*mock_reader_writer, Write(_, _)).WillRepeatedly(Return(true)); + EXPECT_CALL(*mock_reader_writer, WritesDone()).WillRepeatedly(Return(true)); + EXPECT_CALL(*mock_reader_writer, Finish()).WillRepeatedly(Return(::grpc::Status::OK)); + return mock_reader_writer; + }); + + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = + PerformAdaptiveLoadSession(&mock_nighthawk_service_stub, spec, time_source); + ASSERT_FALSE(output_or.ok()); + EXPECT_EQ(output_or.status().code(), absl::StatusCode::kUnknown); + EXPECT_THAT(output_or.status().message(), + HasSubstr("Nighthawk Service did not send a gRPC response.")); +} + +TEST(AdaptiveLoadController, SetsBenchmarkErrorStatusIfNighthawkServiceWriteFails) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec = MakeConvergeableDoomableSessionSpec(); + + nighthawk::client::MockNighthawkServiceStub mock_nighthawk_service_stub; + EXPECT_CALL(mock_nighthawk_service_stub, ExecutionStreamRaw) + .WillRepeatedly([](grpc_impl::ClientContext*) { + auto* mock_reader_writer = + new grpc::testing::MockClientReaderWriter(); + EXPECT_CALL(*mock_reader_writer, Read(_)).WillRepeatedly(Return(true)); + EXPECT_CALL(*mock_reader_writer, Write(_, _)).WillRepeatedly(Return(false)); + EXPECT_CALL(*mock_reader_writer, WritesDone()).WillRepeatedly(Return(true)); + EXPECT_CALL(*mock_reader_writer, Finish()).WillRepeatedly(Return(::grpc::Status::OK)); + return mock_reader_writer; + }); + + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = + PerformAdaptiveLoadSession(&mock_nighthawk_service_stub, spec, time_source); + ASSERT_FALSE(output_or.ok()); + EXPECT_EQ(output_or.status().code(), absl::StatusCode::kUnknown); + EXPECT_THAT(output_or.status().message(), HasSubstr("Failed to write")); +} + +TEST(AdaptiveLoadController, SetsBenchmarkErrorStatusIfNighthawkServiceWritesDoneFails) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec = MakeConvergeableDoomableSessionSpec(); + + nighthawk::client::MockNighthawkServiceStub mock_nighthawk_service_stub; + EXPECT_CALL(mock_nighthawk_service_stub, ExecutionStreamRaw) + .WillRepeatedly([](grpc_impl::ClientContext*) { + auto* mock_reader_writer = + new grpc::testing::MockClientReaderWriter(); + EXPECT_CALL(*mock_reader_writer, Read(_)).WillRepeatedly(Return(true)); + EXPECT_CALL(*mock_reader_writer, Write(_, _)).WillRepeatedly(Return(true)); + EXPECT_CALL(*mock_reader_writer, WritesDone()).WillRepeatedly(Return(false)); + EXPECT_CALL(*mock_reader_writer, Finish()).WillRepeatedly(Return(::grpc::Status::OK)); + return mock_reader_writer; + }); + + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = + PerformAdaptiveLoadSession(&mock_nighthawk_service_stub, spec, time_source); + ASSERT_FALSE(output_or.ok()); + EXPECT_EQ(output_or.status().code(), absl::StatusCode::kUnknown); + EXPECT_THAT(output_or.status().message(), HasSubstr("WritesDone() failed")); +} + +TEST(AdaptiveLoadController, SetsBenchmarkErrorStatusIfNighthawkServiceGrpcStreamClosesAbnormally) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec = MakeConvergeableDoomableSessionSpec(); + + nighthawk::client::MockNighthawkServiceStub mock_nighthawk_service_stub; + EXPECT_CALL(mock_nighthawk_service_stub, ExecutionStreamRaw) + .WillRepeatedly([](grpc_impl::ClientContext*) { + auto* mock_reader_writer = + new grpc::testing::MockClientReaderWriter(); + EXPECT_CALL(*mock_reader_writer, Read(_)) + .WillOnce(Return(true)) + .WillRepeatedly(Return(false)); + EXPECT_CALL(*mock_reader_writer, Write(_, _)).WillRepeatedly(Return(true)); + EXPECT_CALL(*mock_reader_writer, WritesDone()).WillRepeatedly(Return(true)); + // Simulate gRPC abnormal stream shutdown: + EXPECT_CALL(*mock_reader_writer, Finish()) + .WillRepeatedly( + Return(::grpc::Status(::grpc::UNKNOWN, "Finish failure status message"))); + return mock_reader_writer; + }); + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = + PerformAdaptiveLoadSession(&mock_nighthawk_service_stub, spec, time_source); + ASSERT_FALSE(output_or.ok()); + EXPECT_EQ(output_or.status().code(), absl::StatusCode::kUnknown); + EXPECT_THAT(output_or.status().message(), HasSubstr("Finish failure status message")); +} + +TEST(AdaptiveLoadController, UsesBuiltinMetricsPluginForThresholdByDefault) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec = MakeConvergeableDoomableSessionSpec(); + + nighthawk::adaptive_load::MetricSpecWithThreshold* threshold = + spec.mutable_metric_thresholds()->Add(); + threshold->mutable_metric_spec()->set_metric_name("success-rate"); + // metrics_plugin_name not set, defaults to nighthawk.builtin + *threshold->mutable_threshold_spec()->mutable_scoring_function() = + MakeLowerThresholdBinaryScoringFunctionConfig(0.0); + + nighthawk::client::MockNighthawkServiceStub mock_nighthawk_service_stub; + EXPECT_CALL(mock_nighthawk_service_stub, ExecutionStreamRaw) + .WillRepeatedly( + [](grpc_impl::ClientContext*) { return MakeConvergingMockClientReaderWriter(); }); + + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = + PerformAdaptiveLoadSession(&mock_nighthawk_service_stub, spec, time_source); + ASSERT_TRUE(output_or.ok()); + AdaptiveLoadSessionOutput output = output_or.value(); + ASSERT_GT(output.adjusting_stage_results_size(), 0); + ASSERT_GT(output.adjusting_stage_results()[0].metric_evaluations_size(), 1); + EXPECT_EQ(output.adjusting_stage_results()[0].metric_evaluations()[1].metric_value(), 0.03125); +} + +TEST(AdaptiveLoadController, EvaluatesBuiltinMetric) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec = MakeConvergeableDoomableSessionSpec(); + + nighthawk::adaptive_load::MetricSpecWithThreshold* threshold = + spec.mutable_metric_thresholds()->Add(); + threshold->mutable_metric_spec()->set_metric_name("success-rate"); + threshold->mutable_metric_spec()->set_metrics_plugin_name("nighthawk.builtin"); + *threshold->mutable_threshold_spec()->mutable_scoring_function() = + MakeLowerThresholdBinaryScoringFunctionConfig(0.0); + + nighthawk::client::MockNighthawkServiceStub mock_nighthawk_service_stub; + EXPECT_CALL(mock_nighthawk_service_stub, ExecutionStreamRaw) + .WillRepeatedly( + [](grpc_impl::ClientContext*) { return MakeConvergingMockClientReaderWriter(); }); + + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = + PerformAdaptiveLoadSession(&mock_nighthawk_service_stub, spec, time_source); + ASSERT_TRUE(output_or.ok()); + AdaptiveLoadSessionOutput output = output_or.value(); + ASSERT_GT(output.adjusting_stage_results_size(), 0); + ASSERT_GT(output.adjusting_stage_results()[0].metric_evaluations_size(), 1); + EXPECT_EQ(output.adjusting_stage_results()[0].metric_evaluations()[1].metric_value(), 0.03125); + // Requested a lower threshold of 0.0 and achieved 0.03125. + EXPECT_EQ(output.adjusting_stage_results()[0].metric_evaluations()[1].threshold_score(), 1.0); +} + +TEST(AdaptiveLoadController, UsesBuiltinMetricsPluginForInformationalMetricSpecByDefault) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec = MakeConvergeableDoomableSessionSpec(); + + nighthawk::adaptive_load::MetricSpec* metric_spec = + spec.mutable_informational_metric_specs()->Add(); + metric_spec->set_metric_name("success-rate"); + // metrics_plugin_name not set, defaults to nighthawk.builtin + + nighthawk::client::MockNighthawkServiceStub mock_nighthawk_service_stub; + EXPECT_CALL(mock_nighthawk_service_stub, ExecutionStreamRaw) + .WillRepeatedly( + [](grpc_impl::ClientContext*) { return MakeConvergingMockClientReaderWriter(); }); + + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = + PerformAdaptiveLoadSession(&mock_nighthawk_service_stub, spec, time_source); + ASSERT_TRUE(output_or.ok()); + AdaptiveLoadSessionOutput output = output_or.value(); + ASSERT_GT(output.adjusting_stage_results_size(), 0); + ASSERT_GT(output.adjusting_stage_results()[0].metric_evaluations_size(), 1); + EXPECT_EQ(output.adjusting_stage_results()[0].metric_evaluations()[1].metric_value(), 0.03125); +} + +TEST(AdaptiveLoadController, StoresInformationalBuiltinMetric) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec = MakeConvergeableDoomableSessionSpec(); + + nighthawk::adaptive_load::MetricSpec* metric_spec = + spec.mutable_informational_metric_specs()->Add(); + metric_spec->set_metric_name("success-rate"); + metric_spec->set_metrics_plugin_name("nighthawk.builtin"); + + nighthawk::client::MockNighthawkServiceStub mock_nighthawk_service_stub; + EXPECT_CALL(mock_nighthawk_service_stub, ExecutionStreamRaw) + .WillRepeatedly( + [](grpc_impl::ClientContext*) { return MakeConvergingMockClientReaderWriter(); }); + + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = + PerformAdaptiveLoadSession(&mock_nighthawk_service_stub, spec, time_source); + ASSERT_TRUE(output_or.ok()); + AdaptiveLoadSessionOutput output = output_or.value(); + ASSERT_GT(output.adjusting_stage_results_size(), 0); + ASSERT_GT(output.adjusting_stage_results()[0].metric_evaluations_size(), 1); + EXPECT_EQ(output.adjusting_stage_results()[0].metric_evaluations()[1].metric_value(), 0.03125); +} + +TEST(AdaptiveLoadController, EvaluatesCustomMetric) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec = MakeConvergeableDoomableSessionSpec(); + // Configures a metric with value 5.0: + *spec.mutable_metrics_plugin_configs()->Add() = MakeFakeMetricsPluginConfig(); + nighthawk::adaptive_load::MetricSpecWithThreshold* threshold = + spec.mutable_metric_thresholds()->Add(); + threshold->mutable_metric_spec()->set_metric_name("good_metric"); + threshold->mutable_metric_spec()->set_metrics_plugin_name("nighthawk.fake_metrics_plugin"); + *threshold->mutable_threshold_spec()->mutable_scoring_function() = + MakeLowerThresholdBinaryScoringFunctionConfig(4.0); + + nighthawk::client::MockNighthawkServiceStub mock_nighthawk_service_stub; + EXPECT_CALL(mock_nighthawk_service_stub, ExecutionStreamRaw) + .WillRepeatedly( + [](grpc_impl::ClientContext*) { return MakeConvergingMockClientReaderWriter(); }); + + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = + PerformAdaptiveLoadSession(&mock_nighthawk_service_stub, spec, time_source); + ASSERT_TRUE(output_or.ok()); + AdaptiveLoadSessionOutput output = output_or.value(); + ASSERT_GT(output.adjusting_stage_results_size(), 0); + ASSERT_GT(output.adjusting_stage_results()[0].metric_evaluations_size(), 1); + // Requested a lower threshold of 4.0 and achieved 5.0. + EXPECT_EQ(output.adjusting_stage_results()[0].metric_evaluations()[1].threshold_score(), 1.0); +} + +TEST(AdaptiveLoadController, StoresInformationalCustomMetric) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec = MakeConvergeableDoomableSessionSpec(); + *spec.mutable_metrics_plugin_configs()->Add() = MakeFakeMetricsPluginConfig(); + nighthawk::adaptive_load::MetricSpec* metric_spec = + spec.mutable_informational_metric_specs()->Add(); + metric_spec->set_metric_name("good_metric"); + metric_spec->set_metrics_plugin_name("nighthawk.fake_metrics_plugin"); + + nighthawk::client::MockNighthawkServiceStub mock_nighthawk_service_stub; + EXPECT_CALL(mock_nighthawk_service_stub, ExecutionStreamRaw) + .WillRepeatedly( + [](grpc_impl::ClientContext*) { return MakeConvergingMockClientReaderWriter(); }); + + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = + PerformAdaptiveLoadSession(&mock_nighthawk_service_stub, spec, time_source); + ASSERT_TRUE(output_or.ok()); + AdaptiveLoadSessionOutput output = output_or.value(); + ASSERT_GT(output.adjusting_stage_results_size(), 0); + ASSERT_GT(output.adjusting_stage_results()[0].metric_evaluations_size(), 1); + EXPECT_EQ(output.adjusting_stage_results()[0].metric_evaluations()[1].metric_value(), 5.0); +} + +TEST(AdaptiveLoadController, PropagatesErrorFromMetricsPlugin) { + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec = MakeConvergeableDoomableSessionSpec(); + *spec.mutable_metrics_plugin_configs()->Add() = MakeFakeMetricsPluginConfig(); + nighthawk::adaptive_load::MetricSpec* metric_spec = + spec.mutable_informational_metric_specs()->Add(); + metric_spec->set_metric_name("bad_metric"); + metric_spec->set_metrics_plugin_name("nighthawk.fake_metrics_plugin"); + + nighthawk::client::MockNighthawkServiceStub mock_nighthawk_service_stub; + EXPECT_CALL(mock_nighthawk_service_stub, ExecutionStreamRaw) + .WillRepeatedly( + [](grpc_impl::ClientContext*) { return MakeConvergingMockClientReaderWriter(); }); + + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = + PerformAdaptiveLoadSession(&mock_nighthawk_service_stub, spec, time_source); + ASSERT_FALSE(output_or.ok()); + EXPECT_THAT(output_or.status().message(), HasSubstr("Error calling MetricsPlugin")); +} + +TEST(AdaptiveLoadController, CopiesThresholdSpecToOutput) { + // Spec contains a threshold for send-rate: + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec = MakeConvergeableDoomableSessionSpec(); + + nighthawk::client::MockNighthawkServiceStub mock_nighthawk_service_stub; + EXPECT_CALL(mock_nighthawk_service_stub, ExecutionStreamRaw) + .WillRepeatedly( + [](grpc_impl::ClientContext*) { return MakeConvergingMockClientReaderWriter(); }); + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = + PerformAdaptiveLoadSession(&mock_nighthawk_service_stub, spec, time_source); + ASSERT_TRUE(output_or.ok()); + AdaptiveLoadSessionOutput output = output_or.value(); + ASSERT_GT(output.metric_thresholds_size(), 0); + EXPECT_EQ(output.metric_thresholds()[0].metric_spec().metric_name(), "send-rate"); +} + +TEST(AdaptiveLoadController, PropagatesInputVariableSettingErrorInAdjustingStage) { + const std::string kExpectedErrorMessage = "artificial input value setting error"; + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec = MakeConvergeableDoomableSessionSpec(); + *spec.mutable_step_controller_config() = MakeFakeStepControllerPluginConfigWithInputSettingError( + /*fixed_rps_value=*/5, absl::PermissionDeniedError(kExpectedErrorMessage), /*countdown=*/0); + + *spec.mutable_metrics_plugin_configs()->Add() = MakeFakeMetricsPluginConfig(); + nighthawk::adaptive_load::MetricSpec* metric_spec = + spec.mutable_informational_metric_specs()->Add(); + metric_spec->set_metric_name("good_metric"); + metric_spec->set_metrics_plugin_name("nighthawk.fake_metrics_plugin"); + + nighthawk::client::MockNighthawkServiceStub mock_nighthawk_service_stub; + EXPECT_CALL(mock_nighthawk_service_stub, ExecutionStreamRaw) + .WillRepeatedly( + [](grpc_impl::ClientContext*) { return MakeConvergingMockClientReaderWriter(); }); + + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = + PerformAdaptiveLoadSession(&mock_nighthawk_service_stub, spec, time_source); + ASSERT_FALSE(output_or.ok()); + EXPECT_THAT(output_or.status().message(), HasSubstr(kExpectedErrorMessage)); +} + +TEST(AdaptiveLoadController, PropagatesInputVariableSettingErrorInTestingStage) { + const std::string kExpectedErrorMessage = "artificial input value setting error"; + nighthawk::adaptive_load::AdaptiveLoadSessionSpec spec = MakeConvergeableDoomableSessionSpec(); + + // Adjusting stage will converge after 1 iteration, then testing stage will get the input setting + // error. + *spec.mutable_step_controller_config() = MakeFakeStepControllerPluginConfigWithInputSettingError( + /*fixed_rps_value=*/5, absl::PermissionDeniedError(kExpectedErrorMessage), /*countdown=*/1); + + *spec.mutable_metrics_plugin_configs()->Add() = MakeFakeMetricsPluginConfig(); + nighthawk::adaptive_load::MetricSpec* metric_spec = + spec.mutable_informational_metric_specs()->Add(); + metric_spec->set_metric_name("good_metric"); + metric_spec->set_metrics_plugin_name("nighthawk.fake_metrics_plugin"); + + nighthawk::client::MockNighthawkServiceStub mock_nighthawk_service_stub; + EXPECT_CALL(mock_nighthawk_service_stub, ExecutionStreamRaw) + .WillRepeatedly( + [](grpc_impl::ClientContext*) { return MakeConvergingMockClientReaderWriter(); }); + + FakeIncrementingMonotonicTimeSource time_source; + absl::StatusOr output_or = + PerformAdaptiveLoadSession(&mock_nighthawk_service_stub, spec, time_source); + ASSERT_FALSE(output_or.ok()); + EXPECT_THAT(output_or.status().message(), HasSubstr(kExpectedErrorMessage)); +} + +} // namespace + +} // namespace Nighthawk diff --git a/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.cc b/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.cc index 061727805..da47985e7 100644 --- a/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.cc +++ b/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.cc @@ -15,9 +15,10 @@ absl::Status StatusFromProtoRpcStatus(const google::rpc::Status& status_proto) { } // namespace FakeStepController::FakeStepController( - const nighthawk::adaptive_load::FakeStepControllerConfig& config, + nighthawk::adaptive_load::FakeStepControllerConfig config, nighthawk::client::CommandLineOptions command_line_options_template) - : is_converged_{false}, is_doomed_{false}, fixed_rps_value_{config.fixed_rps_value()}, + : input_setting_failure_countdown_{config.artificial_input_setting_failure_countdown()}, + config_{std::move(config)}, is_converged_{false}, is_doomed_{false}, command_line_options_template_{std::move(command_line_options_template)} {} bool FakeStepController::IsConverged() const { return is_converged_; } @@ -33,27 +34,31 @@ bool FakeStepController::IsDoomed(std::string& doomed_reason) const { absl::StatusOr FakeStepController::GetCurrentCommandLineOptions() const { + if (config_.has_artificial_input_setting_failure() && input_setting_failure_countdown_ <= 0) { + return StatusFromProtoRpcStatus(config_.artificial_input_setting_failure()); + } nighthawk::client::CommandLineOptions options = command_line_options_template_; - options.mutable_requests_per_second()->set_value(fixed_rps_value_); + options.mutable_requests_per_second()->set_value(config_.fixed_rps_value()); return options; } void FakeStepController::UpdateAndRecompute( const nighthawk::adaptive_load::BenchmarkResult& benchmark_result) { - if (benchmark_result.status().code() == ::grpc::OK) { - is_doomed_ = false; - doomed_reason_ = ""; - } else { - is_doomed_ = true; - doomed_reason_ = benchmark_result.status().message(); + if (input_setting_failure_countdown_ > 0) { + --input_setting_failure_countdown_; } // "Convergence" is defined as the latest benchmark reporting any score > 0.0. + // "Doom" is defined as any score < 0.0. Neutral is all scores equal to 0.0. is_converged_ = false; + is_doomed_ = false; + doomed_reason_ = ""; for (const nighthawk::adaptive_load::MetricEvaluation& metric_evaluation : benchmark_result.metric_evaluations()) { - if (metric_evaluation.threshold_score() > 0.0) { + if (metric_evaluation.threshold_score() < 0.0) { + is_doomed_ = true; + doomed_reason_ = "artificial doom triggered by negative score"; + } else if (metric_evaluation.threshold_score() > 0.0) { is_converged_ = true; - break; } } } @@ -117,4 +122,20 @@ envoy::config::core::v3::TypedExtensionConfig MakeFakeStepControllerPluginConfig return outer_config; } +envoy::config::core::v3::TypedExtensionConfig +MakeFakeStepControllerPluginConfigWithInputSettingError( + int fixed_rps_value, const absl::Status& artificial_input_setting_failure, int countdown) { + envoy::config::core::v3::TypedExtensionConfig outer_config; + outer_config.set_name("nighthawk.fake_step_controller"); + nighthawk::adaptive_load::FakeStepControllerConfig config; + config.set_fixed_rps_value(fixed_rps_value); + config.mutable_artificial_input_setting_failure()->set_code( + static_cast(artificial_input_setting_failure.code())); + config.mutable_artificial_input_setting_failure()->set_message( + std::string(artificial_input_setting_failure.message())); + config.set_artificial_input_setting_failure_countdown(countdown); + outer_config.mutable_typed_config()->PackFrom(config); + return outer_config; +} + } // namespace Nighthawk diff --git a/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.h b/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.h index a039eba8c..b0780e676 100644 --- a/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.h +++ b/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.h @@ -22,7 +22,7 @@ class FakeStepController : public StepController { * @param config FakeStepControllerConfig proto for setting the fixed RPS value. * @param command_line_options_template A template for producing Nighthawk input. */ - FakeStepController(const nighthawk::adaptive_load::FakeStepControllerConfig& config, + FakeStepController(nighthawk::adaptive_load::FakeStepControllerConfig config, nighthawk::client::CommandLineOptions command_line_options_template); /** * @return bool The current value of |is_converged_|. @@ -42,9 +42,9 @@ class FakeStepController : public StepController { absl::StatusOr GetCurrentCommandLineOptions() const override; /** - * Updates |is_converged_| to reflect whether |benchmark_result| contains any score >0. Sets - * |is_doomed_| based whether the status in |benchmark_result| is OK; copies the status message - * into |doomed_reason_| only when the status is not OK. + * Updates |is_converged_| to reflect whether |benchmark_result| contains any score >0. Updates + * |is_doomed_| to reflect whether |benchmark_result| contains any score <0. A non-converged, + * non-doomed input has scores all equal to 0. * * @param benchmark_result A Nighthawk benchmark result proto. */ @@ -52,10 +52,13 @@ class FakeStepController : public StepController { UpdateAndRecompute(const nighthawk::adaptive_load::BenchmarkResult& benchmark_result) override; private: + // Counts down UpdateAndRecompute() calls. When this reaches zero, GetCurrentCommandLineOptions() + // starts to return an artificial input value setting failure if one is specified in the config. + int input_setting_failure_countdown_; + const nighthawk::adaptive_load::FakeStepControllerConfig config_; bool is_converged_; bool is_doomed_; std::string doomed_reason_; - const int fixed_rps_value_; const nighthawk::client::CommandLineOptions command_line_options_template_; }; @@ -100,4 +103,21 @@ MakeFakeStepControllerPluginConfig(int fixed_rps_value); envoy::config::core::v3::TypedExtensionConfig MakeFakeStepControllerPluginConfigWithValidationError( const absl::Status& artificial_validation_error); +/** + * Creates a valid TypedExtensionConfig proto that activates a FakeStepController with a + * FakeInputVariableSetterConfig that returns an error from GetCurrentCommandLineOptions(). + * + * @param fixed_rps_value Value for RPS to set in the FakeStepControllerConfig proto until the + * countdown reaches zero. + * @param artificial_input_setting_failure An error status. + * @param countdown Number of times UpdateAndRecompute() must be called before + * GetCurrentCommandLineOptions() starts to return the input error status. + * + * @return TypedExtensionConfig A proto that activates FakeStepController by name and includes + * a FakeStepControllerConfig proto wrapped in an Any. + */ +envoy::config::core::v3::TypedExtensionConfig +MakeFakeStepControllerPluginConfigWithInputSettingError( + int fixed_rps_value, const absl::Status& artificial_input_setting_failure, int countdown); + } // namespace Nighthawk diff --git a/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.proto b/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.proto index 8dfe0ff76..f4e1f9286 100644 --- a/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.proto +++ b/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller.proto @@ -5,12 +5,23 @@ package nighthawk.adaptive_load; import "envoy/config/core/v3/extension.proto"; import "google/rpc/status.proto"; -// Configuration for FakeStepController (plugin name: "nighthawk.fake_step_controller") that always -// returns a fixed RPS value and changes converged and doomed states based on the latest reported -// BenchmarkResult. +// Configuration for FakeStepController (plugin name: "nighthawk.fake_step_controller") that returns +// a fixed RPS value and changes converged and doomed states based on the latest reported +// BenchmarkResult. Can also be programmed to return a proto validation failure, return an error +// from input value setting every time, or return an error after some number of UpdateAndRecompute() +// iterations. message FakeStepControllerConfig { - // RPS that should always be returned. Optional, default 0. + // RPS that should always be returned, except when artificial errors are configured. Optional, + // default 0. int32 fixed_rps_value = 1; // Artificial error that the plugin factory should return during validation. Optional. google.rpc.Status artificial_validation_failure = 2; + // Artificial error that should be returned from GetCurrentCommandLineOptions(). Optional. + google.rpc.Status artificial_input_setting_failure = 3; + // Relevant only when |artificial_input_setting_failure| is set. Number of calls to + // UpdateAndRecompute() the controller must receive before it starts to return + // |artificial_input_setting_failure|. Before this total is reached, |fixed_rps_value| is + // returned. Optional, default 0, meaning the failure is returned regardless of calls to + // UpdateAndRecompute(). + int32 artificial_input_setting_failure_countdown = 4; } diff --git a/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller_test.cc b/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller_test.cc index 830189da4..e28437316 100644 --- a/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller_test.cc +++ b/test/adaptive_load/fake_plugins/fake_step_controller/fake_step_controller_test.cc @@ -111,19 +111,66 @@ TEST(FakeStepController, GetCurrentCommandLineOptionsReturnsRpsFromConfig) { kExpectedValue); } +TEST(FakeStepController, GetCurrentCommandLineOptionsReturnsArtificialErrorImmediately) { + FakeStepControllerConfig config; + const int kExpectedCode = ::grpc::DEADLINE_EXCEEDED; + const std::string kExpectedMessage = "artificial input setting error"; + config.mutable_artificial_input_setting_failure()->set_code(kExpectedCode); + config.mutable_artificial_input_setting_failure()->set_message(kExpectedMessage); + // Not setting countdown. + + FakeStepController step_controller(config, CommandLineOptions()); + absl::StatusOr command_line_options_or = + step_controller.GetCurrentCommandLineOptions(); + ASSERT_FALSE(command_line_options_or.ok()); + EXPECT_EQ(static_cast(command_line_options_or.status().code()), kExpectedCode); + EXPECT_EQ(command_line_options_or.status().message(), kExpectedMessage); +} + +TEST(FakeStepController, GetCurrentCommandLineOptionsReturnsArtificialErrorAfterCountdown) { + FakeStepControllerConfig config; + const int kExpectedCode = ::grpc::DEADLINE_EXCEEDED; + const std::string kExpectedMessage = "artificial input setting error"; + config.mutable_artificial_input_setting_failure()->set_code(kExpectedCode); + config.mutable_artificial_input_setting_failure()->set_message(kExpectedMessage); + config.set_artificial_input_setting_failure_countdown(2); + + FakeStepController step_controller(config, CommandLineOptions()); + absl::StatusOr command_line_options_or1 = + step_controller.GetCurrentCommandLineOptions(); + EXPECT_TRUE(command_line_options_or1.ok()); + + step_controller.UpdateAndRecompute(nighthawk::adaptive_load::BenchmarkResult()); + // Countdown should now be 1. + + absl::StatusOr command_line_options_or2 = + step_controller.GetCurrentCommandLineOptions(); + EXPECT_TRUE(command_line_options_or2.ok()); + + step_controller.UpdateAndRecompute(nighthawk::adaptive_load::BenchmarkResult()); + // Countdown should now have reached 0. + + // This should now return the artificial input setting failure: + absl::StatusOr command_line_options_or3 = + step_controller.GetCurrentCommandLineOptions(); + ASSERT_FALSE(command_line_options_or3.ok()); + EXPECT_EQ(static_cast(command_line_options_or3.status().code()), kExpectedCode); + EXPECT_EQ(command_line_options_or3.status().message(), kExpectedMessage); +} + TEST(FakeStepController, IsConvergedInitiallyReturnsFalse) { FakeStepController step_controller(FakeStepControllerConfig{}, CommandLineOptions{}); EXPECT_FALSE(step_controller.IsConverged()); } -TEST(FakeStepController, IsConvergedReturnsFalseAfterBenchmarkResultWithoutPositiveScore) { +TEST(FakeStepController, IsConvergedReturnsFalseAfterNeutralBenchmarkResult) { FakeStepController step_controller(FakeStepControllerConfig{}, CommandLineOptions{}); BenchmarkResult benchmark_result; step_controller.UpdateAndRecompute(benchmark_result); EXPECT_FALSE(step_controller.IsConverged()); } -TEST(FakeStepController, IsConvergedReturnsTrueAfterBenchmarkResultWithPositiveScore) { +TEST(FakeStepController, IsConvergedReturnsTrueAfterPositiveBenchmarkResultScore) { FakeStepController step_controller(FakeStepControllerConfig{}, CommandLineOptions{}); BenchmarkResult benchmark_result; MetricEvaluation* evaluation = benchmark_result.mutable_metric_evaluations()->Add(); @@ -132,36 +179,33 @@ TEST(FakeStepController, IsConvergedReturnsTrueAfterBenchmarkResultWithPositiveS EXPECT_TRUE(step_controller.IsConverged()); } -TEST(FakeStepController, IsDoomedReturnsFalseAfterSuccessfulBenchmarkResult) { +TEST(FakeStepController, IsDoomedReturnsFalseAfterNeutralBenchmarkResult) { FakeStepController step_controller(FakeStepControllerConfig{}, CommandLineOptions{}); BenchmarkResult benchmark_result; - benchmark_result.mutable_status()->set_code(::grpc::OK); step_controller.UpdateAndRecompute(benchmark_result); std::string doomed_reason; EXPECT_FALSE(step_controller.IsDoomed(doomed_reason)); } TEST(FakeStepController, - IsDoomedReturnsFalseAndLeavesDoomedReasonUntouchedAfterSuccessfulBenchmarkResult) { + IsDoomedReturnsFalseAndLeavesDoomedReasonUntouchedAfterNeutralBenchmarkResult) { FakeStepController step_controller(FakeStepControllerConfig{}, CommandLineOptions{}); BenchmarkResult benchmark_result; - benchmark_result.mutable_status()->set_code(::grpc::OK); step_controller.UpdateAndRecompute(benchmark_result); std::string variable_that_should_not_be_written = "original value"; EXPECT_FALSE(step_controller.IsDoomed(variable_that_should_not_be_written)); EXPECT_EQ(variable_that_should_not_be_written, "original value"); } -TEST(FakeStepController, IsDoomedReturnsTrueAndSetsDoomedReasonAfterFailedBenchmarkResult) { - const std::string kErrorMessage = "error from nighthawk"; +TEST(FakeStepController, IsDoomedReturnsTrueAndSetsDoomedReasonAfterNegativeBenchmarkResultScore) { FakeStepController step_controller(FakeStepControllerConfig{}, CommandLineOptions{}); BenchmarkResult benchmark_result; - benchmark_result.mutable_status()->set_code(::grpc::INTERNAL); - benchmark_result.mutable_status()->set_message(kErrorMessage); + MetricEvaluation* evaluation = benchmark_result.mutable_metric_evaluations()->Add(); + evaluation->set_threshold_score(-1.0); step_controller.UpdateAndRecompute(benchmark_result); std::string doomed_reason; EXPECT_TRUE(step_controller.IsDoomed(doomed_reason)); - EXPECT_EQ(doomed_reason, kErrorMessage); + EXPECT_EQ(doomed_reason, "artificial doom triggered by negative score"); } TEST(MakeFakeStepControllerPluginConfig, ActivatesFakeStepControllerPlugin) { @@ -185,7 +229,7 @@ TEST(MakeFakeStepControllerPluginConfig, ProducesFakeStepControllerPluginWithCon } TEST(MakeFakeStepControllerPluginConfigWithValidationError, - ProducesFakeStepControllerPluginWithConfiguredValue) { + ProducesFakeStepControllerPluginWithConfiguredError) { std::string kValidationErrorMessage = "artificial validation error"; absl::StatusOr plugin_or = LoadStepControllerPlugin(MakeFakeStepControllerPluginConfigWithValidationError( @@ -195,5 +239,29 @@ TEST(MakeFakeStepControllerPluginConfigWithValidationError, EXPECT_EQ(plugin_or.status().message(), kValidationErrorMessage); } +TEST(MakeFakeStepControllerPluginConfigWithInputSettingError, + ProducesFakeStepControllerPluginWithConfiguredErrorAndCountdown) { + const int kExpectedRpsValue = 123; + std::string kInputSettingErrorMessage = "artificial input setting error"; + absl::StatusOr plugin_or = LoadStepControllerPlugin( + MakeFakeStepControllerPluginConfigWithInputSettingError( + kExpectedRpsValue, absl::DeadlineExceededError(kInputSettingErrorMessage), + /*countdown=*/1), + nighthawk::client::CommandLineOptions{}); + ASSERT_TRUE(plugin_or.ok()); + auto* plugin = dynamic_cast(plugin_or.value().get()); + ASSERT_NE(plugin, nullptr); + absl::StatusOr command_line_options_or1 = + plugin->GetCurrentCommandLineOptions(); + ASSERT_TRUE(command_line_options_or1.ok()); + EXPECT_EQ(command_line_options_or1.value().requests_per_second().value(), kExpectedRpsValue); + plugin->UpdateAndRecompute(BenchmarkResult()); + absl::StatusOr command_line_options_or2 = + plugin->GetCurrentCommandLineOptions(); + ASSERT_FALSE(command_line_options_or2.ok()); + EXPECT_EQ(command_line_options_or2.status().code(), absl::StatusCode::kDeadlineExceeded); + EXPECT_EQ(command_line_options_or2.status().message(), kInputSettingErrorMessage); +} + } // namespace } // namespace Nighthawk diff --git a/test/adaptive_load/fake_time_source.cc b/test/adaptive_load/fake_time_source.cc new file mode 100644 index 000000000..3bba1a346 --- /dev/null +++ b/test/adaptive_load/fake_time_source.cc @@ -0,0 +1,16 @@ +#include "test/adaptive_load/fake_time_source.h" + +namespace Nighthawk { + +Envoy::SystemTime FakeIncrementingMonotonicTimeSource::systemTime() { + Envoy::SystemTime epoch; + return epoch; +} + +Envoy::MonotonicTime FakeIncrementingMonotonicTimeSource::monotonicTime() { + ++seconds_since_epoch_; + Envoy::MonotonicTime epoch; + return epoch + std::chrono::seconds(seconds_since_epoch_); +} + +} // namespace Nighthawk diff --git a/test/adaptive_load/fake_time_source.h b/test/adaptive_load/fake_time_source.h new file mode 100644 index 000000000..fa3f6ae40 --- /dev/null +++ b/test/adaptive_load/fake_time_source.h @@ -0,0 +1,29 @@ +#pragma once + +#include "envoy/common/time.h" + +namespace Nighthawk { +/** + * Fake time source that ticks 1 second on every query, starting from the Unix epoch. Supports only + * monotonicTime(). + */ +class FakeIncrementingMonotonicTimeSource : public Envoy::TimeSource { +public: + /** + * Not supported. + * + * @return Envoy::SystemTime Fixed value of the Unix epoch. + */ + Envoy::SystemTime systemTime() override; + /** + * Ticks forward 1 second on each call. + * + * @return Envoy::MonotonicTime Fake time value. + */ + Envoy::MonotonicTime monotonicTime() override; + +private: + int seconds_since_epoch_{0}; +}; + +} // namespace Nighthawk diff --git a/test/adaptive_load/step_controller_test.cc b/test/adaptive_load/step_controller_test.cc index d502a21f6..678846b75 100644 --- a/test/adaptive_load/step_controller_test.cc +++ b/test/adaptive_load/step_controller_test.cc @@ -29,12 +29,6 @@ nighthawk::adaptive_load::BenchmarkResult MakeBenchmarkResultWithScore(double sc return result; } -nighthawk::adaptive_load::BenchmarkResult MakeBenchmarkResultWithNighthawkError() { - nighthawk::adaptive_load::BenchmarkResult result; - result.mutable_status()->set_code(1); - return result; -} - TEST(ExponentialSearchStepControllerConfigFactory, GeneratesEmptyConfigProto) { auto& config_factory = Envoy::Config::Utility::getAndCheckFactoryByName( @@ -175,16 +169,6 @@ TEST(ExponentialSearchStepController, ReportsDoomIfOutsideThresholdsOnInitialVal EXPECT_THAT(doom_reason, HasSubstr("already exceed metric thresholds with the initial load")); } -TEST(ExponentialSearchStepController, ReportsDoomAfterNighthawkServiceError) { - nighthawk::adaptive_load::ExponentialSearchStepControllerConfig config; - nighthawk::client::CommandLineOptions options_template; - ExponentialSearchStepController step_controller(config, options_template); - step_controller.UpdateAndRecompute(MakeBenchmarkResultWithNighthawkError()); - std::string doom_reason; - EXPECT_TRUE(step_controller.IsDoomed(doom_reason)); - EXPECT_EQ(doom_reason, "Nighthawk Service returned an error."); -} - TEST(ExponentialSearchStepController, IncreasesRpsExponentiallyIfWithinThresholdUsingDefaultExponent) { const double kInitialInput = 100.0; diff --git a/tools/check_format.sh b/tools/check_format.sh index 4f9b6a210..b47ad09a6 100755 --- a/tools/check_format.sh +++ b/tools/check_format.sh @@ -8,7 +8,7 @@ TO_CHECK="${2:-$PWD}" bazel run @envoy//tools:code_format/check_format.py -- \ --skip_envoy_build_rule_check --namespace_check Nighthawk \ --build_fixer_check_excluded_paths=$(realpath ".") \ - --include_dir_order envoy,nighthawk,external/source/envoy,external,api,common,source,exe,server,client,test_common,test \ + --include_dir_order envoy,nighthawk,external/source/envoy,external,api,common,source,exe,server,client,test_common,grpcpp,test \ $1 $TO_CHECK # The include checker doesn't support per-file checking, so we only