diff --git a/api/envoy/config/retry/other_priority/BUILD b/api/envoy/config/retry/other_priority/BUILD new file mode 100644 index 0000000000000..cb0cfcd025394 --- /dev/null +++ b/api/envoy/config/retry/other_priority/BUILD @@ -0,0 +1,11 @@ +licenses(["notice"]) # Apache 2 + +load("//bazel:api_build_system.bzl", "api_proto_library_internal") + +api_proto_library_internal( + name = "other_priority", + srcs = ["other_priority_config.proto"], + deps = [ + "//envoy/api/v2/core:base", + ], +) diff --git a/api/envoy/config/retry/other_priority/other_priority_config.proto b/api/envoy/config/retry/other_priority/other_priority_config.proto new file mode 100644 index 0000000000000..0113417619ddb --- /dev/null +++ b/api/envoy/config/retry/other_priority/other_priority_config.proto @@ -0,0 +1,37 @@ +syntax = "proto3"; + +package envoy.config.retry.other_priority; + +// A retry host selector that attempts to spread retries between priorities, even if certain +// priorities would not normally be attempted due to higher priorities being available. +// +// As priorities get excluded, load will be distributed amongst the remaining healthy priorities +// based on the relative health of the priorities, matching how load is distributed during regular +// host selection. For example, given priority healths of {100, 50, 50}, the original load will be +// {100, 0, 0} (since P0 has capacity to handle 100% of the traffic). If P0 is excluded, the load +// changes to {0, 50, 50}, because P1 is only able to handle 50% of the traffic, causing the +// remaining to spill over to P2. +// +// Each priority attempted will be excluded until there are no healthy priorities left, at which +// point the list of attempted priorities will be reset, essentially starting from the beginning. +// For example, given three priorities P0, P1, P2 with healthy % of 100, 0 and 50 respectively, the +// following sequence of priorities would be selected (assuming update_frequency = 1): +// Attempt 1: P0 (P0 is 100% healthy) +// Attempt 2: P2 (P0 already attempted, P2 only healthy priority) +// Attempt 3: P0 (no healthy priorities, reset) +// Attempt 4: P2 +// +// Using this PriorityFilter requires rebuilding the priority load, which runs in O(# of +// priorities), which might incur significant overhead for clusters with many priorities. +message OtherPriorityConfig { + // How often the priority load should be updated based on previously attempted priorities. Useful + // to allow each priorities to receive more than one request before being excluded or to reduce + // the number of times that the priority load has to be recomputed. + // + // For example, by setting this to 2, then the first two attempts (initial attempt and first + // retry) will use the unmodified priority load. The third and fourth attempt will use priority + // load which excludes the priorities routed to with the first two attempts, and the fifth and + // sixth attempt will use the priority load excluding the priorities used for the first four + // attempts. + int32 update_frequency = 1; +} diff --git a/include/envoy/upstream/retry.h b/include/envoy/upstream/retry.h index f448b3bc5b998..c601d0fc1fc54 100644 --- a/include/envoy/upstream/retry.h +++ b/include/envoy/upstream/retry.h @@ -109,6 +109,8 @@ class RetryPriorityFactory { const Protobuf::Message& config, uint32_t retry_count) PURE; virtual std::string name() const PURE; + + virtual ProtobufTypes::MessagePtr createEmptyConfigProto() PURE; }; /** @@ -125,6 +127,8 @@ class RetryHostPredicateFactory { * @return name name of this factory. */ virtual std::string name() PURE; + + virtual ProtobufTypes::MessagePtr createEmptyConfigProto() PURE; }; } // namespace Upstream diff --git a/source/common/router/config_impl.cc b/source/common/router/config_impl.cc index 035c08f67cd18..b9053a0bb6dd2 100644 --- a/source/common/router/config_impl.cc +++ b/source/common/router/config_impl.cc @@ -53,15 +53,21 @@ RetryPolicyImpl::RetryPolicyImpl(const envoy::api::v2::route::RouteAction& confi retry_on_ |= RetryStateImpl::parseRetryGrpcOn(config.retry_policy().retry_on()); for (const auto& host_predicate : config.retry_policy().retry_host_predicate()) { - Registry::FactoryRegistry::getFactory( - host_predicate.name()) - ->createHostPredicate(*this, host_predicate.config(), num_retries_); + auto& factory = + ::Envoy::Config::Utility::getAndCheckFactory( + host_predicate.name()); + + auto config = ::Envoy::Config::Utility::translateToFactoryConfig(host_predicate, factory); + factory.createHostPredicate(*this, *config, num_retries_); } const auto retry_priority = config.retry_policy().retry_priority(); if (!retry_priority.name().empty()) { - Registry::FactoryRegistry::getFactory(retry_priority.name()) - ->createRetryPriority(*this, retry_priority.config(), num_retries_); + auto& factory = ::Envoy::Config::Utility::getAndCheckFactory( + retry_priority.name()); + + auto config = ::Envoy::Config::Utility::translateToFactoryConfig(retry_priority, factory); + factory.createRetryPriority(*this, *config, num_retries_); } auto host_selection_attempts = config.retry_policy().host_selection_retry_max_attempts(); diff --git a/source/common/upstream/load_balancer_impl.cc b/source/common/upstream/load_balancer_impl.cc index de3bd6200a4de..54a0f91865fa9 100644 --- a/source/common/upstream/load_balancer_impl.cc +++ b/source/common/upstream/load_balancer_impl.cc @@ -44,25 +44,29 @@ LoadBalancerBase::LoadBalancerBase(const PrioritySet& priority_set, ClusterStats common_config, healthy_panic_threshold, 100, 50)), priority_set_(priority_set) { for (auto& host_set : priority_set_.hostSetsPerPriority()) { - recalculatePerPriorityState(host_set->priority()); + recalculatePerPriorityState(host_set->priority(), priority_set_, per_priority_load_, + per_priority_health_); } - priority_set_.addMemberUpdateCb( - [this](uint32_t priority, const HostVector&, const HostVector&) -> void { - recalculatePerPriorityState(priority); - }); + priority_set_.addMemberUpdateCb([this](uint32_t priority, const HostVector&, + const HostVector&) -> void { + recalculatePerPriorityState(priority, priority_set_, per_priority_load_, per_priority_health_); + }); } -void LoadBalancerBase::recalculatePerPriorityState(uint32_t priority) { - per_priority_load_.resize(priority_set_.hostSetsPerPriority().size()); - per_priority_health_.resize(priority_set_.hostSetsPerPriority().size()); +void LoadBalancerBase::recalculatePerPriorityState(uint32_t priority, + const PrioritySet& priority_set, + PriorityLoad& per_priority_load, + std::vector& per_priority_health) { + per_priority_load.resize(priority_set.hostSetsPerPriority().size()); + per_priority_health.resize(priority_set.hostSetsPerPriority().size()); // Determine the health of the newly modified priority level. // Health ranges from 0-100, and is the ratio of healthy hosts to total hosts, modified by the // overprovisioning factor. - HostSet& host_set = *priority_set_.hostSetsPerPriority()[priority]; - per_priority_health_[priority] = 0; + HostSet& host_set = *priority_set.hostSetsPerPriority()[priority]; + per_priority_health[priority] = 0; if (host_set.hosts().size() > 0) { - per_priority_health_[priority] = + per_priority_health[priority] = std::min(100, (host_set.overprovisioning_factor() * host_set.healthyHosts().size() / host_set.hosts().size())); } @@ -74,32 +78,32 @@ void LoadBalancerBase::recalculatePerPriorityState(uint32_t priority) { // 3 host sets with 20% / 20% / 10% health they will get 40% / 40% / 20% load to ensure total load // adds up to 100. const uint32_t total_health = std::min( - std::accumulate(per_priority_health_.begin(), per_priority_health_.end(), 0), 100); + std::accumulate(per_priority_health.begin(), per_priority_health.end(), 0), 100); if (total_health == 0) { // Everything is terrible. Send all load to P=0. // In this one case sumEntries(per_priority_load_) != 100 since we sinkhole all traffic in P=0. - per_priority_load_[0] = 100; + per_priority_load[0] = 100; return; } size_t total_load = 100; int32_t first_healthy_priority = -1; - for (size_t i = 0; i < per_priority_health_.size(); ++i) { - if (first_healthy_priority < 0 && per_priority_health_[i] > 0) { + for (size_t i = 0; i < per_priority_health.size(); ++i) { + if (first_healthy_priority < 0 && per_priority_health[i] > 0) { first_healthy_priority = i; } // Now assign as much load as possible to the high priority levels and cease assigning load // when total_load runs out. - per_priority_load_[i] = - std::min(total_load, per_priority_health_[i] * 100 / total_health); - total_load -= per_priority_load_[i]; + per_priority_load[i] = + std::min(total_load, per_priority_health[i] * 100 / total_health); + total_load -= per_priority_load[i]; } if (total_load != 0) { ASSERT(first_healthy_priority != -1); // Account for rounding errors by assigning it to the first healthy priority. - ASSERT(total_load < per_priority_load_.size()); - per_priority_load_[first_healthy_priority] += total_load; + ASSERT(total_load < per_priority_load.size()); + per_priority_load[first_healthy_priority] += total_load; } } diff --git a/source/common/upstream/load_balancer_impl.h b/source/common/upstream/load_balancer_impl.h index e7a75b6d05f33..1a216a1885df9 100644 --- a/source/common/upstream/load_balancer_impl.h +++ b/source/common/upstream/load_balancer_impl.h @@ -70,11 +70,15 @@ class LoadBalancerBase : public LoadBalancer { // The priority-ordered set of hosts to use for load balancing. const PrioritySet& priority_set_; +public: // Called when a host set at the given priority level is updated. This updates - // per_priority_health_ for that priority level, and may update per_priority_load_ for all + // per_priority_health for that priority level, and may update per_priority_load for all // priority levels. - void recalculatePerPriorityState(uint32_t priority); + void static recalculatePerPriorityState(uint32_t priority, const PrioritySet& priority_set, + PriorityLoad& priority_load, + std::vector& per_priority_health); +protected: // The percentage load (0-100) for each priority level std::vector per_priority_load_; // The health (0-100) for each priority level. diff --git a/source/extensions/extensions_build_config.bzl b/source/extensions/extensions_build_config.bzl index f3d7f9ef60d25..d18bfd70061a5 100644 --- a/source/extensions/extensions_build_config.bzl +++ b/source/extensions/extensions_build_config.bzl @@ -112,6 +112,9 @@ EXTENSIONS = { # Retry host predicates "envoy.retry_host_predicates.other_hosts": "//source/extensions/retry/host/other_hosts:config", + + # Retry priorities + "envoy.retry_priorities.previous_priorities": "//source/extensions/retry/priority/other_priority:config", } WINDOWS_EXTENSIONS = { diff --git a/source/extensions/retry/host/other_hosts/config.h b/source/extensions/retry/host/other_hosts/config.h index 391b24b0169a2..b8bac31cc18cb 100644 --- a/source/extensions/retry/host/other_hosts/config.h +++ b/source/extensions/retry/host/other_hosts/config.h @@ -18,6 +18,10 @@ class OtherHostsRetryPredicateFactory : public Upstream::RetryHostPredicateFacto } std::string name() override { return RetryHostPredicateValues::get().PreviousHostsPredicate; } + + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return ProtobufTypes::MessagePtr{new Envoy::ProtobufWkt::Empty()}; + } }; } // namespace Host diff --git a/source/extensions/retry/priority/BUILD b/source/extensions/retry/priority/BUILD new file mode 100644 index 0000000000000..6156949edef64 --- /dev/null +++ b/source/extensions/retry/priority/BUILD @@ -0,0 +1,17 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_package", +) + +envoy_package() + +envoy_cc_library( + name = "well_known_names", + hdrs = ["well_known_names.h"], + deps = [ + "//source/common/singleton:const_singleton", + ], +) diff --git a/source/extensions/retry/priority/other_priority/BUILD b/source/extensions/retry/priority/other_priority/BUILD new file mode 100644 index 0000000000000..6014100d4b4a0 --- /dev/null +++ b/source/extensions/retry/priority/other_priority/BUILD @@ -0,0 +1,33 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_package", +) + +envoy_package() + +envoy_cc_library( + name = "other_priority_lib", + srcs = ["other_priority.cc"], + hdrs = ["other_priority.h"], + deps = [ + "//include/envoy/upstream:retry_interface", + "//source/common/upstream:load_balancer_lib", + ], +) + +envoy_cc_library( + name = "config", + srcs = ["config.cc"], + hdrs = ["config.h"], + deps = [ + ":other_priority_lib", + "//include/envoy/registry", + "//include/envoy/upstream:retry_interface", + "//source/common/protobuf", + "//source/extensions/retry/priority:well_known_names", + "@envoy_api//envoy/config/retry/other_priority:other_priority_cc", + ], +) diff --git a/source/extensions/retry/priority/other_priority/config.cc b/source/extensions/retry/priority/other_priority/config.cc new file mode 100644 index 0000000000000..345e482b048ab --- /dev/null +++ b/source/extensions/retry/priority/other_priority/config.cc @@ -0,0 +1,28 @@ +#include "extensions/retry/priority/other_priority/config.h" + +#include "envoy/config/retry/other_priority/other_priority_config.pb.validate.h" +#include "envoy/registry/registry.h" +#include "envoy/upstream/retry.h" + +namespace Envoy { +namespace Extensions { +namespace Retry { +namespace Priority { + +void OtherPriorityRetryPriorityFactory::createRetryPriority( + Upstream::RetryPriorityFactoryCallbacks& callbacks, const Protobuf::Message& config, + uint32_t max_retries) { + callbacks.addRetryPriority(std::make_shared( + MessageUtil::downcastAndValidate< + const envoy::config::retry::other_priority::OtherPriorityConfig&>(config) + .update_frequency(), + max_retries)); +} + +static Registry::RegisterFactory + register_; + +} // namespace Priority +} // namespace Retry +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/retry/priority/other_priority/config.h b/source/extensions/retry/priority/other_priority/config.h new file mode 100644 index 0000000000000..770ad8afa4580 --- /dev/null +++ b/source/extensions/retry/priority/other_priority/config.h @@ -0,0 +1,32 @@ +#pragma once + +#include "envoy/upstream/retry.h" + +#include "common/protobuf/protobuf.h" + +#include "extensions/retry/priority/other_priority/other_priority.h" +#include "extensions/retry/priority/well_known_names.h" + +namespace Envoy { +namespace Extensions { +namespace Retry { +namespace Priority { + +class OtherPriorityRetryPriorityFactory : public Upstream::RetryPriorityFactory { +public: + void createRetryPriority(Upstream::RetryPriorityFactoryCallbacks& callbacks, + const Protobuf::Message& config, uint32_t max_retries) override; + + std::string name() const override { + return RetryPriorityValues::get().PreviousPrioritiesRetryPriority; + } + + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return ProtobufTypes::MessagePtr(new ::Envoy::ProtobufWkt::Empty()); + } +}; + +} // namespace Priority +} // namespace Retry +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/retry/priority/other_priority/other_priority.cc b/source/extensions/retry/priority/other_priority/other_priority.cc new file mode 100644 index 0000000000000..1d6cf7a6cebe5 --- /dev/null +++ b/source/extensions/retry/priority/other_priority/other_priority.cc @@ -0,0 +1,95 @@ +#include "extensions/retry/priority/other_priority/other_priority.h" + +namespace Envoy { +namespace Extensions { +namespace Retry { +namespace Priority { +const Upstream::PriorityLoad& OtherPriorityRetryPriority::determinePriorityLoad( + const Upstream::PrioritySet& priority_set, + const Upstream::PriorityLoad& original_priority_load) { + // If we've not seen enough retries to modify the priority load, just + // return the original. + // If this retry should trigger an update, recalculate the priority load by excluding attempted + // priorities. + if (attempted_priorities_.size() < update_frequency_) { + return original_priority_load; + } else if (attempted_priorities_.size() % update_frequency_ == 0) { + if (excluded_priorities_.size() < priority_set.hostSetsPerPriority().size()) { + excluded_priorities_.resize(priority_set.hostSetsPerPriority().size()); + } + + for (const auto priority : attempted_priorities_) { + excluded_priorities_[priority] = true; + } + + adjustForAttemptedPriorities(priority_set); + } + + return per_priority_load_; +} + +void OtherPriorityRetryPriority::adjustForAttemptedPriorities( + const Upstream::PrioritySet& priority_set) { + for (auto& host_set : priority_set.hostSetsPerPriority()) { + recalculatePerPriorityState(host_set->priority(), priority_set); + } + + // If all priorities are unhealthy to begin with, there's nothing to do. + if (!std::accumulate(per_priority_load_.begin(), per_priority_load_.end(), 0)) { + return; + } + + auto adjustedHealthAndSum = adjustedHealth(); + // If there are no healthy priorities left, we reset the attempted priorities and recompute the + // adjusted health. + // This allows us to fall back to the unmodified priority load when we run out of priorites + // instead of failing to route requests. + if (adjustedHealthAndSum.second == 0) { + for (size_t i = 0; i < excluded_priorities_.size(); ++i) { + excluded_priorities_[i] = false; + } + attempted_priorities_.clear(); + adjustedHealthAndSum = adjustedHealth(); + } + + const auto& adjusted_per_priority_health = adjustedHealthAndSum.first; + auto total_health = adjustedHealthAndSum.second; + + std::fill(per_priority_load_.begin(), per_priority_load_.end(), 0); + // We then adjust the load by rebalancing priorities with the adjusted health values. + size_t total_load = 100; + // The outer loop is used to eliminate rounding errors: any remaining load will be assigned to the + // first healthy priority. + while (total_load != 0) { + for (size_t i = 0; i < adjusted_per_priority_health.size(); ++i) { + // Now assign as much load as possible to the high priority levels and cease assigning load + // when total_load runs out. + auto delta = + std::min(total_load, adjusted_per_priority_health[i] * 100 / total_health); + per_priority_load_[i] += delta; + total_load -= delta; + } + } +} + +std::pair, uint32_t> OtherPriorityRetryPriority::adjustedHealth() const { + // Create an adjusted health view of the priorities, where attempted priorities are + // given a zero weight. + uint32_t total_health = 0; + std::vector adjusted_per_priority_health(per_priority_health_.size()); + adjusted_per_priority_health.resize(per_priority_health_.size()); + + for (size_t i = 0; i < per_priority_health_.size(); ++i) { + if (!excluded_priorities_[i]) { + adjusted_per_priority_health[i] = per_priority_health_[i]; + total_health += per_priority_health_[i]; + } + } + + return {std::move(adjusted_per_priority_health), std::min(total_health, 100u)}; +} + +} // namespace Priority +} // namespace Retry +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/retry/priority/other_priority/other_priority.h b/source/extensions/retry/priority/other_priority/other_priority.h new file mode 100644 index 0000000000000..b8fcd8e9e12e0 --- /dev/null +++ b/source/extensions/retry/priority/other_priority/other_priority.h @@ -0,0 +1,50 @@ +#pragma once + +#include "envoy/upstream/retry.h" + +#include "common/upstream/load_balancer_impl.h" + +namespace Envoy { +namespace Extensions { +namespace Retry { +namespace Priority { + +class OtherPriorityRetryPriority : public Upstream::RetryPriority { +public: + OtherPriorityRetryPriority(uint32_t update_frequency, uint32_t max_retries) + : update_frequency_(update_frequency) { + attempted_priorities_.reserve(max_retries); + } + + const Upstream::PriorityLoad& + determinePriorityLoad(const Upstream::PrioritySet& priority_set, + const Upstream::PriorityLoad& original_priority) override; + + void onHostAttempted(Upstream::HostDescriptionConstSharedPtr attempted_host) override { + attempted_priorities_.emplace_back(attempted_host->priority()); + } + +private: + void recalculatePerPriorityState(uint32_t priority, const Upstream::PrioritySet& priority_set) { + // Recalcuate health and priority the same way the load balancer does it. + Upstream::LoadBalancerBase::recalculatePerPriorityState( + priority, priority_set, per_priority_load_, per_priority_health_); + } + + std::pair, uint32_t> adjustedHealth() const; + + // Distributes priority load between priorities that should be considered after + // excluding attempted priorities. + void adjustForAttemptedPriorities(const Upstream::PrioritySet& priority_set); + + const uint32_t update_frequency_; + std::vector attempted_priorities_; + std::vector excluded_priorities_; + Upstream::PriorityLoad per_priority_load_; + std::vector per_priority_health_; +}; + +} // namespace Priority +} // namespace Retry +} // namespace Extensions +} // namespace Envoy diff --git a/source/extensions/retry/priority/well_known_names.h b/source/extensions/retry/priority/well_known_names.h new file mode 100644 index 0000000000000..70989080e450c --- /dev/null +++ b/source/extensions/retry/priority/well_known_names.h @@ -0,0 +1,24 @@ +#pragma once + +#include "common/singleton/const_singleton.h" + +namespace Envoy { +namespace Extensions { +namespace Retry { +namespace Priority { + +/** + * Well-known retry priority load names. + */ +class RetryPriorityNameValues { +public: + // Previous priority retry priority. Excludes previously attempted priorities during retries. + const std::string PreviousPrioritiesRetryPriority = "envoy.retry_priorities.previous_priorities"; +}; + +typedef ConstSingleton RetryPriorityValues; + +} // namespace Priority +} // namespace Retry +} // namespace Extensions +} // namespace Envoy diff --git a/test/extensions/retry/priority/other_priority/BUILD b/test/extensions/retry/priority/other_priority/BUILD new file mode 100644 index 0000000000000..59840223e9228 --- /dev/null +++ b/test/extensions/retry/priority/other_priority/BUILD @@ -0,0 +1,23 @@ +licenses(["notice"]) # Apache 2 + +load( + "//bazel:envoy_build_system.bzl", + "envoy_package", +) +load( + "//test/extensions:extensions_build_system.bzl", + "envoy_extension_cc_test", +) + +envoy_package() + +envoy_extension_cc_test( + name = "config_test", + srcs = ["config_test.cc"], + extension_name = "envoy.retry_priorities.previous_priorities", + deps = [ + "//source/extensions/retry/priority:well_known_names", + "//source/extensions/retry/priority/other_priority:config", + "//test/mocks/upstream:upstream_mocks", + ], +) diff --git a/test/extensions/retry/priority/other_priority/config_test.cc b/test/extensions/retry/priority/other_priority/config_test.cc new file mode 100644 index 0000000000000..b35a2a05d3230 --- /dev/null +++ b/test/extensions/retry/priority/other_priority/config_test.cc @@ -0,0 +1,157 @@ +#include "envoy/config/retry/other_priority/other_priority_config.pb.validate.h" +#include "envoy/registry/registry.h" +#include "envoy/upstream/retry.h" + +#include "extensions/retry/priority/other_priority/config.h" +#include "extensions/retry/priority/well_known_names.h" + +#include "test/mocks/upstream/mocks.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using namespace testing; + +namespace Envoy { +namespace Extensions { +namespace Retry { +namespace Priority { + +class RetryPriorityTest : public ::testing::Test, Upstream::RetryPriorityFactoryCallbacks { +public: + void initialize() { + auto factory = Registry::FactoryRegistry::getFactory( + RetryPriorityValues::get().PreviousPrioritiesRetryPriority); + + envoy::config::retry::other_priority::OtherPriorityConfig config; + config.set_update_frequency(update_frequency_); + factory->createRetryPriority(*this, config, 3); + } + + // Upstream::RetryPriorityFactoryCallbacks + void addRetryPriority(Upstream::RetryPrioritySharedPtr retry_priority) override { + retry_priority_ = retry_priority; + } + + void addHosts(size_t priority, int count, int healthy_count) { + auto host_set = priority_set_.getMockHostSet(priority); + + host_set->hosts_.resize(count); + host_set->healthy_hosts_.resize(healthy_count); + host_set->runCallbacks({}, {}); + } + + std::vector host_sets_; + uint32_t update_frequency_{1}; + NiceMock priority_set_; + Upstream::RetryPrioritySharedPtr retry_priority_; +}; + +TEST_F(RetryPriorityTest, DefaultFrequency) { + initialize(); + + const Upstream::PriorityLoad original_priority_load{100, 0}; + addHosts(0, 2, 2); + addHosts(1, 2, 2); + + auto host1 = std::make_shared>(); + ON_CALL(*host1, priority()).WillByDefault(Return(0)); + + auto host2 = std::make_shared>(); + ON_CALL(*host2, priority()).WillByDefault(Return(1)); + + // Before any hosts attempted, load should be unchanged. + ASSERT_EQ(original_priority_load, + retry_priority_->determinePriorityLoad(priority_set_, original_priority_load)); + + const Upstream::PriorityLoad expected_priority_load{0, 100}; + // After attempting a host in P0, P1 should receive all the load. + retry_priority_->onHostAttempted(host1); + ASSERT_EQ(expected_priority_load, + retry_priority_->determinePriorityLoad(priority_set_, original_priority_load)); + + // After we've tried host2, we've attempted all priorities and should reset back to the original + // priority load. + retry_priority_->onHostAttempted(host2); + ASSERT_EQ(original_priority_load, + retry_priority_->determinePriorityLoad(priority_set_, original_priority_load)); +} + +// Tests that spillover happens as we ignore attempted priorities. +TEST_F(RetryPriorityTest, DefaultFrequencyDegradedPriorities) { + initialize(); + + const Upstream::PriorityLoad original_priority_load{42, 28, 30}; + addHosts(0, 10, 3); + addHosts(1, 10, 2); + addHosts(2, 10, 10); + + auto host1 = std::make_shared>(); + ON_CALL(*host1, priority()).WillByDefault(Return(0)); + + auto host2 = std::make_shared>(); + ON_CALL(*host2, priority()).WillByDefault(Return(1)); + + auto host3 = std::make_shared>(); + ON_CALL(*host3, priority()).WillByDefault(Return(2)); + + // Before any hosts attempted, load should be unchanged. + ASSERT_EQ(original_priority_load, + retry_priority_->determinePriorityLoad(priority_set_, original_priority_load)); + + { + // After attempting a host in P0, load should be split between P1 and P2 since P1 is degraded. + const Upstream::PriorityLoad expected_priority_load{0, 28, 72}; + retry_priority_->onHostAttempted(host1); + ASSERT_EQ(expected_priority_load, + retry_priority_->determinePriorityLoad(priority_set_, original_priority_load)); + } + + // After we've tried host2, everything should go to P2. + const Upstream::PriorityLoad expected_priority_load{0, 0, 100}; + retry_priority_->onHostAttempted(host2); + ASSERT_EQ(expected_priority_load, + retry_priority_->determinePriorityLoad(priority_set_, original_priority_load)); + + // Once we've exhausted all priorities, we should return to the originial load. + retry_priority_->onHostAttempted(host3); + ASSERT_EQ(original_priority_load, + retry_priority_->determinePriorityLoad(priority_set_, original_priority_load)); +} + +// Tests that we can override the frequency at which we update the priority load with the +// update_frequency parameter. +TEST_F(RetryPriorityTest, OverridenFrequency) { + update_frequency_ = 2; + initialize(); + + const Upstream::PriorityLoad original_priority_load{100, 0}; + addHosts(0, 2, 2); + addHosts(1, 2, 2); + + auto host1 = std::make_shared>(); + ON_CALL(*host1, priority()).WillByDefault(Return(0)); + + auto host2 = std::make_shared>(); + ON_CALL(*host2, priority()).WillByDefault(Return(1)); + + // Before any hosts attempted, load should be unchanged. + ASSERT_EQ(original_priority_load, + retry_priority_->determinePriorityLoad(priority_set_, original_priority_load)); + + // After attempting a single host in P0, we should leave the priority load unchanged. + retry_priority_->onHostAttempted(host1); + ASSERT_EQ(original_priority_load, + retry_priority_->determinePriorityLoad(priority_set_, original_priority_load)); + + // After a second attempt, the prioity load should change. + const Upstream::PriorityLoad expected_priority_load{0, 100}; + retry_priority_->onHostAttempted(host1); + ASSERT_EQ(expected_priority_load, + retry_priority_->determinePriorityLoad(priority_set_, original_priority_load)); +} + +} // namespace Priority +} // namespace Retry +} // namespace Extensions +} // namespace Envoy diff --git a/test/integration/test_host_predicate_config.h b/test/integration/test_host_predicate_config.h index 47b092658d4e2..a8668e1b883f6 100644 --- a/test/integration/test_host_predicate_config.h +++ b/test/integration/test_host_predicate_config.h @@ -13,5 +13,8 @@ class TestHostPredicateFactory : public Upstream::RetryHostPredicateFactory { const Protobuf::Message&, uint32_t) override { callbacks.addHostPredicate(std::make_shared()); } + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return ProtobufTypes::MessagePtr{new Envoy::ProtobufWkt::Empty()}; + } }; } // namespace Envoy diff --git a/test/mocks/upstream/mocks.h b/test/mocks/upstream/mocks.h index db7b7f34c69c1..d6e51b61c5da4 100644 --- a/test/mocks/upstream/mocks.h +++ b/test/mocks/upstream/mocks.h @@ -124,6 +124,9 @@ class MockRetryPriorityFactory : public RetryPriorityFactory { } std::string name() const override { return "envoy.mock_retry_priority"; } + ProtobufTypes::MessagePtr createEmptyConfigProto() override { + return ProtobufTypes::MessagePtr{new Envoy::ProtobufWkt::Empty()}; + } private: RetryPrioritySharedPtr retry_priority_;