From 3293441cae611a0fee7607a23f32d5c1f117ec08 Mon Sep 17 00:00:00 2001 From: Snow Pettersen Date: Sun, 28 Apr 2019 12:25:32 -0400 Subject: [PATCH 01/28] upstream: exclude hosts from lb calculations until first active hc result Signed-off-by: Snow Pettersen --- include/envoy/upstream/upstream.h | 11 +++ .../upstream/health_discovery_service.cc | 2 +- source/common/upstream/load_balancer_impl.cc | 6 +- .../common/upstream/original_dst_cluster.cc | 4 +- source/common/upstream/subset_lb.cc | 2 +- source/common/upstream/subset_lb.h | 2 +- source/common/upstream/upstream_impl.cc | 66 +++++++++---- source/common/upstream/upstream_impl.h | 24 +++-- .../upstream/load_balancer_impl_test.cc | 92 +++++-------------- test/common/upstream/upstream_impl_test.cc | 26 +++--- test/mocks/upstream/host.cc | 1 + test/mocks/upstream/host.h | 1 + test/mocks/upstream/mocks.cc | 1 + test/mocks/upstream/mocks.h | 7 +- 14 files changed, 130 insertions(+), 115 deletions(-) diff --git a/include/envoy/upstream/upstream.h b/include/envoy/upstream/upstream.h index 9841ec7d09328..764a02010bdde 100644 --- a/include/envoy/upstream/upstream.h +++ b/include/envoy/upstream/upstream.h @@ -190,6 +190,8 @@ class Host : virtual public HostDescription { * @param new_used supplies the new value of host being in use to be stored. */ virtual void used(bool new_used) PURE; + + virtual bool warmed() const PURE; }; typedef std::shared_ptr HostConstSharedPtr; @@ -326,6 +328,12 @@ class HostSet { * @return uint32_t the overprovisioning factor of this host set. */ virtual uint32_t overprovisioningFactor() const PURE; + + /** + * @return uint32_t the number of warmed hosts in this host set. A host is considered warming + * if active health checking is enabled and the host has yet to be health checked for the first time. + */ + virtual uint32_t warmedHostCount() const PURE; }; typedef std::unique_ptr HostSetPtr; @@ -381,6 +389,7 @@ class PrioritySet { HostsPerLocalityConstSharedPtr hosts_per_locality; HostsPerLocalityConstSharedPtr healthy_hosts_per_locality; HostsPerLocalityConstSharedPtr degraded_hosts_per_locality; + std::shared_ptr> warmed_hosts_per_locality; }; /** @@ -396,6 +405,7 @@ class PrioritySet { virtual void updateHosts(uint32_t priority, UpdateHostsParams&& update_host_params, LocalityWeightsConstSharedPtr locality_weights, const HostVector& hosts_added, const HostVector& hosts_removed, + uint32_t warmed_host_count, absl::optional overprovisioning_factor) PURE; /** @@ -417,6 +427,7 @@ class PrioritySet { virtual void updateHosts(uint32_t priority, UpdateHostsParams&& update_host_params, LocalityWeightsConstSharedPtr locality_weights, const HostVector& hosts_added, const HostVector& hosts_removed, + uint32_t warmed_host_count, absl::optional overprovisioning_factor) PURE; }; diff --git a/source/common/upstream/health_discovery_service.cc b/source/common/upstream/health_discovery_service.cc index d104759accdb8..fbb72d4488251 100644 --- a/source/common/upstream/health_discovery_service.cc +++ b/source/common/upstream/health_discovery_service.cc @@ -258,7 +258,7 @@ void HdsCluster::initialize(std::function callback) { priority_set_.updateHosts( 0, HostSetImpl::partitionHosts(initial_hosts_, HostsPerLocalityImpl::empty()), {}, - *initial_hosts_, {}, absl::nullopt); + *initial_hosts_, {}, initial_hosts_->size(), absl::nullopt); } void HdsCluster::setOutlierDetector(const Outlier::DetectorSharedPtr&) { diff --git a/source/common/upstream/load_balancer_impl.cc b/source/common/upstream/load_balancer_impl.cc index 838d2fb32f0a6..83534cf26914f 100644 --- a/source/common/upstream/load_balancer_impl.cc +++ b/source/common/upstream/load_balancer_impl.cc @@ -136,7 +136,7 @@ void LoadBalancerBase::recalculatePerPriorityState(uint32_t priority, // by the overprovisioning factor. HostSet& host_set = *priority_set.hostSetsPerPriority()[priority]; per_priority_health.get()[priority] = 0; - if (!host_set.hosts().empty()) { + if (!host_set.hosts().empty() && host_set.warmedHostCount() > 0) { // Each priority level's health is ratio of healthy hosts to total number of hosts in a priority // multiplied by overprovisioning factor of 1.4 and capped at 100%. It means that if all // hosts are healthy that priority's health is 100%*1.4=140% and is capped at 100% which results @@ -144,12 +144,12 @@ void LoadBalancerBase::recalculatePerPriorityState(uint32_t priority, // capped at 100%). per_priority_health.get()[priority] = std::min(100, (host_set.overprovisioningFactor() * - host_set.healthyHosts().size() / host_set.hosts().size())); + host_set.healthyHosts().size() / host_set.warmedHostCount())); // We perform the same computation for degraded hosts. per_priority_degraded.get()[priority] = std::min(100, (host_set.overprovisioningFactor() * - host_set.degradedHosts().size() / host_set.hosts().size())); + host_set.degradedHosts().size() / host_set.warmedHostCount())); } // Now that we've updated health for the changed priority level, we need to calculate percentage diff --git a/source/common/upstream/original_dst_cluster.cc b/source/common/upstream/original_dst_cluster.cc index f2a0af8911efc..116d4234d09f9 100644 --- a/source/common/upstream/original_dst_cluster.cc +++ b/source/common/upstream/original_dst_cluster.cc @@ -153,7 +153,7 @@ void OriginalDstCluster::addHost(HostSharedPtr& host) { new_hosts->emplace_back(host); priority_set_.updateHosts(0, HostSetImpl::partitionHosts(new_hosts, HostsPerLocalityImpl::empty()), - {}, {std::move(host)}, {}, absl::nullopt); + {}, {std::move(host)}, {}, new_hosts->size(), absl::nullopt); } void OriginalDstCluster::cleanup() { @@ -180,7 +180,7 @@ void OriginalDstCluster::cleanup() { if (!to_be_removed.empty()) { priority_set_.updateHosts(0, HostSetImpl::partitionHosts(new_hosts, HostsPerLocalityImpl::empty()), - {}, {}, to_be_removed, absl::nullopt); + {}, {}, to_be_removed, new_hosts->size(), absl::nullopt); } cleanup_timer_->enableTimer(cleanup_interval_ms_); diff --git a/source/common/upstream/subset_lb.cc b/source/common/upstream/subset_lb.cc index 0e47d3955410f..e053d661558bc 100644 --- a/source/common/upstream/subset_lb.cc +++ b/source/common/upstream/subset_lb.cc @@ -591,7 +591,7 @@ void SubsetLoadBalancer::HostSubsetImpl::update(const HostVector& hosts_added, hosts, hosts_per_locality, healthy_hosts, healthy_hosts_per_locality, degraded_hosts, degraded_hosts_per_locality), determineLocalityWeights(*hosts_per_locality), filtered_added, - filtered_removed); + filtered_removed, {}); } LocalityWeightsConstSharedPtr SubsetLoadBalancer::HostSubsetImpl::determineLocalityWeights( diff --git a/source/common/upstream/subset_lb.h b/source/common/upstream/subset_lb.h index 3980cce1330a1..74bd9dccee148 100644 --- a/source/common/upstream/subset_lb.h +++ b/source/common/upstream/subset_lb.h @@ -42,7 +42,7 @@ class SubsetLoadBalancer : public LoadBalancer, Logger::Loggable HostsPerLocalityImpl::filter( void HostSetImpl::updateHosts(PrioritySet::UpdateHostsParams&& update_hosts_params, LocalityWeightsConstSharedPtr locality_weights, const HostVector& hosts_added, const HostVector& hosts_removed, + uint32_t warmed_host_count, absl::optional overprovisioning_factor) { if (overprovisioning_factor.has_value()) { ASSERT(overprovisioning_factor.value() > 0); @@ -323,14 +324,18 @@ void HostSetImpl::updateHosts(PrioritySet::UpdateHostsParams&& update_hosts_para hosts_per_locality_ = std::move(update_hosts_params.hosts_per_locality); healthy_hosts_per_locality_ = std::move(update_hosts_params.healthy_hosts_per_locality); degraded_hosts_per_locality_ = std::move(update_hosts_params.degraded_hosts_per_locality); + warmed_counts_per_locality_ = std::move(update_hosts_params.warmed_hosts_per_locality); locality_weights_ = std::move(locality_weights); + warmed_host_count_ = warmed_host_count; rebuildLocalityScheduler(healthy_locality_scheduler_, healthy_locality_entries_, *healthy_hosts_per_locality_, healthy_hosts_->get(), hosts_per_locality_, - locality_weights_, overprovisioning_factor_); + warmed_counts_per_locality_, locality_weights_, + overprovisioning_factor_); rebuildLocalityScheduler(degraded_locality_scheduler_, degraded_locality_entries_, *degraded_hosts_per_locality_, degraded_hosts_->get(), - hosts_per_locality_, locality_weights_, overprovisioning_factor_); + hosts_per_locality_, warmed_counts_per_locality_, locality_weights_, + overprovisioning_factor_); runUpdateCallbacks(hosts_added, hosts_removed); } @@ -340,7 +345,9 @@ void HostSetImpl::rebuildLocalityScheduler( std::vector>& locality_entries, const HostsPerLocality& eligible_hosts_per_locality, const HostVector& eligible_hosts, HostsPerLocalityConstSharedPtr all_hosts_per_locality, + std::shared_ptr> warmed_counts_per_locality, LocalityWeightsConstSharedPtr locality_weights, uint32_t overprovisioning_factor) { + ASSERT((all_hosts_per_locality == nullptr) == (warmed_counts_per_locality == nullptr)); // Rebuild the locality scheduler by computing the effective weight of each // locality in this priority. The scheduler is reset by default, and is rebuilt only if we have // locality weights (i.e. using EDS) and there is at least one eligible host in this priority. @@ -363,7 +370,7 @@ void HostSetImpl::rebuildLocalityScheduler( locality_entries.clear(); for (uint32_t i = 0; i < all_hosts_per_locality->get().size(); ++i) { const double effective_weight = - effectiveLocalityWeight(i, eligible_hosts_per_locality, *all_hosts_per_locality, + effectiveLocalityWeight(i, eligible_hosts_per_locality, *warmed_counts_per_locality, *locality_weights, overprovisioning_factor); if (effective_weight > 0) { locality_entries.emplace_back(std::make_shared(i, effective_weight)); @@ -412,6 +419,7 @@ HostSetImpl::updateHostsParams(HostVectorConstSharedPtr hosts, HostsPerLocalityConstSharedPtr hosts_per_locality, HealthyHostVectorConstSharedPtr healthy_hosts, HostsPerLocalityConstSharedPtr healthy_hosts_per_locality) { + return updateHostsParams(std::move(hosts), std::move(hosts_per_locality), std::move(healthy_hosts), std::move(healthy_hosts_per_locality), std::make_shared(), @@ -425,12 +433,20 @@ HostSetImpl::updateHostsParams(HostVectorConstSharedPtr hosts, HostsPerLocalityConstSharedPtr healthy_hosts_per_locality, DegradedHostVectorConstSharedPtr degraded_hosts, HostsPerLocalityConstSharedPtr degraded_hosts_per_locality) { + auto warmed_counts_per_locality = std::make_shared>(); + + for (const auto& hosts : hosts_per_locality->get()) { + warmed_counts_per_locality->emplace_back( + std::count_if(hosts.begin(), hosts.end(), [](const auto& host) { return host->warmed(); })); + } + return PrioritySet::UpdateHostsParams{std::move(hosts), std::move(healthy_hosts), std::move(degraded_hosts), std::move(hosts_per_locality), std::move(healthy_hosts_per_locality), - std::move(degraded_hosts_per_locality)}; + std::move(degraded_hosts_per_locality), + std::move(warmed_counts_per_locality)}; } PrioritySet::UpdateHostsParams @@ -449,16 +465,15 @@ HostSetImpl::partitionHosts(HostVectorConstSharedPtr hosts, double HostSetImpl::effectiveLocalityWeight(uint32_t index, const HostsPerLocality& eligible_hosts_per_locality, - const HostsPerLocality& all_hosts_per_locality, + const std::vector& warmed_count_per_locality, const LocalityWeights& locality_weights, uint32_t overprovisioning_factor) { - const auto& locality_hosts = all_hosts_per_locality.get()[index]; const auto& locality_eligible_hosts = eligible_hosts_per_locality.get()[index]; - if (locality_hosts.empty()) { + const auto warmed_count = warmed_count_per_locality[index]; + if (warmed_count == 0) { return 0.0; } - const double locality_availability_ratio = - 1.0 * locality_eligible_hosts.size() / locality_hosts.size(); + const double locality_availability_ratio = 1.0 * locality_eligible_hosts.size() / warmed_count; const uint32_t weight = locality_weights[index]; // Availability ranges from 0-1.0, and is the ratio of eligible hosts to total hosts, modified by // the overprovisioning factor. @@ -486,12 +501,13 @@ PrioritySetImpl::getOrCreateHostSet(uint32_t priority, void PrioritySetImpl::updateHosts(uint32_t priority, UpdateHostsParams&& update_hosts_params, LocalityWeightsConstSharedPtr locality_weights, const HostVector& hosts_added, const HostVector& hosts_removed, + uint32_t warmed_host_count, absl::optional overprovisioning_factor) { // Ensure that we have a HostSet for the given priority. getOrCreateHostSet(priority, overprovisioning_factor); static_cast(host_sets_[priority].get()) ->updateHosts(std::move(update_hosts_params), std::move(locality_weights), hosts_added, - hosts_removed, overprovisioning_factor); + hosts_removed, warmed_host_count, overprovisioning_factor); if (!batch_update_) { runUpdateCallbacks(hosts_added, hosts_removed); @@ -514,7 +530,8 @@ void PrioritySetImpl::batchHostUpdate(BatchUpdateCb& callback) { void PrioritySetImpl::BatchUpdateScope::updateHosts( uint32_t priority, PrioritySet::UpdateHostsParams&& update_hosts_params, LocalityWeightsConstSharedPtr locality_weights, const HostVector& hosts_added, - const HostVector& hosts_removed, absl::optional overprovisioning_factor) { + const HostVector& hosts_removed, uint32_t warmed_host_count, + absl::optional overprovisioning_factor) { // We assume that each call updates a different priority. ASSERT(priorities_.find(priority) == priorities_.end()); priorities_.insert(priority); @@ -528,7 +545,7 @@ void PrioritySetImpl::BatchUpdateScope::updateHosts( } parent_.updateHosts(priority, std::move(update_hosts_params), locality_weights, hosts_added, - hosts_removed, overprovisioning_factor); + hosts_removed, warmed_host_count, overprovisioning_factor); } ClusterStats ClusterInfoImpl::generateStats(Stats::Scope& scope) { @@ -828,10 +845,18 @@ void ClusterImplBase::reloadHealthyHostsHelper(const HostSharedPtr&) { const auto& host_set = host_sets[priority]; // TODO(htuch): Can we skip these copies by exporting out const shared_ptr from HostSet? HostVectorConstSharedPtr hosts_copy(new HostVector(host_set->hosts())); + + uint32_t warmed_hosts_count = host_set->hosts().size(); + if (health_checker_ != nullptr) { + warmed_hosts_count -= + std::count_if(host_set->hosts().begin(), host_set->hosts().end(), [](const auto& host) { + return host->getActiveHealthFailureType() == Host::ActiveHealthFailureType::UNKNOWN; + }); + } HostsPerLocalityConstSharedPtr hosts_per_locality_copy = host_set->hostsPerLocality().clone(); - prioritySet().updateHosts(priority, - HostSetImpl::partitionHosts(hosts_copy, hosts_per_locality_copy), - host_set->localityWeights(), {}, {}, absl::nullopt); + prioritySet().updateHosts( + priority, HostSetImpl::partitionHosts(hosts_copy, hosts_per_locality_copy), + host_set->localityWeights(), {}, {}, warmed_hosts_count, absl::nullopt); } } @@ -991,7 +1016,13 @@ void PriorityStateManager::updateClusterPrioritySet( // We use std::map to guarantee a stable ordering for zone aware routing. std::map hosts_per_locality; + uint32_t warmed_host_count = hosts->size(); + for (const HostSharedPtr& host : *hosts) { + if (parent_.healthChecker() != nullptr && + host->getActiveHealthFailureType() == Host::ActiveHealthFailureType::UNKNOWN) { + warmed_host_count--; + } // Take into consideration when a non-EDS cluster has active health checking, i.e. to mark all // the hosts unhealthy (host->healthFlagSet(Host::HealthFlag::FAILED_ACTIVE_HC)) and then fire // update callbacks to start the health checking process. @@ -1034,12 +1065,13 @@ void PriorityStateManager::updateClusterPrioritySet( if (update_cb_ != nullptr) { update_cb_->updateHosts(priority, HostSetImpl::partitionHosts(hosts, per_locality_shared), std::move(locality_weights), hosts_added.value_or(*hosts), - hosts_removed.value_or({}), overprovisioning_factor); + hosts_removed.value_or({}), warmed_host_count, + overprovisioning_factor); } else { parent_.prioritySet().updateHosts( priority, HostSetImpl::partitionHosts(hosts, per_locality_shared), std::move(locality_weights), hosts_added.value_or(*hosts), - hosts_removed.value_or({}), overprovisioning_factor); + hosts_removed.value_or({}), warmed_host_count, overprovisioning_factor); } } diff --git a/source/common/upstream/upstream_impl.h b/source/common/upstream/upstream_impl.h index 5bbf793e7bf4b..cc2bcb149af6e 100644 --- a/source/common/upstream/upstream_impl.h +++ b/source/common/upstream/upstream_impl.h @@ -224,6 +224,10 @@ class HostImpl : public HostDescriptionImpl, void weight(uint32_t new_weight) override; bool used() const override { return used_; } void used(bool new_used) override { used_ = new_used; } + bool warmed() const override { + return !(healthFlagGet(HealthFlag::FAILED_ACTIVE_HC) && + getActiveHealthFailureType() == ActiveHealthFailureType::UNKNOWN); + } protected: static Network::ClientConnectionPtr @@ -277,12 +281,13 @@ class HostsPerLocalityImpl : public HostsPerLocality { */ class HostSetImpl : public HostSet { public: - HostSetImpl(uint32_t priority, absl::optional overprovisioning_factor) + HostSetImpl(uint32_t priority, uint32_t warmed_host_count, + absl::optional overprovisioning_factor) : priority_(priority), overprovisioning_factor_(overprovisioning_factor.has_value() ? overprovisioning_factor.value() : kDefaultOverProvisioningFactor), - hosts_(new HostVector()), healthy_hosts_(new HealthyHostVector()), - degraded_hosts_(new DegradedHostVector()) {} + warmed_host_count_(warmed_host_count), hosts_(new HostVector()), + healthy_hosts_(new HealthyHostVector()), degraded_hosts_(new DegradedHostVector()) {} /** * Install a callback that will be invoked when the host set membership changes. @@ -309,6 +314,7 @@ class HostSetImpl : public HostSet { absl::optional chooseDegradedLocality() override; uint32_t priority() const override { return priority_; } uint32_t overprovisioningFactor() const override { return overprovisioning_factor_; } + uint32_t warmedHostCount() const override { return warmed_host_count_; } // Utility methods for creating UpdateHostsParams. static PrioritySet::UpdateHostsParams @@ -331,7 +337,7 @@ class HostSetImpl : public HostSet { void updateHosts(PrioritySet::UpdateHostsParams&& update_hosts_params, LocalityWeightsConstSharedPtr locality_weights, const HostVector& hosts_added, - const HostVector& hosts_removed, + const HostVector& hosts_removed, uint32_t warmed_host_count, absl::optional overprovisioning_factor = absl::nullopt); protected: @@ -344,18 +350,20 @@ class HostSetImpl : public HostSet { // locality. static double effectiveLocalityWeight(uint32_t index, const HostsPerLocality& eligible_hosts_per_locality, - const HostsPerLocality& all_hosts_per_locality, + const std::vector& warmed_counts_per_locality, const LocalityWeights& locality_weights, uint32_t overprovisioning_factor); uint32_t priority_; uint32_t overprovisioning_factor_; + uint32_t warmed_host_count_; HostVectorConstSharedPtr hosts_; HealthyHostVectorConstSharedPtr healthy_hosts_; DegradedHostVectorConstSharedPtr degraded_hosts_; HostsPerLocalityConstSharedPtr hosts_per_locality_{HostsPerLocalityImpl::empty()}; HostsPerLocalityConstSharedPtr healthy_hosts_per_locality_{HostsPerLocalityImpl::empty()}; HostsPerLocalityConstSharedPtr degraded_hosts_per_locality_{HostsPerLocalityImpl::empty()}; + std::shared_ptr> warmed_counts_per_locality_{}; // TODO(mattklein123): Remove mutable. mutable Common::CallbackManager member_update_cb_helper_; @@ -387,6 +395,7 @@ class HostSetImpl : public HostSet { std::vector>& locality_entries, const HostsPerLocality& eligible_hosts_per_locality, const HostVector& eligible_hosts, HostsPerLocalityConstSharedPtr all_hosts_per_locality, + std::shared_ptr> warmed_counts_per_locality, LocalityWeightsConstSharedPtr locality_weights, uint32_t overprovisioning_factor); static absl::optional chooseLocality(EdfScheduler* locality_scheduler); @@ -422,7 +431,7 @@ class PrioritySetImpl : public PrioritySet { void updateHosts(uint32_t priority, UpdateHostsParams&& update_hosts_params, LocalityWeightsConstSharedPtr locality_weights, const HostVector& hosts_added, - const HostVector& hosts_removed, + const HostVector& hosts_removed, uint32_t warmed_host_count, absl::optional overprovisioning_factor = absl::nullopt) override; void batchHostUpdate(BatchUpdateCb& callback) override; @@ -431,7 +440,7 @@ class PrioritySetImpl : public PrioritySet { // Allows subclasses of PrioritySetImpl to create their own type of HostSetImpl. virtual HostSetImplPtr createHostSet(uint32_t priority, absl::optional overprovisioning_factor) { - return std::make_unique(priority, overprovisioning_factor); + return std::make_unique(priority, 0, overprovisioning_factor); } protected: @@ -469,6 +478,7 @@ class PrioritySetImpl : public PrioritySet { PrioritySet::UpdateHostsParams&& update_hosts_params, LocalityWeightsConstSharedPtr locality_weights, const HostVector& hosts_added, const HostVector& hosts_removed, + uint32_t warmed_host_count, absl::optional overprovisioning_factor) override; std::unordered_set all_hosts_added_; diff --git a/test/common/upstream/load_balancer_impl_test.cc b/test/common/upstream/load_balancer_impl_test.cc index 801064893fa05..82b6d44b64be7 100644 --- a/test/common/upstream/load_balancer_impl_test.cc +++ b/test/common/upstream/load_balancer_impl_test.cc @@ -133,7 +133,7 @@ TEST_P(LoadBalancerBaseTest, PrioritySelection) { MockHostSet& tertiary_host_set_ = *priority_set_.getMockHostSet(2); updateHostSet(tertiary_host_set_, 1 /* num_hosts */, 1 /* num_healthy_hosts */); EXPECT_EQ(0, lb_.percentageLoad(0)); - EXPECT_EQ(0, lb_.percentageLoad(1)); + EXPECT_EQ(0, lb_.percentageLoad(1)); EXPECT_EQ(100, lb_.percentageLoad(2)); priority_load.healthy_priority_load_ = HealthyLoad({0u, 0u, 100}); EXPECT_EQ(&tertiary_host_set_, &lb_.chooseHostSet(&context).first); @@ -435,11 +435,22 @@ class RoundRobinLoadBalancerTest : public LoadBalancerTestBase { random_, common_config_)); } + // Updates priority 0 with the given hosts and hosts_per_locality. + void updateHosts(HostVectorConstSharedPtr hosts, + HostsPerLocalityConstSharedPtr hosts_per_locality) { + local_priority_set_->updateHosts( + 0, + HostSetImpl::updateHostsParams(hosts, hosts_per_locality, + std::make_shared(*hosts), + hosts_per_locality), + {}, empty_host_vector_, empty_host_vector_, hosts->size(), absl::nullopt); + } + std::shared_ptr local_priority_set_; std::shared_ptr lb_; HostsPerLocalityConstSharedPtr empty_locality_; HostVector empty_host_vector_; -}; +}; // namespace // For the tests which mutate primary and failover host sets explicitly, only // run once. @@ -553,12 +564,7 @@ TEST_P(FailoverTest, ExtendPrioritiesWithLocalPrioritySet) { // Update the local hosts. We're not doing locality based routing in this // test, but it should at least do no harm. HostVectorSharedPtr hosts(new HostVector({makeTestHost(info_, "tcp://127.0.0.1:82")})); - local_priority_set_->updateHosts( - 0, - HostSetImpl::updateHostsParams(hosts, HostsPerLocalityImpl::empty(), - std::make_shared(*hosts), - HostsPerLocalityImpl::empty()), - {}, empty_host_vector_, empty_host_vector_, absl::nullopt); + updateHosts(hosts, HostsPerLocalityImpl::empty()); EXPECT_EQ(tertiary_host_set_.hosts_[0], lb_->chooseHost(nullptr)); } @@ -825,12 +831,7 @@ TEST_P(RoundRobinLoadBalancerTest, ZoneAwareSmallCluster) { common_config_.mutable_zone_aware_lb_config()->mutable_routing_enabled()->set_value(98); common_config_.mutable_zone_aware_lb_config()->mutable_min_cluster_size()->set_value(7); init(true); - local_priority_set_->updateHosts( - 0, - HostSetImpl::updateHostsParams(hosts, hosts_per_locality, - std::make_shared(*hosts), - hosts_per_locality), - {}, empty_host_vector_, empty_host_vector_, absl::nullopt); + updateHosts(hosts, hosts_per_locality); EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 0)) .WillRepeatedly(Return(50)); @@ -853,12 +854,7 @@ TEST_P(RoundRobinLoadBalancerTest, ZoneAwareSmallCluster) { EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.zone_routing.min_cluster_size", 7)) .WillRepeatedly(Return(1)); // Trigger reload. - local_priority_set_->updateHosts( - 0, - HostSetImpl::updateHostsParams(hosts, hosts_per_locality, - std::make_shared(*hosts), - hosts_per_locality), - {}, empty_host_vector_, empty_host_vector_, absl::nullopt); + updateHosts(hosts, hosts_per_locality); EXPECT_EQ(hostSet().healthy_hosts_per_locality_->get()[0][0], lb_->chooseHost(nullptr)); } @@ -883,12 +879,7 @@ TEST_P(RoundRobinLoadBalancerTest, NoZoneAwareDifferentZoneSize) { common_config_.mutable_zone_aware_lb_config()->mutable_routing_enabled()->set_value(98); common_config_.mutable_zone_aware_lb_config()->mutable_min_cluster_size()->set_value(7); init(true); - local_priority_set_->updateHosts( - 0, - HostSetImpl::updateHostsParams(hosts, local_hosts_per_locality, - std::make_shared(*hosts), - local_hosts_per_locality), - {}, empty_host_vector_, empty_host_vector_, absl::nullopt); + updateHosts(hosts, local_hosts_per_locality); EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 100)) .WillRepeatedly(Return(50)); @@ -924,12 +915,7 @@ TEST_P(RoundRobinLoadBalancerTest, ZoneAwareRoutingLargeZoneSwitchOnOff) { hostSet().hosts_ = *hosts; hostSet().healthy_hosts_per_locality_ = hosts_per_locality; init(true); - local_priority_set_->updateHosts( - 0, - HostSetImpl::updateHostsParams(hosts, hosts_per_locality, - std::make_shared(*hosts), - hosts_per_locality), - {}, empty_host_vector_, empty_host_vector_, absl::nullopt); + updateHosts(hosts, hosts_per_locality); // There is only one host in the given zone for zone aware routing. EXPECT_EQ(hostSet().healthy_hosts_per_locality_->get()[0][0], lb_->chooseHost(nullptr)); @@ -976,12 +962,7 @@ TEST_P(RoundRobinLoadBalancerTest, ZoneAwareRoutingSmallZone) { hostSet().hosts_ = *upstream_hosts; hostSet().healthy_hosts_per_locality_ = upstream_hosts_per_locality; init(true); - local_priority_set_->updateHosts( - 0, - HostSetImpl::updateHostsParams(local_hosts, local_hosts_per_locality, - std::make_shared(*local_hosts), - local_hosts_per_locality), - {}, empty_host_vector_, empty_host_vector_, absl::nullopt); + updateHosts(local_hosts, local_hosts_per_locality); // There is only one host in the given zone for zone aware routing. EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(100)); @@ -1050,12 +1031,7 @@ TEST_P(RoundRobinLoadBalancerTest, LowPrecisionForDistribution) { // To trigger update callback. auto local_hosts_per_locality_shared = makeHostsPerLocality(std::move(local_hosts_per_locality)); - local_priority_set_->updateHosts( - 0, - HostSetImpl::updateHostsParams(local_hosts, local_hosts_per_locality_shared, - std::make_shared(*local_hosts), - local_hosts_per_locality_shared), - {}, empty_host_vector_, empty_host_vector_, absl::nullopt); + updateHosts(local_hosts, local_hosts_per_locality_shared); // Force request out of small zone and to randomly select zone. EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(9999)).WillOnce(Return(2)); @@ -1075,12 +1051,7 @@ TEST_P(RoundRobinLoadBalancerTest, NoZoneAwareRoutingOneZone) { hostSet().hosts_ = *hosts; hostSet().healthy_hosts_per_locality_ = hosts_per_locality; init(true); - local_priority_set_->updateHosts( - 0, - HostSetImpl::updateHostsParams(hosts, hosts_per_locality, - std::make_shared(*hosts), - hosts_per_locality), - {}, empty_host_vector_, empty_host_vector_, absl::nullopt); + updateHosts(hosts, hosts_per_locality); EXPECT_EQ(hostSet().healthy_hosts_[0], lb_->chooseHost(nullptr)); } @@ -1094,12 +1065,7 @@ TEST_P(RoundRobinLoadBalancerTest, NoZoneAwareRoutingNotHealthy) { hostSet().hosts_ = *hosts; hostSet().healthy_hosts_per_locality_ = hosts_per_locality; init(true); - local_priority_set_->updateHosts( - 0, - HostSetImpl::updateHostsParams(hosts, hosts_per_locality, - std::make_shared(*hosts), - hosts_per_locality), - {}, empty_host_vector_, empty_host_vector_, absl::nullopt); + updateHosts(hosts, hosts_per_locality); // local zone has no healthy hosts, take from the all healthy hosts. EXPECT_EQ(hostSet().healthy_hosts_[0], lb_->chooseHost(nullptr)); @@ -1129,12 +1095,7 @@ TEST_P(RoundRobinLoadBalancerTest, NoZoneAwareRoutingLocalEmpty) { hostSet().hosts_ = *upstream_hosts; hostSet().healthy_hosts_per_locality_ = upstream_hosts_per_locality; init(true); - local_priority_set_->updateHosts( - 0, - HostSetImpl::updateHostsParams(local_hosts, local_hosts_per_locality, - std::make_shared(*local_hosts), - local_hosts_per_locality), - {}, empty_host_vector_, empty_host_vector_, absl::nullopt); + updateHosts(local_hosts, local_hosts_per_locality); // Local cluster is not OK, we'll do regular routing. EXPECT_EQ(hostSet().healthy_hosts_[0], lb_->chooseHost(nullptr)); @@ -1161,12 +1122,7 @@ TEST_P(RoundRobinLoadBalancerTest, NoZoneAwareRoutingNoLocalLocality) { hostSet().hosts_ = *upstream_hosts; hostSet().healthy_hosts_per_locality_ = upstream_hosts_per_locality; init(true); - local_priority_set_->updateHosts( - 0, - HostSetImpl::updateHostsParams(local_hosts, local_hosts_per_locality, - std::make_shared(*local_hosts), - local_hosts_per_locality), - {}, empty_host_vector_, empty_host_vector_, absl::nullopt); + updateHosts(local_hosts, local_hosts_per_locality); // Local cluster is not OK, we'll do regular routing. EXPECT_EQ(hostSet().healthy_hosts_[0], lb_->chooseHost(nullptr)); diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc index 3f1218f2f4d3b..c8205887e1189 100644 --- a/test/common/upstream/upstream_impl_test.cc +++ b/test/common/upstream/upstream_impl_test.cc @@ -1663,7 +1663,7 @@ class TestBatchUpdateCb : public PrioritySet::BatchUpdateCb { HostSetImpl::updateHostsParams(hosts_, hosts_per_locality_, std::make_shared(*hosts_), hosts_per_locality_), - {}, hosts_added, hosts_removed, absl::nullopt); + {}, hosts_added, hosts_removed, hosts_->size(), absl::nullopt); } // Remove the host from P1. @@ -1676,7 +1676,7 @@ class TestBatchUpdateCb : public PrioritySet::BatchUpdateCb { HostSetImpl::updateHostsParams(empty_hosts, HostsPerLocalityImpl::empty(), std::make_shared(*empty_hosts), HostsPerLocalityImpl::empty()), - {}, hosts_added, hosts_removed, absl::nullopt); + {}, hosts_added, hosts_removed, hosts_->size(), absl::nullopt); } } @@ -1731,7 +1731,7 @@ TEST(PrioritySet, Extend) { HostSetImpl::updateHostsParams(hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - {}, hosts_added, hosts_removed, absl::nullopt); + {}, hosts_added, hosts_removed, hosts->size(), absl::nullopt); } EXPECT_EQ(1, priority_changes); EXPECT_EQ(1, membership_changes); @@ -2282,7 +2282,7 @@ TEST(HostsPerLocalityImpl, Filter) { class HostSetImplLocalityTest : public testing::Test { public: LocalityWeightsConstSharedPtr locality_weights_; - HostSetImpl host_set_{0, kDefaultOverProvisioningFactor}; + HostSetImpl host_set_{0, 0, kDefaultOverProvisioningFactor}; std::shared_ptr info_{new NiceMock()}; HostVector hosts_{ makeTestHost(info_, "tcp://127.0.0.1:80"), makeTestHost(info_, "tcp://127.0.0.1:81"), @@ -2303,7 +2303,7 @@ TEST_F(HostSetImplLocalityTest, AllUnhealthy) { LocalityWeightsConstSharedPtr locality_weights{new LocalityWeights{1, 1, 1}}; auto hosts = makeHostsFromHostsPerLocality(hosts_per_locality); host_set_.updateHosts(HostSetImpl::updateHostsParams(hosts, hosts_per_locality), locality_weights, - {}, {}, absl::nullopt); + {}, {}, hosts->size(), absl::nullopt); EXPECT_FALSE(host_set_.chooseHealthyLocality().has_value()); } @@ -2316,7 +2316,7 @@ TEST_F(HostSetImplLocalityTest, EmptyLocality) { host_set_.updateHosts(HostSetImpl::updateHostsParams( hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - locality_weights, {}, {}, absl::nullopt); + locality_weights, {}, {}, hosts->size(), absl::nullopt); // Verify that we are not RRing between localities. EXPECT_EQ(0, host_set_.chooseHealthyLocality().value()); EXPECT_EQ(0, host_set_.chooseHealthyLocality().value()); @@ -2330,7 +2330,7 @@ TEST_F(HostSetImplLocalityTest, AllZeroWeights) { host_set_.updateHosts(HostSetImpl::updateHostsParams( hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - locality_weights, {}, {}); + locality_weights, {}, {}, hosts->size()); EXPECT_FALSE(host_set_.chooseHealthyLocality().has_value()); } @@ -2343,7 +2343,7 @@ TEST_F(HostSetImplLocalityTest, Unweighted) { host_set_.updateHosts(HostSetImpl::updateHostsParams( hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - locality_weights, {}, {}, absl::nullopt); + locality_weights, {}, {}, hosts->size(), absl::nullopt); EXPECT_EQ(0, host_set_.chooseHealthyLocality().value()); EXPECT_EQ(1, host_set_.chooseHealthyLocality().value()); EXPECT_EQ(2, host_set_.chooseHealthyLocality().value()); @@ -2360,7 +2360,7 @@ TEST_F(HostSetImplLocalityTest, Weighted) { host_set_.updateHosts(HostSetImpl::updateHostsParams( hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - locality_weights, {}, {}, absl::nullopt); + locality_weights, {}, {}, hosts->size(), absl::nullopt); EXPECT_EQ(1, host_set_.chooseHealthyLocality().value()); EXPECT_EQ(0, host_set_.chooseHealthyLocality().value()); EXPECT_EQ(1, host_set_.chooseHealthyLocality().value()); @@ -2378,7 +2378,7 @@ TEST_F(HostSetImplLocalityTest, MissingWeight) { host_set_.updateHosts(HostSetImpl::updateHostsParams( hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - locality_weights, {}, {}, absl::nullopt); + locality_weights, {}, {}, hosts->size(), absl::nullopt); EXPECT_EQ(0, host_set_.chooseHealthyLocality().value()); EXPECT_EQ(2, host_set_.chooseHealthyLocality().value()); EXPECT_EQ(0, host_set_.chooseHealthyLocality().value()); @@ -2406,7 +2406,7 @@ TEST_F(HostSetImplLocalityTest, UnhealthyFailover) { hosts, hosts_per_locality, makeHostsFromHostsPerLocality(healthy_hosts_per_locality), healthy_hosts_per_locality), - locality_weights, {}, {}, absl::nullopt); + locality_weights, {}, {}, hosts->size(), absl::nullopt); }; const auto expectPicks = [this](uint32_t locality_0_picks, uint32_t locality_1_picks) { @@ -2438,7 +2438,7 @@ TEST_F(HostSetImplLocalityTest, UnhealthyFailover) { TEST(OverProvisioningFactorTest, LocalityPickChanges) { auto setUpHostSetWithOPFAndTestPicks = [](const uint32_t overprovisioning_factor, const uint32_t pick_0, const uint32_t pick_1) { - HostSetImpl host_set(0, overprovisioning_factor); + HostSetImpl host_set(0, 0, overprovisioning_factor); std::shared_ptr cluster_info{new NiceMock()}; HostVector hosts{makeTestHost(cluster_info, "tcp://127.0.0.1:80"), makeTestHost(cluster_info, "tcp://127.0.0.1:81"), @@ -2454,7 +2454,7 @@ TEST(OverProvisioningFactorTest, LocalityPickChanges) { host_set.updateHosts(HostSetImpl::updateHostsParams(std::make_shared(hosts), hosts_per_locality, healthy_hosts, healthy_hosts_per_locality), - locality_weights, {}, {}, absl::nullopt); + locality_weights, {}, {}, hosts.size(), absl::nullopt); uint32_t cnts[] = {0, 0}; for (uint32_t i = 0; i < 100; ++i) { absl::optional locality_index = host_set.chooseHealthyLocality(); diff --git a/test/mocks/upstream/host.cc b/test/mocks/upstream/host.cc index 6bd306c87aaed..536428cba59ae 100644 --- a/test/mocks/upstream/host.cc +++ b/test/mocks/upstream/host.cc @@ -46,6 +46,7 @@ MockHost::MockHost() { ON_CALL(*this, cluster()).WillByDefault(ReturnRef(cluster_)); ON_CALL(*this, outlierDetector()).WillByDefault(ReturnRef(outlier_detector_)); ON_CALL(*this, stats()).WillByDefault(ReturnRef(stats_)); + ON_CALL(*this, warmed()).WillByDefault(Return(true)); } MockHost::~MockHost() {} diff --git a/test/mocks/upstream/host.h b/test/mocks/upstream/host.h index d546111bdead6..e2f6a06140214 100644 --- a/test/mocks/upstream/host.h +++ b/test/mocks/upstream/host.h @@ -163,6 +163,7 @@ class MockHost : public Host { MOCK_CONST_METHOD0(locality, const envoy::api::v2::core::Locality&()); MOCK_CONST_METHOD0(priority, uint32_t()); MOCK_METHOD1(priority, void(uint32_t)); + MOCK_CONST_METHOD0(warmed, bool()); testing::NiceMock cluster_; testing::NiceMock outlier_detector_; diff --git a/test/mocks/upstream/mocks.cc b/test/mocks/upstream/mocks.cc index 22e89185082af..c88150f08aa60 100644 --- a/test/mocks/upstream/mocks.cc +++ b/test/mocks/upstream/mocks.cc @@ -36,6 +36,7 @@ MockHostSet::MockHostSet(uint32_t priority, uint32_t overprovisioning_factor) ON_CALL(*this, localityWeights()).WillByDefault(Invoke([this]() -> LocalityWeightsConstSharedPtr { return locality_weights_; })); + ON_CALL(*this, warmedHostCount()).WillByDefault(Invoke([this]() { return hosts_.size(); })); } MockHostSet::~MockHostSet() = default; diff --git a/test/mocks/upstream/mocks.h b/test/mocks/upstream/mocks.h index 10086aa425786..88c35aed7f28b 100644 --- a/test/mocks/upstream/mocks.h +++ b/test/mocks/upstream/mocks.h @@ -63,6 +63,7 @@ class MockHostSet : public HostSet { void setOverprovisioningFactor(const uint32_t overprovisioning_factor) { overprovisioning_factor_ = overprovisioning_factor; } + MOCK_CONST_METHOD0(warmedHostCount, uint32_t()); HostVector hosts_; HostVector healthy_hosts_; @@ -90,15 +91,17 @@ class MockPrioritySet : public PrioritySet { MOCK_CONST_METHOD1(addPriorityUpdateCb, Common::CallbackHandle*(PriorityUpdateCb callback)); MOCK_CONST_METHOD0(hostSetsPerPriority, const std::vector&()); MOCK_METHOD0(hostSetsPerPriority, std::vector&()); - MOCK_METHOD6(updateHosts, void(uint32_t priority, UpdateHostsParams&& update_hosts_params, + MOCK_METHOD7(updateHosts, void(uint32_t priority, UpdateHostsParams&& update_hosts_params, LocalityWeightsConstSharedPtr locality_weights, const HostVector& hosts_added, const HostVector& hosts_removed, + uint32_t warmed_host_count, absl::optional overprovisioning_factor)); MOCK_METHOD1(batchHostUpdate, void(BatchUpdateCb&)); - MOCK_CONST_METHOD6(updateHosts, + MOCK_CONST_METHOD7(updateHosts, void(uint32_t priority, UpdateHostsParams&& update_hosts_params, LocalityWeightsConstSharedPtr locality_weights, const HostVector& hosts_added, const HostVector& hosts_removed, + uint32_t warmed_host_count, absl::optional overprovisioning_factor)); MockHostSet* getMockHostSet(uint32_t priority) { From fd70113cd8917283917bec7734bd360dfd7a6489 Mon Sep 17 00:00:00 2001 From: Snow Pettersen Date: Wed, 1 May 2019 17:08:56 -0400 Subject: [PATCH 02/28] fix upstream tests Signed-off-by: Snow Pettersen --- .../common/upstream/cluster_manager_impl.cc | 9 +++--- source/common/upstream/cluster_manager_impl.h | 1 + source/common/upstream/subset_lb.cc | 2 +- .../upstream/cluster_manager_impl_test.cc | 30 +++++++++---------- .../upstream/load_balancer_simulation_test.cc | 4 +-- .../upstream/original_dst_cluster_test.cc | 2 +- test/common/upstream/subset_lb_test.cc | 8 ++--- .../clusters/custom_static_cluster.cc | 2 +- 8 files changed, 30 insertions(+), 28 deletions(-) diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index c73bf5b639c92..8bf234e721f4e 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -692,13 +692,14 @@ void ClusterManagerImpl::postThreadLocalClusterUpdate(const Cluster& cluster, ui healthy_hosts_copy, degraded_hosts_copy, hosts_per_locality_copy, healthy_hosts_per_locality_copy, degraded_hosts_per_locality_copy, locality_weights = host_set->localityWeights(), hosts_added, hosts_removed, - overprovisioning_factor = host_set->overprovisioningFactor()]() { + overprovisioning_factor = host_set->overprovisioningFactor(), + warmed_host_count = host_set->warmedHostCount()]() { ThreadLocalClusterManagerImpl::updateClusterMembership( name, priority, HostSetImpl::updateHostsParams(hosts_copy, hosts_per_locality_copy, healthy_hosts_copy, healthy_hosts_per_locality_copy, degraded_hosts_copy, degraded_hosts_per_locality_copy), - locality_weights, hosts_added, hosts_removed, *tls_, overprovisioning_factor); + locality_weights, hosts_added, hosts_removed, *tls_, warmed_host_count, overprovisioning_factor); }); } @@ -970,7 +971,7 @@ void ClusterManagerImpl::ThreadLocalClusterManagerImpl::updateClusterMembership( const std::string& name, uint32_t priority, PrioritySet::UpdateHostsParams&& update_hosts_params, LocalityWeightsConstSharedPtr locality_weights, const HostVector& hosts_added, - const HostVector& hosts_removed, ThreadLocal::Slot& tls, uint64_t overprovisioning_factor) { + const HostVector& hosts_removed, ThreadLocal::Slot& tls, uint32_t warmed_host_count, uint64_t overprovisioning_factor) { ThreadLocalClusterManagerImpl& config = tls.getTyped(); @@ -980,7 +981,7 @@ void ClusterManagerImpl::ThreadLocalClusterManagerImpl::updateClusterMembership( hosts_added.size(), hosts_removed.size()); cluster_entry->priority_set_.updateHosts(priority, std::move(update_hosts_params), std::move(locality_weights), hosts_added, hosts_removed, - overprovisioning_factor); + warmed_host_count, overprovisioning_factor); // If an LB is thread aware, create a new worker local LB on membership changes. if (cluster_entry->lb_factory_ != nullptr) { diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index 4cb0914ee20c4..d3dba20b2f691 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -330,6 +330,7 @@ class ClusterManagerImpl : public ClusterManager, Logger::Loggablesize(), absl::nullopt); } LocalityWeightsConstSharedPtr SubsetLoadBalancer::HostSubsetImpl::determineLocalityWeights( diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index 5647c8d8a6e24..3aa5b6ba3b4f5 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -1308,7 +1308,7 @@ TEST_F(ClusterManagerImplTest, HostsPostedToTlsCluster) { cluster1->priority_set_.updateHosts( 0, HostSetImpl::partitionHosts(hosts_ptr, HostsPerLocalityImpl::empty()), nullptr, hosts, {}, - 100); + hosts.size(), 100); auto* tls_cluster = cluster_manager_->get(cluster1->info_->name()); @@ -2177,7 +2177,7 @@ TEST_F(ClusterManagerImplTest, MergedUpdates) { HostSetImpl::updateHostsParams(hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - {}, hosts_added, hosts_removed, absl::nullopt); + {}, hosts_added, hosts_removed, hosts->size(), absl::nullopt); EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.update_merge_cancelled").value()); @@ -2189,13 +2189,13 @@ TEST_F(ClusterManagerImplTest, MergedUpdates) { HostSetImpl::updateHostsParams(hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - {}, hosts_added, hosts_removed, absl::nullopt); + {}, hosts_added, hosts_removed, hosts->size(), absl::nullopt); cluster.prioritySet().updateHosts( 0, HostSetImpl::updateHostsParams(hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - {}, hosts_added, hosts_removed, absl::nullopt); + {}, hosts_added, hosts_removed, hosts->size(), absl::nullopt); EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.update_merge_cancelled").value()); @@ -2214,7 +2214,7 @@ TEST_F(ClusterManagerImplTest, MergedUpdates) { HostSetImpl::updateHostsParams(hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - {}, hosts_added, hosts_removed, absl::nullopt); + {}, hosts_added, hosts_removed, hosts->size(), absl::nullopt); EXPECT_EQ(2, factory_.stats_.counter("cluster_manager.cluster_updated").value()); EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.update_merge_cancelled").value()); @@ -2228,7 +2228,7 @@ TEST_F(ClusterManagerImplTest, MergedUpdates) { HostSetImpl::updateHostsParams(hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - {}, hosts_added, hosts_removed, absl::nullopt); + {}, hosts_added, hosts_removed, hosts->size(), absl::nullopt); (*hosts)[0]->healthFlagSet(Host::HealthFlag::FAILED_EDS_HEALTH); cluster.prioritySet().updateHosts( @@ -2236,7 +2236,7 @@ TEST_F(ClusterManagerImplTest, MergedUpdates) { HostSetImpl::updateHostsParams(hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - {}, hosts_added, hosts_removed, absl::nullopt); + {}, hosts_added, hosts_removed, hosts->size(), absl::nullopt); (*hosts)[0]->weight(100); cluster.prioritySet().updateHosts( @@ -2244,7 +2244,7 @@ TEST_F(ClusterManagerImplTest, MergedUpdates) { HostSetImpl::updateHostsParams(hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - {}, hosts_added, hosts_removed, absl::nullopt); + {}, hosts_added, hosts_removed, hosts->size(), absl::nullopt); // Updates not delivered yet. EXPECT_EQ(2, factory_.stats_.counter("cluster_manager.cluster_updated").value()); @@ -2258,7 +2258,7 @@ TEST_F(ClusterManagerImplTest, MergedUpdates) { HostSetImpl::updateHostsParams(hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - {}, hosts_added, hosts_removed, absl::nullopt); + {}, hosts_added, hosts_removed, hosts->size(), absl::nullopt); EXPECT_EQ(3, factory_.stats_.counter("cluster_manager.cluster_updated").value()); EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value()); @@ -2295,7 +2295,7 @@ TEST_F(ClusterManagerImplTest, MergedUpdatesOutOfWindow) { HostSetImpl::updateHostsParams(hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - {}, hosts_added, hosts_removed, absl::nullopt); + {}, hosts_added, hosts_removed, hosts->size(), absl::nullopt); EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value()); EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.update_out_of_merge_window").value()); @@ -2323,7 +2323,7 @@ TEST_F(ClusterManagerImplTest, MergedUpdatesInsideWindow) { HostSetImpl::updateHostsParams(hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - {}, hosts_added, hosts_removed, absl::nullopt); + {}, hosts_added, hosts_removed, hosts->size(), absl::nullopt); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.update_out_of_merge_window").value()); @@ -2359,7 +2359,7 @@ TEST_F(ClusterManagerImplTest, MergedUpdatesOutOfWindowDisabled) { HostSetImpl::updateHostsParams(hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - {}, hosts_added, hosts_removed, absl::nullopt); + {}, hosts_added, hosts_removed, hosts->size(), absl::nullopt); EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.update_out_of_merge_window").value()); @@ -2426,7 +2426,7 @@ TEST_F(ClusterManagerImplTest, MergedUpdatesDestroyedOnUpdate) { HostSetImpl::updateHostsParams(hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - {}, hosts_added, hosts_removed, absl::nullopt); + {}, hosts_added, hosts_removed, hosts->size(), absl::nullopt); EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.update_merge_cancelled").value()); @@ -2438,13 +2438,13 @@ TEST_F(ClusterManagerImplTest, MergedUpdatesDestroyedOnUpdate) { HostSetImpl::updateHostsParams(hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - {}, hosts_added, hosts_removed, absl::nullopt); + {}, hosts_added, hosts_removed, hosts->size(), absl::nullopt); cluster.prioritySet().updateHosts( 0, HostSetImpl::updateHostsParams(hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - {}, hosts_added, hosts_removed, absl::nullopt); + {}, hosts_added, hosts_removed, hosts->size(), absl::nullopt); EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.update_merge_cancelled").value()); diff --git a/test/common/upstream/load_balancer_simulation_test.cc b/test/common/upstream/load_balancer_simulation_test.cc index 76f820a161bdd..a5fc2c406e3fa 100644 --- a/test/common/upstream/load_balancer_simulation_test.cc +++ b/test/common/upstream/load_balancer_simulation_test.cc @@ -59,7 +59,7 @@ TEST(DISABLED_LeastRequestLoadBalancerWeightTest, Weight) { HostSetImpl::updateHostsParams(updated_hosts, updated_locality_hosts, std::make_shared(*updated_hosts), updated_locality_hosts), - {}, hosts, {}, absl::nullopt); + {}, hosts, {}, updated_hosts->size(), absl::nullopt); Stats::IsolatedStoreImpl stats_store; ClusterStats stats{ClusterInfoImpl::generateStats(stats_store)}; @@ -165,7 +165,7 @@ class DISABLED_SimulationTest : public testing::Test { HostSetImpl::updateHostsParams( originating_hosts, per_zone_local_shared, std::make_shared(*originating_hosts), per_zone_local_shared), - {}, empty_vector_, empty_vector_, absl::nullopt); + {}, empty_vector_, empty_vector_, originating_hosts->size(), absl::nullopt); HostConstSharedPtr selected = lb.chooseHost(nullptr); hits[selected->address()->asString()]++; diff --git a/test/common/upstream/original_dst_cluster_test.cc b/test/common/upstream/original_dst_cluster_test.cc index 23314f2f25fb3..3a121bc8e1e9c 100644 --- a/test/common/upstream/original_dst_cluster_test.cc +++ b/test/common/upstream/original_dst_cluster_test.cc @@ -468,7 +468,7 @@ TEST_F(OriginalDstClusterTest, MultipleClusters) { second.updateHosts(0, HostSetImpl::updateHostsParams(new_hosts, empty_hosts_per_locality, healthy_hosts, empty_hosts_per_locality), - {}, added, removed, absl::nullopt); + {}, added, removed, new_hosts->size(), absl::nullopt); }); EXPECT_CALL(membership_updated_, ready()); diff --git a/test/common/upstream/subset_lb_test.cc b/test/common/upstream/subset_lb_test.cc index f2b97e03278d6..b00a714cb8214 100644 --- a/test/common/upstream/subset_lb_test.cc +++ b/test/common/upstream/subset_lb_test.cc @@ -219,7 +219,7 @@ class SubsetLoadBalancerTest : public testing::TestWithParam { HostSetImpl::updateHostsParams(local_hosts_, local_hosts_per_locality_, std::make_shared(*local_hosts_), local_hosts_per_locality_), - {}, {}, {}, absl::nullopt); + {}, {}, {}, local_hosts_->size(), absl::nullopt); lb_.reset(new SubsetLoadBalancer( lb_type_, priority_set_, &local_priority_set_, stats_, stats_store_, runtime_, random_, @@ -318,7 +318,7 @@ class SubsetLoadBalancerTest : public testing::TestWithParam { HostSetImpl::updateHostsParams(local_hosts_, local_hosts_per_locality_, std::make_shared(*local_hosts_), local_hosts_per_locality_), - {}, {}, remove, absl::nullopt); + {}, {}, remove, local_hosts_->size(), absl::nullopt); } for (const auto& host : add) { @@ -335,7 +335,7 @@ class SubsetLoadBalancerTest : public testing::TestWithParam { HostSetImpl::updateHostsParams(local_hosts_, local_hosts_per_locality_, std::make_shared(*local_hosts_), local_hosts_per_locality_), - {}, add, {}, absl::nullopt); + {}, add, {}, local_hosts_->size(), absl::nullopt); } } else if (!add.empty() || !remove.empty()) { local_priority_set_.updateHosts( @@ -343,7 +343,7 @@ class SubsetLoadBalancerTest : public testing::TestWithParam { HostSetImpl::updateHostsParams(local_hosts_, local_hosts_per_locality_, std::make_shared(*local_hosts_), local_hosts_per_locality_), - {}, add, remove, absl::nullopt); + {}, add, remove, local_hosts_->size(), absl::nullopt); } } diff --git a/test/integration/clusters/custom_static_cluster.cc b/test/integration/clusters/custom_static_cluster.cc index d4dae7cce4da9..7003315801875 100644 --- a/test/integration/clusters/custom_static_cluster.cc +++ b/test/integration/clusters/custom_static_cluster.cc @@ -12,7 +12,7 @@ void CustomStaticCluster::startPreInit() { this->priority_set_.updateHosts( priority_, Upstream::HostSetImpl::partitionHosts(hosts_ptr, Upstream::HostsPerLocalityImpl::empty()), {}, - hosts, {}, absl::nullopt); + hosts, {}, hosts.size(), absl::nullopt); onPreInitComplete(); } From 83f454b331e3f0a09eea6ce6fae56aca6fd87ca2 Mon Sep 17 00:00:00 2001 From: Snow Pettersen Date: Thu, 2 May 2019 20:34:55 -0400 Subject: [PATCH 03/28] fix locality test Signed-off-by: Snow Pettersen --- include/envoy/upstream/upstream.h | 7 ++- source/common/upstream/upstream_impl.cc | 54 +++++++++++-------- source/common/upstream/upstream_impl.h | 11 +++- .../upstream/load_balancer_impl_test.cc | 35 ++++++++++++ test/common/upstream/upstream_impl_test.cc | 35 ++++++++++++ test/mocks/upstream/cluster_info.h | 1 + 6 files changed, 118 insertions(+), 25 deletions(-) diff --git a/include/envoy/upstream/upstream.h b/include/envoy/upstream/upstream.h index 764a02010bdde..1ef72c74a593b 100644 --- a/include/envoy/upstream/upstream.h +++ b/include/envoy/upstream/upstream.h @@ -784,7 +784,12 @@ class ClusterInfo { * @return whether to skip waiting for health checking before draining connections * after a host is removed from service discovery. */ - virtual bool drainConnectionsOnHostRemoval() const PURE; + virtual bool drainConnectionsOnHostRemoval() const PURE; + + /** + * @return true if this cluster has a health checker installed. + */ + virtual bool healthChecker() const PURE; /** * @return eds cluster service_name of the cluster. diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index a917b07184990..8327501e84e43 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -420,26 +420,29 @@ HostSetImpl::updateHostsParams(HostVectorConstSharedPtr hosts, HealthyHostVectorConstSharedPtr healthy_hosts, HostsPerLocalityConstSharedPtr healthy_hosts_per_locality) { - return updateHostsParams(std::move(hosts), std::move(hosts_per_locality), - std::move(healthy_hosts), std::move(healthy_hosts_per_locality), - std::make_shared(), - HostsPerLocalityImpl::empty()); -} - -PrioritySet::UpdateHostsParams -HostSetImpl::updateHostsParams(HostVectorConstSharedPtr hosts, - HostsPerLocalityConstSharedPtr hosts_per_locality, - HealthyHostVectorConstSharedPtr healthy_hosts, - HostsPerLocalityConstSharedPtr healthy_hosts_per_locality, - DegradedHostVectorConstSharedPtr degraded_hosts, - HostsPerLocalityConstSharedPtr degraded_hosts_per_locality) { - auto warmed_counts_per_locality = std::make_shared>(); + // Note that we cannot do this in general on any threads due to potential changes to the host + // health between now and when the list of healthy hosts was calculated. At the moment this is + // only called in test: in prod we always use the overload with all the parameters, either + // directly or through partitionHosts. + auto warmed_per_locality = std::make_shared>(); for (const auto& hosts : hosts_per_locality->get()) { - warmed_counts_per_locality->emplace_back( + warmed_per_locality->emplace_back( std::count_if(hosts.begin(), hosts.end(), [](const auto& host) { return host->warmed(); })); } + return updateHostsParams(std::move(hosts), std::move(hosts_per_locality), + std::move(healthy_hosts), std::move(healthy_hosts_per_locality), + std::make_shared(), + HostsPerLocalityImpl::empty(), std::move(warmed_per_locality)); +} +PrioritySet::UpdateHostsParams HostSetImpl::updateHostsParams( + HostVectorConstSharedPtr hosts, HostsPerLocalityConstSharedPtr hosts_per_locality, + HealthyHostVectorConstSharedPtr healthy_hosts, + HostsPerLocalityConstSharedPtr healthy_hosts_per_locality, + DegradedHostVectorConstSharedPtr degraded_hosts, + HostsPerLocalityConstSharedPtr degraded_hosts_per_locality, + std::shared_ptr> warmed_counts_per_locality) { return PrioritySet::UpdateHostsParams{std::move(hosts), std::move(healthy_hosts), std::move(degraded_hosts), @@ -456,11 +459,18 @@ HostSetImpl::partitionHosts(HostVectorConstSharedPtr hosts, auto healthy_and_degraded_hosts_per_locality = ClusterImplBase::partitionHostsPerLocality(*hosts_per_locality); + auto warmed_counts_per_locality = std::make_shared>(); + for (const auto& hosts : hosts_per_locality->get()) { + warmed_counts_per_locality->emplace_back( + std::count_if(hosts.begin(), hosts.end(), [](const auto& host) { return host->warmed(); })); + } + return updateHostsParams(std::move(hosts), std::move(hosts_per_locality), std::move(healthy_and_degraded_hosts.first), std::move(healthy_and_degraded_hosts_per_locality.first), std::move(healthy_and_degraded_hosts.second), - std::move(healthy_and_degraded_hosts_per_locality.second)); + std::move(healthy_and_degraded_hosts_per_locality.second), + std::move(warmed_counts_per_locality)); } double HostSetImpl::effectiveLocalityWeight(uint32_t index, @@ -584,7 +594,8 @@ ClusterInfoImpl::ClusterInfoImpl(const envoy::api::v2::Cluster& config, metadata_(config.metadata()), typed_metadata_(config.metadata()), common_lb_config_(config.common_lb_config()), cluster_socket_options_(parseClusterSocketOptions(config, bind_config)), - drain_connections_on_host_removal_(config.drain_connections_on_host_removal()) { + drain_connections_on_host_removal_(config.drain_connections_on_host_removal()), + health_checker_(!config.health_checks().empty()) { switch (config.lb_policy()) { case envoy::api::v2::Cluster::ROUND_ROBIN: lb_type_ = LoadBalancerType::RoundRobin; @@ -848,10 +859,8 @@ void ClusterImplBase::reloadHealthyHostsHelper(const HostSharedPtr&) { uint32_t warmed_hosts_count = host_set->hosts().size(); if (health_checker_ != nullptr) { - warmed_hosts_count -= - std::count_if(host_set->hosts().begin(), host_set->hosts().end(), [](const auto& host) { - return host->getActiveHealthFailureType() == Host::ActiveHealthFailureType::UNKNOWN; - }); + warmed_hosts_count -= std::count_if(host_set->hosts().begin(), host_set->hosts().end(), + [](const auto& host) { return !host->warmed(); }); } HostsPerLocalityConstSharedPtr hosts_per_locality_copy = host_set->hostsPerLocality().clone(); prioritySet().updateHosts( @@ -1019,8 +1028,7 @@ void PriorityStateManager::updateClusterPrioritySet( uint32_t warmed_host_count = hosts->size(); for (const HostSharedPtr& host : *hosts) { - if (parent_.healthChecker() != nullptr && - host->getActiveHealthFailureType() == Host::ActiveHealthFailureType::UNKNOWN) { + if (parent_.healthChecker() != nullptr && !host->warmed()) { warmed_host_count--; } // Take into consideration when a non-EDS cluster has active health checking, i.e. to mark all diff --git a/source/common/upstream/upstream_impl.h b/source/common/upstream/upstream_impl.h index cc2bcb149af6e..bf7241468f8ce 100644 --- a/source/common/upstream/upstream_impl.h +++ b/source/common/upstream/upstream_impl.h @@ -225,6 +225,12 @@ class HostImpl : public HostDescriptionImpl, bool used() const override { return used_; } void used(bool new_used) override { used_ = new_used; } bool warmed() const override { + // If there is no health checker, there is no warming process. + if (!cluster_->healthChecker()) { + return true; + } + + // Otherwise, the host is warming. return !(healthFlagGet(HealthFlag::FAILED_ACTIVE_HC) && getActiveHealthFailureType() == ActiveHealthFailureType::UNKNOWN); } @@ -331,7 +337,8 @@ class HostSetImpl : public HostSet { HealthyHostVectorConstSharedPtr healthy_hosts, HostsPerLocalityConstSharedPtr healthy_hosts_per_locality, DegradedHostVectorConstSharedPtr degraded_hosts, - HostsPerLocalityConstSharedPtr degraded_hosts_per_locality); + HostsPerLocalityConstSharedPtr degraded_hosts_per_locality, + std::shared_ptr> warmed_counts_per_locality); static PrioritySet::UpdateHostsParams partitionHosts(HostVectorConstSharedPtr hosts, HostsPerLocalityConstSharedPtr hosts_per_locality); @@ -558,6 +565,7 @@ class ClusterInfoImpl : public ClusterInfo { }; bool drainConnectionsOnHostRemoval() const override { return drain_connections_on_host_removal_; } + bool healthChecker() const override { return health_checker_; } absl::optional eds_service_name() const override { return eds_service_name_; } @@ -603,6 +611,7 @@ class ClusterInfoImpl : public ClusterInfo { const envoy::api::v2::Cluster::CommonLbConfig common_lb_config_; const Network::ConnectionSocket::OptionsSharedPtr cluster_socket_options_; const bool drain_connections_on_host_removal_; + const bool health_checker_; absl::optional eds_service_name_; }; diff --git a/test/common/upstream/load_balancer_impl_test.cc b/test/common/upstream/load_balancer_impl_test.cc index 82b6d44b64be7..5d303603c5a14 100644 --- a/test/common/upstream/load_balancer_impl_test.cc +++ b/test/common/upstream/load_balancer_impl_test.cc @@ -568,6 +568,41 @@ TEST_P(FailoverTest, ExtendPrioritiesWithLocalPrioritySet) { EXPECT_EQ(tertiary_host_set_.hosts_[0], lb_->chooseHost(nullptr)); } +// Verifies that the number of warmed hosts is used to compute priority spillover. +TEST_P(FailoverTest, PrioritiesWithNotAllWarmedHosts) { + // To begin with we set up the following: + // P0: 1 healthy, 1 unhealthy, 1 warmed. + // P1: 1 healthy. + // We then expect no spillover, since P0 is still overprovisioned. + host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80"), makeTestHost(info_, "tcp://127.0.0.1:81")}; + host_set_.healthy_hosts_ = {host_set_.hosts_[0]}; + failover_host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:82")}; + failover_host_set_.healthy_hosts_ = failover_host_set_.hosts_; + ON_CALL(host_set_, warmedHostCount()).WillByDefault(Return(2)); + init(true); + + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(nullptr)); + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(nullptr)); + EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(nullptr)); +} + +// Verifies that we handle zero warmed hosts. +TEST_P(FailoverTest, PrioritiesWithZeroWarmedHosts) { + // To begin with we set up the following: + // P0: 2 unhealthy, 0 warmed. + // P1: 1 healthy. + // We then expect all the traffic to spill over to P1 since P0 has an effective load of zero. + host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80"), makeTestHost(info_, "tcp://127.0.0.1:81")}; + failover_host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:82")}; + failover_host_set_.healthy_hosts_ = failover_host_set_.hosts_; + ON_CALL(host_set_, warmedHostCount()).WillByDefault(Return(0)); + init(true); + + EXPECT_EQ(failover_host_set_.hosts_[0], lb_->chooseHost(nullptr)); + EXPECT_EQ(failover_host_set_.hosts_[0], lb_->chooseHost(nullptr)); + EXPECT_EQ(failover_host_set_.hosts_[0], lb_->chooseHost(nullptr)); +} + INSTANTIATE_TEST_SUITE_P(PrimaryOrFailover, FailoverTest, ::testing::Values(true)); TEST_P(RoundRobinLoadBalancerTest, NoHosts) { diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc index c8205887e1189..459d6777baa0d 100644 --- a/test/common/upstream/upstream_impl_test.cc +++ b/test/common/upstream/upstream_impl_test.cc @@ -37,6 +37,7 @@ using testing::_; using testing::ContainerEq; using testing::Invoke; using testing::NiceMock; +using testing::Return; using testing::ReturnRef; namespace Envoy { @@ -2307,6 +2308,40 @@ TEST_F(HostSetImplLocalityTest, AllUnhealthy) { EXPECT_FALSE(host_set_.chooseHealthyLocality().has_value()); } +// When a locality has endpoints that have not yet been warmed, weigth calculation should ignore +// these hosts. +TEST_F(HostSetImplLocalityTest, NotWarmedHostsLocality) { + // We need a health checker in order for hosts to be considered not warmed. + ON_CALL(*info_, healthChecker()).WillByDefault(Return(true)); + + // We have two localities with 3 hosts in L1, 2 hosts in L2. Two of the hosts in L1 are not warmed + // yet, so even though they are unhealthy we should not adjust the locality weight. + HostsPerLocalitySharedPtr hosts_per_locality = + makeHostsPerLocality({{hosts_[0], hosts_[1], hosts_[2]}, {hosts_[3], hosts_[4]}}); + LocalityWeightsConstSharedPtr locality_weights{new LocalityWeights{1, 1}}; + auto hosts = makeHostsFromHostsPerLocality(hosts_per_locality); + HostsPerLocalitySharedPtr healthy_hosts_per_locality = + makeHostsPerLocality({{hosts_[0]}, {hosts_[3], hosts_[4]}}); + + // Mark hosts 1 and 2 as not warmed. + hosts_[1]->healthFlagSet(Host::HealthFlag::FAILED_ACTIVE_HC); + hosts_[1]->setActiveHealthFailureType(Host::ActiveHealthFailureType::UNKNOWN); + hosts_[2]->healthFlagSet(Host::HealthFlag::FAILED_ACTIVE_HC); + hosts_[2]->setActiveHealthFailureType(Host::ActiveHealthFailureType::UNKNOWN); + + host_set_.updateHosts( + HostSetImpl::updateHostsParams( + hosts, hosts_per_locality, + makeHostsFromHostsPerLocality(healthy_hosts_per_locality), + healthy_hosts_per_locality), + locality_weights, {}, {}, hosts->size(), absl::nullopt); + // We should RR between localities with equal weight. + EXPECT_EQ(0, host_set_.chooseHealthyLocality().value()); + EXPECT_EQ(1, host_set_.chooseHealthyLocality().value()); + EXPECT_EQ(0, host_set_.chooseHealthyLocality().value()); + EXPECT_EQ(1, host_set_.chooseHealthyLocality().value()); +} + // When a locality has zero hosts, it should be treated as if it has zero healthy. TEST_F(HostSetImplLocalityTest, EmptyLocality) { HostsPerLocalitySharedPtr hosts_per_locality = diff --git a/test/mocks/upstream/cluster_info.h b/test/mocks/upstream/cluster_info.h index dd498b8fae26c..75cd5a22be5a1 100644 --- a/test/mocks/upstream/cluster_info.h +++ b/test/mocks/upstream/cluster_info.h @@ -84,6 +84,7 @@ class MockClusterInfo : public ClusterInfo { MOCK_CONST_METHOD0(typedMetadata, const Envoy::Config::TypedMetadata&()); MOCK_CONST_METHOD0(clusterSocketOptions, const Network::ConnectionSocket::OptionsSharedPtr&()); MOCK_CONST_METHOD0(drainConnectionsOnHostRemoval, bool()); + MOCK_CONST_METHOD0(healthChecker, bool()); MOCK_CONST_METHOD0(eds_service_name, absl::optional()); std::string name_{"fake_cluster"}; From f7b142f259d2eab8af114ca2462a70bb44c884e3 Mon Sep 17 00:00:00 2001 From: Snow Pettersen Date: Thu, 2 May 2019 20:37:44 -0400 Subject: [PATCH 04/28] add test for zero warmed in locality Signed-off-by: Snow Pettersen --- test/common/upstream/upstream_impl_test.cc | 30 ++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc index 459d6777baa0d..dec82dc926b98 100644 --- a/test/common/upstream/upstream_impl_test.cc +++ b/test/common/upstream/upstream_impl_test.cc @@ -2342,6 +2342,36 @@ TEST_F(HostSetImplLocalityTest, NotWarmedHostsLocality) { EXPECT_EQ(1, host_set_.chooseHealthyLocality().value()); } +// Verifies that we handle the case when there are no warmed hosts in a set with unhealthy hosts. +TEST_F(HostSetImplLocalityTest, ZeroWarmedHostsLocality) { + // We need a health checker in order for hosts to be considered not warmed. + ON_CALL(*info_, healthChecker()).WillByDefault(Return(true)); + + // We have two localities with 2 hosts in L1, 2 hosts in L2. The two hosts in L1 are not warmed. + HostsPerLocalitySharedPtr hosts_per_locality = + makeHostsPerLocality({{hosts_[1], hosts_[2]}, {hosts_[3], hosts_[4]}}); + LocalityWeightsConstSharedPtr locality_weights{new LocalityWeights{1, 1}}; + auto hosts = makeHostsFromHostsPerLocality(hosts_per_locality); + HostsPerLocalitySharedPtr healthy_hosts_per_locality = + makeHostsPerLocality({{}, {hosts_[3], hosts_[4]}}); + + // Mark hosts 1 and 2 as not warmed. + hosts_[1]->healthFlagSet(Host::HealthFlag::FAILED_ACTIVE_HC); + hosts_[1]->setActiveHealthFailureType(Host::ActiveHealthFailureType::UNKNOWN); + hosts_[2]->healthFlagSet(Host::HealthFlag::FAILED_ACTIVE_HC); + hosts_[2]->setActiveHealthFailureType(Host::ActiveHealthFailureType::UNKNOWN); + + host_set_.updateHosts( + HostSetImpl::updateHostsParams( + hosts, hosts_per_locality, + makeHostsFromHostsPerLocality(healthy_hosts_per_locality), + healthy_hosts_per_locality), + locality_weights, {}, {}, hosts->size(), absl::nullopt); + // We should only target the second locality since there are no healthy in the first. + EXPECT_EQ(1, host_set_.chooseHealthyLocality().value()); + EXPECT_EQ(1, host_set_.chooseHealthyLocality().value()); +} + // When a locality has zero hosts, it should be treated as if it has zero healthy. TEST_F(HostSetImplLocalityTest, EmptyLocality) { HostsPerLocalitySharedPtr hosts_per_locality = From 8d0b3438cc2df75878a28a288eaa115c1f3a5167 Mon Sep 17 00:00:00 2001 From: Snow Pettersen Date: Fri, 3 May 2019 02:36:04 -0400 Subject: [PATCH 05/28] fix most of the tests Signed-off-by: Snow Pettersen --- api/envoy/api/v2/cds.proto | 21 ++++++ include/envoy/upstream/upstream.h | 21 ++++-- .../common/upstream/cluster_manager_impl.cc | 12 +++- source/common/upstream/eds.cc | 9 ++- .../upstream/health_checker_base_impl.cc | 6 ++ source/common/upstream/subset_lb.cc | 16 +++-- source/common/upstream/upstream_impl.cc | 72 +++++++++---------- source/common/upstream/upstream_impl.h | 31 ++++---- test/common/upstream/eds_test.cc | 26 ++++++- .../upstream/health_checker_impl_test.cc | 13 +++- test/common/upstream/subset_lb_test.cc | 1 + test/common/upstream/upstream_impl_test.cc | 12 ++-- test/mocks/upstream/cluster_info.h | 2 +- test/mocks/upstream/mocks.cc | 3 + test/mocks/upstream/mocks.h | 23 +++--- 15 files changed, 172 insertions(+), 96 deletions(-) diff --git a/api/envoy/api/v2/cds.proto b/api/envoy/api/v2/cds.proto index d659ef80118d0..dbe5c0af8e3c6 100644 --- a/api/envoy/api/v2/cds.proto +++ b/api/envoy/api/v2/cds.proto @@ -526,6 +526,27 @@ message Cluster { // because merging those updates isn't currently safe. See // https://github.com/envoyproxy/envoy/pull/3941. google.protobuf.Duration update_merge_window = 4; + + // If set to true, Envoy will not consider new hosts when computing load balancing weights until + // they have been health checked for the first time. This will have no effect unless + // active health checking is also configured. + // + // Ignoring a host means that for any load balancing calculations that adjust weights based + // on the ratio of eligible hosts and total hosts (priority spillover, locality weighting, etc.) + // will exclude these hosts in the denominator. + // + // For example, with hosts in two priorities P0 and P1, where P0 looks like + // {healthy, unhealthy (new), unhealthy (new)} + // and where P1 looks like + // {healthy, healthy} + // all traffic will still hit P0, as 1 / (3 - 2) = 1. + // + // Enabling this will allow scaling up the number of hosts for a given cluster without entering panic + // mode or triggering priority spillover, assuming the hosts pass the first health check. + // + // If panic mode is triggered, new hosts are still eligible for traffic; they simply do not + // contribute to the calculation when deciding whether panic mode is enabled or not. + bool ignore_new_hosts_until_first_hc = 5; } // Common configuration for all load balancer implementations. diff --git a/include/envoy/upstream/upstream.h b/include/envoy/upstream/upstream.h index 1ef72c74a593b..b47e6ae232735 100644 --- a/include/envoy/upstream/upstream.h +++ b/include/envoy/upstream/upstream.h @@ -54,7 +54,9 @@ class Host : virtual public HostDescription { m(DEGRADED_EDS_HEALTH, 0x10) \ /* The host is pending removal from discovery but is stabilized due to */ \ /* active HC. */ \ - m(PENDING_DYNAMIC_REMOVAL, 0x20) + m(PENDING_DYNAMIC_REMOVAL, 0x20) \ + /* The host is pending its initial active health check. */ \ + m(PENDING_ACTIVE_HC, 0x40) // clang-format on #define DECLARE_ENUM(name, value) name = value, @@ -302,6 +304,11 @@ class HostSet { */ virtual const HostsPerLocality& degradedHostsPerLocality() const PURE; + /** + * @return same as hostsPerLocality but only contains warmed hosts. + */ + virtual const HostsPerLocality& warmedHostsPerLocality() const PURE; + /** * @return weights for each locality in the host set. */ @@ -331,7 +338,8 @@ class HostSet { /** * @return uint32_t the number of warmed hosts in this host set. A host is considered warming - * if active health checking is enabled and the host has yet to be health checked for the first time. + * if active health checking is enabled and the host has yet to be health checked for the first + * time. */ virtual uint32_t warmedHostCount() const PURE; }; @@ -389,7 +397,7 @@ class PrioritySet { HostsPerLocalityConstSharedPtr hosts_per_locality; HostsPerLocalityConstSharedPtr healthy_hosts_per_locality; HostsPerLocalityConstSharedPtr degraded_hosts_per_locality; - std::shared_ptr> warmed_hosts_per_locality; + HostsPerLocalityConstSharedPtr warmed_hosts_per_locality; }; /** @@ -784,12 +792,13 @@ class ClusterInfo { * @return whether to skip waiting for health checking before draining connections * after a host is removed from service discovery. */ - virtual bool drainConnectionsOnHostRemoval() const PURE; + virtual bool drainConnectionsOnHostRemoval() const PURE; /** - * @return true if this cluster has a health checker installed. + * @return true if this cluster is configured to ignore hosts for the purpose of load balancing + * computations until they have been health checked for the first time. */ - virtual bool healthChecker() const PURE; + virtual bool warmHosts() const PURE; /** * @return eds cluster service_name of the cluster. diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index 8bf234e721f4e..1ca31a2636127 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -688,9 +688,12 @@ void ClusterManagerImpl::postThreadLocalClusterUpdate(const Cluster& cluster, ui HostsPerLocalityConstSharedPtr degraded_hosts_per_locality_copy = host_set->degradedHostsPerLocality().clone(); + auto warmed_hosts_per_locality_copy = host_set->warmedHostsPerLocality().clone(); + tls_->runOnAllThreads([this, name = cluster.info()->name(), priority, hosts_copy, healthy_hosts_copy, degraded_hosts_copy, hosts_per_locality_copy, healthy_hosts_per_locality_copy, degraded_hosts_per_locality_copy, + warmed_hosts_per_locality_copy, locality_weights = host_set->localityWeights(), hosts_added, hosts_removed, overprovisioning_factor = host_set->overprovisioningFactor(), warmed_host_count = host_set->warmedHostCount()]() { @@ -698,8 +701,10 @@ void ClusterManagerImpl::postThreadLocalClusterUpdate(const Cluster& cluster, ui name, priority, HostSetImpl::updateHostsParams(hosts_copy, hosts_per_locality_copy, healthy_hosts_copy, healthy_hosts_per_locality_copy, degraded_hosts_copy, - degraded_hosts_per_locality_copy), - locality_weights, hosts_added, hosts_removed, *tls_, warmed_host_count, overprovisioning_factor); + degraded_hosts_per_locality_copy, + warmed_hosts_per_locality_copy), + locality_weights, hosts_added, hosts_removed, *tls_, warmed_host_count, + overprovisioning_factor); }); } @@ -971,7 +976,8 @@ void ClusterManagerImpl::ThreadLocalClusterManagerImpl::updateClusterMembership( const std::string& name, uint32_t priority, PrioritySet::UpdateHostsParams&& update_hosts_params, LocalityWeightsConstSharedPtr locality_weights, const HostVector& hosts_added, - const HostVector& hosts_removed, ThreadLocal::Slot& tls, uint32_t warmed_host_count, uint64_t overprovisioning_factor) { + const HostVector& hosts_removed, ThreadLocal::Slot& tls, uint32_t warmed_host_count, + uint64_t overprovisioning_factor) { ThreadLocalClusterManagerImpl& config = tls.getTyped(); diff --git a/source/common/upstream/eds.cc b/source/common/upstream/eds.cc index c76247c92dd0e..0da1e5bd89f3d 100644 --- a/source/common/upstream/eds.cc +++ b/source/common/upstream/eds.cc @@ -185,9 +185,12 @@ void EdsClusterImpl::reloadHealthyHostsHelper(const HostSharedPtr& host) { HostsPerLocalityConstSharedPtr hosts_per_locality_copy = host_set->hostsPerLocality().filter( {[&host_to_exclude](const Host& host) { return &host != host_to_exclude.get(); }})[0]; - prioritySet().updateHosts(priority, - HostSetImpl::partitionHosts(hosts_copy, hosts_per_locality_copy), - host_set->localityWeights(), {}, hosts_to_remove, absl::nullopt); + // TODO(snowp): Do this inside the loop over hosts above. + const uint32_t warmed_hosts = std::count_if(hosts_copy->begin(), hosts_copy->end(), + [](const auto& host) { return host->warmed(); }); + prioritySet().updateHosts( + priority, HostSetImpl::partitionHosts(hosts_copy, hosts_per_locality_copy), + host_set->localityWeights(), {}, hosts_to_remove, warmed_hosts, absl::nullopt); } if (host_to_exclude != nullptr) { diff --git a/source/common/upstream/health_checker_base_impl.cc b/source/common/upstream/health_checker_base_impl.cc index 24cfabac7b541..0dec6ebf001f9 100644 --- a/source/common/upstream/health_checker_base_impl.cc +++ b/source/common/upstream/health_checker_base_impl.cc @@ -230,6 +230,7 @@ void HealthCheckerImplBase::ActiveHealthCheckSession::handleSuccess(bool degrade // it to healthy. This makes startup faster with a small reduction in overall reliability // depending on the HC settings. if (first_check_ || ++num_healthy_ == parent_.healthy_threshold_) { + host_->healthFlagClear(Host::HealthFlag::PENDING_ACTIVE_HC); host_->healthFlagClear(Host::HealthFlag::FAILED_ACTIVE_HC); parent_.incHealthy(); changed_state = HealthTransition::Changed; @@ -291,6 +292,11 @@ HealthTransition HealthCheckerImplBase::ActiveHealthCheckSession::setUnhealthy( } } + if (host_->healthFlagGet(Host::HealthFlag::PENDING_ACTIVE_HC)) { + host_->healthFlagClear(Host::HealthFlag::PENDING_ACTIVE_HC); + changed_state = HealthTransition::Changed; + } + if ((first_check_ || parent_.always_log_health_check_failures_) && parent_.event_logger_) { parent_.event_logger_->logUnhealthy(parent_.healthCheckerType(), host_, type, first_check_); } diff --git a/source/common/upstream/subset_lb.cc b/source/common/upstream/subset_lb.cc index acfc35441114f..7f8538dd2cf00 100644 --- a/source/common/upstream/subset_lb.cc +++ b/source/common/upstream/subset_lb.cc @@ -568,6 +568,8 @@ void SubsetLoadBalancer::HostSubsetImpl::update(const HostVector& hosts_added, original_host_set_.healthyHostsPerLocality().filter({cached_predicate})[0]; HostsPerLocalityConstSharedPtr degraded_hosts_per_locality = original_host_set_.degradedHostsPerLocality().filter({cached_predicate})[0]; + auto warmed_hosts_per_locality = + original_host_set_.warmedHostsPerLocality().filter({cached_predicate})[0]; // We can use the cached predicate here, since we trust that the hosts in hosts_added were also // present in the list of all hosts. @@ -587,11 +589,15 @@ void SubsetLoadBalancer::HostSubsetImpl::update(const HostVector& hosts_added, } } - HostSetImpl::updateHosts(HostSetImpl::updateHostsParams( - hosts, hosts_per_locality, healthy_hosts, healthy_hosts_per_locality, - degraded_hosts, degraded_hosts_per_locality), - determineLocalityWeights(*hosts_per_locality), filtered_added, - filtered_removed, hosts->size(), absl::nullopt); + // TODO(snowp): Right now we just pass hosts->size(), really this need to be filtering down the list + // of warmed hosts provided by the original_host_set_. We cannot compute the list of warmed hosts + // directly as subset updates happen on the worker threads. + HostSetImpl::updateHosts( + HostSetImpl::updateHostsParams(hosts, hosts_per_locality, healthy_hosts, + healthy_hosts_per_locality, degraded_hosts, + degraded_hosts_per_locality, warmed_hosts_per_locality), + determineLocalityWeights(*hosts_per_locality), filtered_added, filtered_removed, + hosts->size(), absl::nullopt); } LocalityWeightsConstSharedPtr SubsetLoadBalancer::HostSubsetImpl::determineLocalityWeights( diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index 8327501e84e43..2e0a701fc77a6 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -324,17 +324,16 @@ void HostSetImpl::updateHosts(PrioritySet::UpdateHostsParams&& update_hosts_para hosts_per_locality_ = std::move(update_hosts_params.hosts_per_locality); healthy_hosts_per_locality_ = std::move(update_hosts_params.healthy_hosts_per_locality); degraded_hosts_per_locality_ = std::move(update_hosts_params.degraded_hosts_per_locality); - warmed_counts_per_locality_ = std::move(update_hosts_params.warmed_hosts_per_locality); + warmed_hosts_per_locality_ = std::move(update_hosts_params.warmed_hosts_per_locality); locality_weights_ = std::move(locality_weights); warmed_host_count_ = warmed_host_count; rebuildLocalityScheduler(healthy_locality_scheduler_, healthy_locality_entries_, *healthy_hosts_per_locality_, healthy_hosts_->get(), hosts_per_locality_, - warmed_counts_per_locality_, locality_weights_, - overprovisioning_factor_); + warmed_hosts_per_locality_, locality_weights_, overprovisioning_factor_); rebuildLocalityScheduler(degraded_locality_scheduler_, degraded_locality_entries_, *degraded_hosts_per_locality_, degraded_hosts_->get(), - hosts_per_locality_, warmed_counts_per_locality_, locality_weights_, + hosts_per_locality_, warmed_hosts_per_locality_, locality_weights_, overprovisioning_factor_); runUpdateCallbacks(hosts_added, hosts_removed); @@ -345,7 +344,7 @@ void HostSetImpl::rebuildLocalityScheduler( std::vector>& locality_entries, const HostsPerLocality& eligible_hosts_per_locality, const HostVector& eligible_hosts, HostsPerLocalityConstSharedPtr all_hosts_per_locality, - std::shared_ptr> warmed_counts_per_locality, + HostsPerLocalityConstSharedPtr warmed_counts_per_locality, LocalityWeightsConstSharedPtr locality_weights, uint32_t overprovisioning_factor) { ASSERT((all_hosts_per_locality == nullptr) == (warmed_counts_per_locality == nullptr)); // Rebuild the locality scheduler by computing the effective weight of each @@ -419,30 +418,25 @@ HostSetImpl::updateHostsParams(HostVectorConstSharedPtr hosts, HostsPerLocalityConstSharedPtr hosts_per_locality, HealthyHostVectorConstSharedPtr healthy_hosts, HostsPerLocalityConstSharedPtr healthy_hosts_per_locality) { + // TODO(snowp): Move this function into test/ + // This overload is only used as a convenience method in test, so it should live there. - // Note that we cannot do this in general on any threads due to potential changes to the host - // health between now and when the list of healthy hosts was calculated. At the moment this is - // only called in test: in prod we always use the overload with all the parameters, either - // directly or through partitionHosts. - - auto warmed_per_locality = std::make_shared>(); - for (const auto& hosts : hosts_per_locality->get()) { - warmed_per_locality->emplace_back( - std::count_if(hosts.begin(), hosts.end(), [](const auto& host) { return host->warmed(); })); - } + auto warmed_per_locality = + hosts_per_locality->filter({[](const auto& host) { return host.warmed(); }})[0]; return updateHostsParams(std::move(hosts), std::move(hosts_per_locality), std::move(healthy_hosts), std::move(healthy_hosts_per_locality), std::make_shared(), HostsPerLocalityImpl::empty(), std::move(warmed_per_locality)); } -PrioritySet::UpdateHostsParams HostSetImpl::updateHostsParams( - HostVectorConstSharedPtr hosts, HostsPerLocalityConstSharedPtr hosts_per_locality, - HealthyHostVectorConstSharedPtr healthy_hosts, - HostsPerLocalityConstSharedPtr healthy_hosts_per_locality, - DegradedHostVectorConstSharedPtr degraded_hosts, - HostsPerLocalityConstSharedPtr degraded_hosts_per_locality, - std::shared_ptr> warmed_counts_per_locality) { +PrioritySet::UpdateHostsParams +HostSetImpl::updateHostsParams(HostVectorConstSharedPtr hosts, + HostsPerLocalityConstSharedPtr hosts_per_locality, + HealthyHostVectorConstSharedPtr healthy_hosts, + HostsPerLocalityConstSharedPtr healthy_hosts_per_locality, + DegradedHostVectorConstSharedPtr degraded_hosts, + HostsPerLocalityConstSharedPtr degraded_hosts_per_locality, + HostsPerLocalityConstSharedPtr warmed_counts_per_locality) { return PrioritySet::UpdateHostsParams{std::move(hosts), std::move(healthy_hosts), std::move(degraded_hosts), @@ -456,30 +450,24 @@ PrioritySet::UpdateHostsParams HostSetImpl::partitionHosts(HostVectorConstSharedPtr hosts, HostsPerLocalityConstSharedPtr hosts_per_locality) { auto healthy_and_degraded_hosts = ClusterImplBase::partitionHostList(*hosts); - auto healthy_and_degraded_hosts_per_locality = + auto healthy_degraded_warmed_hosts_per_locality = ClusterImplBase::partitionHostsPerLocality(*hosts_per_locality); - auto warmed_counts_per_locality = std::make_shared>(); - for (const auto& hosts : hosts_per_locality->get()) { - warmed_counts_per_locality->emplace_back( - std::count_if(hosts.begin(), hosts.end(), [](const auto& host) { return host->warmed(); })); - } - return updateHostsParams(std::move(hosts), std::move(hosts_per_locality), std::move(healthy_and_degraded_hosts.first), - std::move(healthy_and_degraded_hosts_per_locality.first), + std::move(std::get<0>(healthy_degraded_warmed_hosts_per_locality)), std::move(healthy_and_degraded_hosts.second), - std::move(healthy_and_degraded_hosts_per_locality.second), - std::move(warmed_counts_per_locality)); + std::move(std::get<1>(healthy_degraded_warmed_hosts_per_locality)), + std::move(std::get<2>(healthy_degraded_warmed_hosts_per_locality))); } double HostSetImpl::effectiveLocalityWeight(uint32_t index, const HostsPerLocality& eligible_hosts_per_locality, - const std::vector& warmed_count_per_locality, + const HostsPerLocality& warmed_count_per_locality, const LocalityWeights& locality_weights, uint32_t overprovisioning_factor) { const auto& locality_eligible_hosts = eligible_hosts_per_locality.get()[index]; - const auto warmed_count = warmed_count_per_locality[index]; + const auto warmed_count = warmed_count_per_locality.get()[index].size(); if (warmed_count == 0) { return 0.0; } @@ -595,7 +583,8 @@ ClusterInfoImpl::ClusterInfoImpl(const envoy::api::v2::Cluster& config, common_lb_config_(config.common_lb_config()), cluster_socket_options_(parseClusterSocketOptions(config, bind_config)), drain_connections_on_host_removal_(config.drain_connections_on_host_removal()), - health_checker_(!config.health_checks().empty()) { + warm_hosts_(!config.health_checks().empty() && + common_lb_config_.ignore_new_hosts_until_first_hc()) { switch (config.lb_policy()) { case envoy::api::v2::Cluster::ROUND_ROBIN: lb_type_ = LoadBalancerType::RoundRobin; @@ -740,13 +729,16 @@ ClusterImplBase::partitionHostList(const HostVector& hosts) { return {healthy_list, degraded_list}; } -std::pair +std::tuple ClusterImplBase::partitionHostsPerLocality(const HostsPerLocality& hosts) { auto filtered_clones = hosts.filter({[](const Host& host) { return host.health() == Host::Health::Healthy; }, - [](const Host& host) { return host.health() == Host::Health::Degraded; }}); + [](const Host& host) { return host.health() == Host::Health::Degraded; }, + [](const Host& host) { return host.warmed(); }}); - return {std::move(filtered_clones[0]), std::move(filtered_clones[1])}; + return {std::move(filtered_clones[0]), std::move(filtered_clones[1]), + std::move(filtered_clones[2])}; } bool ClusterInfoImpl::maintenanceMode() const { @@ -1235,9 +1227,11 @@ bool BaseDynamicClusterImpl::updateDynamicHostList(const HostVector& new_hosts, max_host_weight = host->weight(); } - // If we are depending on a health checker, we initialize to unhealthy. + // If we are depending on a health checker, we initialize to unhealthy and set the + // flag indicating that we have not performed any active health checks yet. if (health_checker_ != nullptr) { host->healthFlagSet(Host::HealthFlag::FAILED_ACTIVE_HC); + host->healthFlagSet(Host::HealthFlag::PENDING_ACTIVE_HC); } updated_hosts[host->address()->asString()] = host; diff --git a/source/common/upstream/upstream_impl.h b/source/common/upstream/upstream_impl.h index bf7241468f8ce..241ed1f581ef9 100644 --- a/source/common/upstream/upstream_impl.h +++ b/source/common/upstream/upstream_impl.h @@ -225,14 +225,7 @@ class HostImpl : public HostDescriptionImpl, bool used() const override { return used_; } void used(bool new_used) override { used_ = new_used; } bool warmed() const override { - // If there is no health checker, there is no warming process. - if (!cluster_->healthChecker()) { - return true; - } - - // Otherwise, the host is warming. - return !(healthFlagGet(HealthFlag::FAILED_ACTIVE_HC) && - getActiveHealthFailureType() == ActiveHealthFailureType::UNKNOWN); + return !cluster_->warmHosts() || !healthFlagGet(HealthFlag::PENDING_ACTIVE_HC); } protected: @@ -315,6 +308,9 @@ class HostSetImpl : public HostSet { const HostsPerLocality& degradedHostsPerLocality() const override { return *degraded_hosts_per_locality_; } + const HostsPerLocality& warmedHostsPerLocality() const override { + return *warmed_hosts_per_locality_; + } LocalityWeightsConstSharedPtr localityWeights() const override { return locality_weights_; } absl::optional chooseHealthyLocality() override; absl::optional chooseDegradedLocality() override; @@ -338,7 +334,7 @@ class HostSetImpl : public HostSet { HostsPerLocalityConstSharedPtr healthy_hosts_per_locality, DegradedHostVectorConstSharedPtr degraded_hosts, HostsPerLocalityConstSharedPtr degraded_hosts_per_locality, - std::shared_ptr> warmed_counts_per_locality); + HostsPerLocalityConstSharedPtr warmed_counts_per_locality); static PrioritySet::UpdateHostsParams partitionHosts(HostVectorConstSharedPtr hosts, HostsPerLocalityConstSharedPtr hosts_per_locality); @@ -357,7 +353,7 @@ class HostSetImpl : public HostSet { // locality. static double effectiveLocalityWeight(uint32_t index, const HostsPerLocality& eligible_hosts_per_locality, - const std::vector& warmed_counts_per_locality, + const HostsPerLocality& warmed_counts_per_locality, const LocalityWeights& locality_weights, uint32_t overprovisioning_factor); @@ -370,7 +366,7 @@ class HostSetImpl : public HostSet { HostsPerLocalityConstSharedPtr hosts_per_locality_{HostsPerLocalityImpl::empty()}; HostsPerLocalityConstSharedPtr healthy_hosts_per_locality_{HostsPerLocalityImpl::empty()}; HostsPerLocalityConstSharedPtr degraded_hosts_per_locality_{HostsPerLocalityImpl::empty()}; - std::shared_ptr> warmed_counts_per_locality_{}; + HostsPerLocalityConstSharedPtr warmed_hosts_per_locality_{HostsPerLocalityImpl::empty()}; // TODO(mattklein123): Remove mutable. mutable Common::CallbackManager member_update_cb_helper_; @@ -402,7 +398,7 @@ class HostSetImpl : public HostSet { std::vector>& locality_entries, const HostsPerLocality& eligible_hosts_per_locality, const HostVector& eligible_hosts, HostsPerLocalityConstSharedPtr all_hosts_per_locality, - std::shared_ptr> warmed_counts_per_locality, + HostsPerLocalityConstSharedPtr warmed_counts_per_locality, LocalityWeightsConstSharedPtr locality_weights, uint32_t overprovisioning_factor); static absl::optional chooseLocality(EdfScheduler* locality_scheduler); @@ -565,7 +561,7 @@ class ClusterInfoImpl : public ClusterInfo { }; bool drainConnectionsOnHostRemoval() const override { return drain_connections_on_host_removal_; } - bool healthChecker() const override { return health_checker_; } + bool warmHosts() const override { return warm_hosts_; } absl::optional eds_service_name() const override { return eds_service_name_; } @@ -611,7 +607,7 @@ class ClusterInfoImpl : public ClusterInfo { const envoy::api::v2::Cluster::CommonLbConfig common_lb_config_; const Network::ConnectionSocket::OptionsSharedPtr cluster_socket_options_; const bool drain_connections_on_host_removal_; - const bool health_checker_; + const bool warm_hosts_; absl::optional eds_service_name_; }; @@ -660,9 +656,10 @@ class ClusterImplBase : public Cluster, protected Logger::Loggable partitionHostList(const HostVector& hosts); - // Partitions the provided list of hosts per locality into two new lists containing the healthy - // and degraded hosts respectively. - static std::pair + // Partitions the provided list of hosts per locality into three new lists containing the healthy, + // degraded and warmed hosts respectively. + static std::tuple partitionHostsPerLocality(const HostsPerLocality& hosts); // Upstream::Cluster diff --git a/test/common/upstream/eds_test.cc b/test/common/upstream/eds_test.cc index acbebded4e96b..175f4e8814962 100644 --- a/test/common/upstream/eds_test.cc +++ b/test/common/upstream/eds_test.cc @@ -141,6 +141,10 @@ class EdsWithHealthCheckUpdateTest : public EdsTest { EXPECT_TRUE(hosts[0]->healthFlagGet(Host::HealthFlag::FAILED_ACTIVE_HC)); EXPECT_TRUE(hosts[1]->healthFlagGet(Host::HealthFlag::FAILED_ACTIVE_HC)); + // Remove the pending HC flag. This is normally done by the health checker. + hosts[0]->healthFlagClear(Host::HealthFlag::PENDING_ACTIVE_HC); + hosts[1]->healthFlagClear(Host::HealthFlag::PENDING_ACTIVE_HC); + // Mark the hosts as healthy hosts[0]->healthFlagClear(Host::HealthFlag::FAILED_ACTIVE_HC); hosts[1]->healthFlagClear(Host::HealthFlag::FAILED_ACTIVE_HC); @@ -480,7 +484,7 @@ TEST_F(EdsTest, EndpointHealthStatus) { // Verify that a host is removed if it is removed from discovery, stabilized, and then later // fails active HC. -TEST_F(EdsTest, EndpoingRemovalAfterHcFail) { +TEST_F(EdsTest, EndpointRemovalAfterHcFail) { envoy::api::v2::ClusterLoadAssignment cluster_load_assignment; cluster_load_assignment.set_cluster_name("fare"); @@ -508,6 +512,10 @@ TEST_F(EdsTest, EndpoingRemovalAfterHcFail) { auto& hosts = cluster_->prioritySet().hostSetsPerPriority()[0]->hosts(); EXPECT_EQ(hosts.size(), 2); + // Remove the pending HC flag. This is normally done by the health checker. + hosts[0]->healthFlagClear(Host::HealthFlag::PENDING_ACTIVE_HC); + hosts[1]->healthFlagClear(Host::HealthFlag::PENDING_ACTIVE_HC); + // Mark the hosts as healthy hosts[0]->healthFlagClear(Host::HealthFlag::FAILED_ACTIVE_HC); hosts[1]->healthFlagClear(Host::HealthFlag::FAILED_ACTIVE_HC); @@ -613,6 +621,10 @@ TEST_F(EdsTest, EndpointRemovalEdsFailButActiveHcSuccess) { auto& hosts = cluster_->prioritySet().hostSetsPerPriority()[0]->hosts(); EXPECT_EQ(hosts.size(), 2); + // Remove the pending HC flag. This is normally done by the health checker. + hosts[0]->healthFlagClear(Host::HealthFlag::PENDING_ACTIVE_HC); + hosts[1]->healthFlagClear(Host::HealthFlag::PENDING_ACTIVE_HC); + // Mark the hosts as healthy hosts[0]->healthFlagClear(Host::HealthFlag::FAILED_ACTIVE_HC); hosts[1]->healthFlagClear(Host::HealthFlag::FAILED_ACTIVE_HC); @@ -677,6 +689,11 @@ TEST_F(EdsTest, EndpointRemovalClusterDrainOnHostRemoval) { EXPECT_TRUE(hosts[0]->healthFlagGet(Host::HealthFlag::FAILED_ACTIVE_HC)); EXPECT_TRUE(hosts[1]->healthFlagGet(Host::HealthFlag::FAILED_ACTIVE_HC)); + + // Remove the pending HC flag. This is normally done by the health checker. + hosts[0]->healthFlagClear(Host::HealthFlag::PENDING_ACTIVE_HC); + hosts[1]->healthFlagClear(Host::HealthFlag::PENDING_ACTIVE_HC); + // Mark the hosts as healthy hosts[0]->healthFlagClear(Host::HealthFlag::FAILED_ACTIVE_HC); hosts[1]->healthFlagClear(Host::HealthFlag::FAILED_ACTIVE_HC); @@ -729,6 +746,7 @@ TEST_F(EdsTest, EndpointMovedToNewPriority) { for (auto& host : hosts) { EXPECT_TRUE(host->healthFlagGet(Host::HealthFlag::FAILED_ACTIVE_HC)); host->healthFlagClear(Host::HealthFlag::FAILED_ACTIVE_HC); + host->healthFlagClear(Host::HealthFlag::PENDING_ACTIVE_HC); } } @@ -797,8 +815,9 @@ TEST_F(EdsTest, EndpointMoved) { EXPECT_TRUE(hosts[0]->healthFlagGet(Host::HealthFlag::FAILED_ACTIVE_HC)); EXPECT_EQ(0, hosts[0]->priority()); - // Mark the host as healthy + // Mark the host as healthy and remove the pending active hc flag. hosts[0]->healthFlagClear(Host::HealthFlag::FAILED_ACTIVE_HC); + hosts[0]->healthFlagClear(Host::HealthFlag::PENDING_ACTIVE_HC); } { @@ -807,8 +826,9 @@ TEST_F(EdsTest, EndpointMoved) { EXPECT_TRUE(hosts[0]->healthFlagGet(Host::HealthFlag::FAILED_ACTIVE_HC)); EXPECT_EQ(1, hosts[0]->priority()); - // Mark the host as healthy + // Mark the host as healthy and remove the pending active hc flag. hosts[0]->healthFlagClear(Host::HealthFlag::FAILED_ACTIVE_HC); + hosts[0]->healthFlagClear(Host::HealthFlag::PENDING_ACTIVE_HC); } // Moves the endpoints between priorities diff --git a/test/common/upstream/health_checker_impl_test.cc b/test/common/upstream/health_checker_impl_test.cc index f098a43137d96..f86ef25410b43 100644 --- a/test/common/upstream/health_checker_impl_test.cc +++ b/test/common/upstream/health_checker_impl_test.cc @@ -3053,6 +3053,8 @@ TEST_F(GrpcHealthCheckerImplTest, SuccessStartFailedSuccessFirst) { makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")}; cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagSet( Host::HealthFlag::FAILED_ACTIVE_HC); + cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagSet( + Host::HealthFlag::PENDING_ACTIVE_HC); expectSessionCreate(); expectHealthcheckStart(0); @@ -3066,6 +3068,8 @@ TEST_F(GrpcHealthCheckerImplTest, SuccessStartFailedSuccessFirst) { EXPECT_CALL(*event_logger_, logAddHealthy(_, _, true)); respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::SERVING); expectHostHealthy(true); + EXPECT_FALSE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet( + Host::HealthFlag::PENDING_ACTIVE_HC)); } // Test host recovery after first failed check requires several successful checks. @@ -3075,6 +3079,8 @@ TEST_F(GrpcHealthCheckerImplTest, SuccessStartFailedFailFirst) { makeTestHost(cluster_->info_, "tcp://127.0.0.1:80")}; cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagSet( Host::HealthFlag::FAILED_ACTIVE_HC); + cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagSet( + Host::HealthFlag::PENDING_ACTIVE_HC); expectSessionCreate(); expectHealthcheckStart(0); @@ -3082,11 +3088,14 @@ TEST_F(GrpcHealthCheckerImplTest, SuccessStartFailedFailFirst) { // Failing first disables fast success. expectHealthcheckStop(0); - // Host was unhealthy from the start, no state change. - EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Unchanged)); + // Host was unhealthy from the start, but we expect a state change due to the pending active hc + // flag changing. + EXPECT_CALL(*this, onHostStatus(_, HealthTransition::Changed)); EXPECT_CALL(*event_logger_, logUnhealthy(_, _, _, true)); respondServiceStatus(0, grpc::health::v1::HealthCheckResponse::NOT_SERVING); expectHostHealthy(false); + EXPECT_FALSE(cluster_->prioritySet().getMockHostSet(0)->hosts_[0]->healthFlagGet( + Host::HealthFlag::PENDING_ACTIVE_HC)); // Next successful healthcheck does not move host int healthy state (because we configured // healthchecker this way). diff --git a/test/common/upstream/subset_lb_test.cc b/test/common/upstream/subset_lb_test.cc index b00a714cb8214..8bbd6520ad42d 100644 --- a/test/common/upstream/subset_lb_test.cc +++ b/test/common/upstream/subset_lb_test.cc @@ -155,6 +155,7 @@ class SubsetLoadBalancerTest : public testing::TestWithParam { host_set.hosts_ = all_hosts; host_set.hosts_per_locality_ = makeHostsPerLocality({first_locality, second_locality}); + host_set.warmed_hosts_per_locality_ = makeHostsPerLocality({first_locality, second_locality}); host_set.healthy_hosts_ = host_set.hosts_; host_set.healthy_hosts_per_locality_ = host_set.hosts_per_locality_; host_set.locality_weights_ = std::make_shared(locality_weights); diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc index dec82dc926b98..00e7b7d30b1d6 100644 --- a/test/common/upstream/upstream_impl_test.cc +++ b/test/common/upstream/upstream_impl_test.cc @@ -401,6 +401,7 @@ TEST_F(StrictDnsClusterImplTest, HostRemovalActiveHealthSkipped) { for (size_t i = 0; i < hosts.size(); ++i) { EXPECT_TRUE(hosts[i]->healthFlagGet(Host::HealthFlag::FAILED_ACTIVE_HC)); hosts[i]->healthFlagClear(Host::HealthFlag::FAILED_ACTIVE_HC); + hosts[i]->healthFlagClear(Host::HealthFlag::PENDING_ACTIVE_HC); } } @@ -453,6 +454,7 @@ TEST_F(StrictDnsClusterImplTest, HostRemovalAfterHcFail) { for (size_t i = 0; i < 2; ++i) { EXPECT_TRUE(hosts[i]->healthFlagGet(Host::HealthFlag::FAILED_ACTIVE_HC)); hosts[i]->healthFlagClear(Host::HealthFlag::FAILED_ACTIVE_HC); + hosts[i]->healthFlagClear(Host::HealthFlag::PENDING_ACTIVE_HC); if (i == 1) { EXPECT_CALL(initialized, ready()); } @@ -2311,8 +2313,7 @@ TEST_F(HostSetImplLocalityTest, AllUnhealthy) { // When a locality has endpoints that have not yet been warmed, weigth calculation should ignore // these hosts. TEST_F(HostSetImplLocalityTest, NotWarmedHostsLocality) { - // We need a health checker in order for hosts to be considered not warmed. - ON_CALL(*info_, healthChecker()).WillByDefault(Return(true)); + ON_CALL(*info_, warmHosts()).WillByDefault(Return(true)); // We have two localities with 3 hosts in L1, 2 hosts in L2. Two of the hosts in L1 are not warmed // yet, so even though they are unhealthy we should not adjust the locality weight. @@ -2325,9 +2326,9 @@ TEST_F(HostSetImplLocalityTest, NotWarmedHostsLocality) { // Mark hosts 1 and 2 as not warmed. hosts_[1]->healthFlagSet(Host::HealthFlag::FAILED_ACTIVE_HC); - hosts_[1]->setActiveHealthFailureType(Host::ActiveHealthFailureType::UNKNOWN); + hosts_[1]->healthFlagSet(Host::HealthFlag::PENDING_ACTIVE_HC); hosts_[2]->healthFlagSet(Host::HealthFlag::FAILED_ACTIVE_HC); - hosts_[2]->setActiveHealthFailureType(Host::ActiveHealthFailureType::UNKNOWN); + hosts_[2]->healthFlagSet(Host::HealthFlag::PENDING_ACTIVE_HC); host_set_.updateHosts( HostSetImpl::updateHostsParams( @@ -2344,8 +2345,7 @@ TEST_F(HostSetImplLocalityTest, NotWarmedHostsLocality) { // Verifies that we handle the case when there are no warmed hosts in a set with unhealthy hosts. TEST_F(HostSetImplLocalityTest, ZeroWarmedHostsLocality) { - // We need a health checker in order for hosts to be considered not warmed. - ON_CALL(*info_, healthChecker()).WillByDefault(Return(true)); + ON_CALL(*info_, warmHosts()).WillByDefault(Return(true)); // We have two localities with 2 hosts in L1, 2 hosts in L2. The two hosts in L1 are not warmed. HostsPerLocalitySharedPtr hosts_per_locality = diff --git a/test/mocks/upstream/cluster_info.h b/test/mocks/upstream/cluster_info.h index 75cd5a22be5a1..21fd638414aed 100644 --- a/test/mocks/upstream/cluster_info.h +++ b/test/mocks/upstream/cluster_info.h @@ -84,7 +84,7 @@ class MockClusterInfo : public ClusterInfo { MOCK_CONST_METHOD0(typedMetadata, const Envoy::Config::TypedMetadata&()); MOCK_CONST_METHOD0(clusterSocketOptions, const Network::ConnectionSocket::OptionsSharedPtr&()); MOCK_CONST_METHOD0(drainConnectionsOnHostRemoval, bool()); - MOCK_CONST_METHOD0(healthChecker, bool()); + MOCK_CONST_METHOD0(warmHosts, bool()); MOCK_CONST_METHOD0(eds_service_name, absl::optional()); std::string name_{"fake_cluster"}; diff --git a/test/mocks/upstream/mocks.cc b/test/mocks/upstream/mocks.cc index c88150f08aa60..7ec848f4f77de 100644 --- a/test/mocks/upstream/mocks.cc +++ b/test/mocks/upstream/mocks.cc @@ -33,6 +33,9 @@ MockHostSet::MockHostSet(uint32_t priority, uint32_t overprovisioning_factor) ON_CALL(*this, degradedHostsPerLocality()) .WillByDefault( Invoke([this]() -> const HostsPerLocality& { return *degraded_hosts_per_locality_; })); + ON_CALL(*this, warmedHostsPerLocality()) + .WillByDefault( + Invoke([this]() -> const HostsPerLocality& { return *warmed_hosts_per_locality_; })); ON_CALL(*this, localityWeights()).WillByDefault(Invoke([this]() -> LocalityWeightsConstSharedPtr { return locality_weights_; })); diff --git a/test/mocks/upstream/mocks.h b/test/mocks/upstream/mocks.h index 88c35aed7f28b..5795afa21f5ad 100644 --- a/test/mocks/upstream/mocks.h +++ b/test/mocks/upstream/mocks.h @@ -55,6 +55,7 @@ class MockHostSet : public HostSet { MOCK_CONST_METHOD0(hostsPerLocality, const HostsPerLocality&()); MOCK_CONST_METHOD0(healthyHostsPerLocality, const HostsPerLocality&()); MOCK_CONST_METHOD0(degradedHostsPerLocality, const HostsPerLocality&()); + MOCK_CONST_METHOD0(warmedHostsPerLocality, const HostsPerLocality&()); MOCK_CONST_METHOD0(localityWeights, LocalityWeightsConstSharedPtr()); MOCK_METHOD0(chooseHealthyLocality, absl::optional()); MOCK_METHOD0(chooseDegradedLocality, absl::optional()); @@ -71,6 +72,7 @@ class MockHostSet : public HostSet { HostsPerLocalitySharedPtr hosts_per_locality_{new HostsPerLocalityImpl()}; HostsPerLocalitySharedPtr healthy_hosts_per_locality_{new HostsPerLocalityImpl()}; HostsPerLocalitySharedPtr degraded_hosts_per_locality_{new HostsPerLocalityImpl()}; + HostsPerLocalitySharedPtr warmed_hosts_per_locality_{new HostsPerLocalityImpl()}; LocalityWeightsConstSharedPtr locality_weights_{{}}; Common::CallbackManager member_update_cb_helper_; uint32_t priority_{}; @@ -91,18 +93,17 @@ class MockPrioritySet : public PrioritySet { MOCK_CONST_METHOD1(addPriorityUpdateCb, Common::CallbackHandle*(PriorityUpdateCb callback)); MOCK_CONST_METHOD0(hostSetsPerPriority, const std::vector&()); MOCK_METHOD0(hostSetsPerPriority, std::vector&()); - MOCK_METHOD7(updateHosts, void(uint32_t priority, UpdateHostsParams&& update_hosts_params, - LocalityWeightsConstSharedPtr locality_weights, - const HostVector& hosts_added, const HostVector& hosts_removed, - uint32_t warmed_host_count, - absl::optional overprovisioning_factor)); + MOCK_METHOD7(updateHosts, + void(uint32_t priority, UpdateHostsParams&& update_hosts_params, + LocalityWeightsConstSharedPtr locality_weights, const HostVector& hosts_added, + const HostVector& hosts_removed, uint32_t warmed_host_count, + absl::optional overprovisioning_factor)); MOCK_METHOD1(batchHostUpdate, void(BatchUpdateCb&)); - MOCK_CONST_METHOD7(updateHosts, - void(uint32_t priority, UpdateHostsParams&& update_hosts_params, - LocalityWeightsConstSharedPtr locality_weights, - const HostVector& hosts_added, const HostVector& hosts_removed, - uint32_t warmed_host_count, - absl::optional overprovisioning_factor)); + MOCK_CONST_METHOD7(updateHosts, void(uint32_t priority, UpdateHostsParams&& update_hosts_params, + LocalityWeightsConstSharedPtr locality_weights, + const HostVector& hosts_added, + const HostVector& hosts_removed, uint32_t warmed_host_count, + absl::optional overprovisioning_factor)); MockHostSet* getMockHostSet(uint32_t priority) { getHostSet(priority); // Ensure the host set exists. From 75c8a7c05268234afed34fcf106d0d5cdaae7971 Mon Sep 17 00:00:00 2001 From: Snow Pettersen Date: Fri, 3 May 2019 02:53:11 -0400 Subject: [PATCH 06/28] actually adjust panic based on warmed hosts Signed-off-by: Snow Pettersen --- source/common/upstream/load_balancer_impl.cc | 8 ++++---- source/common/upstream/subset_lb.h | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/source/common/upstream/load_balancer_impl.cc b/source/common/upstream/load_balancer_impl.cc index 83534cf26914f..1093e336ff949 100644 --- a/source/common/upstream/load_balancer_impl.cc +++ b/source/common/upstream/load_balancer_impl.cc @@ -442,13 +442,13 @@ HostConstSharedPtr LoadBalancerBase::chooseHost(LoadBalancerContext* context) { bool LoadBalancerBase::isGlobalPanic(const HostSet& host_set) { uint64_t global_panic_threshold = std::min( 100, runtime_.snapshot().getInteger(RuntimePanicThreshold, default_healthy_panic_percent_)); - double healthy_percent = host_set.hosts().empty() + double healthy_percent = host_set.warmedHostCount() == 0 ? 0 - : 100.0 * host_set.healthyHosts().size() / host_set.hosts().size(); + : 100.0 * host_set.healthyHosts().size() / host_set.warmedHostCount(); - double degraded_percent = host_set.hosts().empty() + double degraded_percent = host_set.warmedHostCount() == 0 ? 0 - : 100.0 * host_set.degradedHosts().size() / host_set.hosts().size(); + : 100.0 * host_set.degradedHosts().size() / host_set.warmedHostCount(); // If the % of healthy hosts in the cluster is less than our panic threshold, we use all hosts. if ((healthy_percent + degraded_percent) < global_panic_threshold) { return true; diff --git a/source/common/upstream/subset_lb.h b/source/common/upstream/subset_lb.h index 74bd9dccee148..45ac38e29f09d 100644 --- a/source/common/upstream/subset_lb.h +++ b/source/common/upstream/subset_lb.h @@ -42,7 +42,8 @@ class SubsetLoadBalancer : public LoadBalancer, Logger::Loggable Date: Fri, 3 May 2019 02:56:27 -0400 Subject: [PATCH 07/28] format + other fixes Signed-off-by: Snow Pettersen --- api/envoy/admin/v2alpha/clusters.proto | 3 +++ api/envoy/api/v2/cds.proto | 4 ++-- source/common/upstream/subset_lb.cc | 6 +++--- source/server/http/admin.cc | 4 ++++ test/common/upstream/load_balancer_impl_test.cc | 8 +++++--- 5 files changed, 17 insertions(+), 8 deletions(-) diff --git a/api/envoy/admin/v2alpha/clusters.proto b/api/envoy/admin/v2alpha/clusters.proto index a910f35dc470e..31c2c40b10084 100644 --- a/api/envoy/admin/v2alpha/clusters.proto +++ b/api/envoy/admin/v2alpha/clusters.proto @@ -82,6 +82,9 @@ message HostHealthStatus { // health checking. bool pending_dynamic_removal = 5; + // The host has not yet been health checked. + bool pending_active_hc = 5; + // Health status as reported by EDS. Note: only HEALTHY and UNHEALTHY are currently supported // here. // TODO(mrice32): pipe through remaining EDS health status possibilities. diff --git a/api/envoy/api/v2/cds.proto b/api/envoy/api/v2/cds.proto index dbe5c0af8e3c6..18096506bf37b 100644 --- a/api/envoy/api/v2/cds.proto +++ b/api/envoy/api/v2/cds.proto @@ -541,8 +541,8 @@ message Cluster { // {healthy, healthy} // all traffic will still hit P0, as 1 / (3 - 2) = 1. // - // Enabling this will allow scaling up the number of hosts for a given cluster without entering panic - // mode or triggering priority spillover, assuming the hosts pass the first health check. + // Enabling this will allow scaling up the number of hosts for a given cluster without entering + // panic mode or triggering priority spillover, assuming the hosts pass the first health check. // // If panic mode is triggered, new hosts are still eligible for traffic; they simply do not // contribute to the calculation when deciding whether panic mode is enabled or not. diff --git a/source/common/upstream/subset_lb.cc b/source/common/upstream/subset_lb.cc index 7f8538dd2cf00..895dd08624569 100644 --- a/source/common/upstream/subset_lb.cc +++ b/source/common/upstream/subset_lb.cc @@ -589,9 +589,9 @@ void SubsetLoadBalancer::HostSubsetImpl::update(const HostVector& hosts_added, } } - // TODO(snowp): Right now we just pass hosts->size(), really this need to be filtering down the list - // of warmed hosts provided by the original_host_set_. We cannot compute the list of warmed hosts - // directly as subset updates happen on the worker threads. + // TODO(snowp): Right now we just pass hosts->size(), really this need to be filtering down the + // list of warmed hosts provided by the original_host_set_. We cannot compute the list of warmed + // hosts directly as subset updates happen on the worker threads. HostSetImpl::updateHosts( HostSetImpl::updateHostsParams(hosts, hosts_per_locality, healthy_hosts, healthy_hosts_per_locality, degraded_hosts, diff --git a/source/server/http/admin.cc b/source/server/http/admin.cc index daf4818a74cb0..d893b8be8c1eb 100644 --- a/source/server/http/admin.cc +++ b/source/server/http/admin.cc @@ -186,6 +186,10 @@ void setHealthFlag(Upstream::Host::HealthFlag flag, const Upstream::Host& host, health_status.set_pending_dynamic_removal( host.healthFlagGet(Upstream::Host::HealthFlag::PENDING_DYNAMIC_REMOVAL)); break; + case Upstream::Host::HealthFlag::PENDING_ACTIVE_HC: + health_status.set_pending_active_hc( + host.healthFlagGet(Upstream::Host::HealthFlag::PENDING_ACTIVE_HC)); + break; } } } // namespace diff --git a/test/common/upstream/load_balancer_impl_test.cc b/test/common/upstream/load_balancer_impl_test.cc index 5d303603c5a14..0069b50ee4c5d 100644 --- a/test/common/upstream/load_balancer_impl_test.cc +++ b/test/common/upstream/load_balancer_impl_test.cc @@ -133,7 +133,7 @@ TEST_P(LoadBalancerBaseTest, PrioritySelection) { MockHostSet& tertiary_host_set_ = *priority_set_.getMockHostSet(2); updateHostSet(tertiary_host_set_, 1 /* num_hosts */, 1 /* num_healthy_hosts */); EXPECT_EQ(0, lb_.percentageLoad(0)); - EXPECT_EQ(0, lb_.percentageLoad(1)); + EXPECT_EQ(0, lb_.percentageLoad(1)); EXPECT_EQ(100, lb_.percentageLoad(2)); priority_load.healthy_priority_load_ = HealthyLoad({0u, 0u, 100}); EXPECT_EQ(&tertiary_host_set_, &lb_.chooseHostSet(&context).first); @@ -574,7 +574,8 @@ TEST_P(FailoverTest, PrioritiesWithNotAllWarmedHosts) { // P0: 1 healthy, 1 unhealthy, 1 warmed. // P1: 1 healthy. // We then expect no spillover, since P0 is still overprovisioned. - host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80"), makeTestHost(info_, "tcp://127.0.0.1:81")}; + host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80"), + makeTestHost(info_, "tcp://127.0.0.1:81")}; host_set_.healthy_hosts_ = {host_set_.hosts_[0]}; failover_host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:82")}; failover_host_set_.healthy_hosts_ = failover_host_set_.hosts_; @@ -592,7 +593,8 @@ TEST_P(FailoverTest, PrioritiesWithZeroWarmedHosts) { // P0: 2 unhealthy, 0 warmed. // P1: 1 healthy. // We then expect all the traffic to spill over to P1 since P0 has an effective load of zero. - host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80"), makeTestHost(info_, "tcp://127.0.0.1:81")}; + host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80"), + makeTestHost(info_, "tcp://127.0.0.1:81")}; failover_host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:82")}; failover_host_set_.healthy_hosts_ = failover_host_set_.hosts_; ON_CALL(host_set_, warmedHostCount()).WillByDefault(Return(0)); From 821913e00909689d2cfae0f15d90e917fb295e17 Mon Sep 17 00:00:00 2001 From: Snow Pettersen Date: Fri, 3 May 2019 11:35:05 -0400 Subject: [PATCH 08/28] fix proto field id + format Signed-off-by: Snow Pettersen --- api/envoy/admin/v2alpha/clusters.proto | 2 +- source/common/upstream/load_balancer_impl.cc | 16 +++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/api/envoy/admin/v2alpha/clusters.proto b/api/envoy/admin/v2alpha/clusters.proto index 31c2c40b10084..b74ace25b5cc7 100644 --- a/api/envoy/admin/v2alpha/clusters.proto +++ b/api/envoy/admin/v2alpha/clusters.proto @@ -83,7 +83,7 @@ message HostHealthStatus { bool pending_dynamic_removal = 5; // The host has not yet been health checked. - bool pending_active_hc = 5; + bool pending_active_hc = 6; // Health status as reported by EDS. Note: only HEALTHY and UNHEALTHY are currently supported // here. diff --git a/source/common/upstream/load_balancer_impl.cc b/source/common/upstream/load_balancer_impl.cc index 1093e336ff949..13a43e2adb32a 100644 --- a/source/common/upstream/load_balancer_impl.cc +++ b/source/common/upstream/load_balancer_impl.cc @@ -442,13 +442,15 @@ HostConstSharedPtr LoadBalancerBase::chooseHost(LoadBalancerContext* context) { bool LoadBalancerBase::isGlobalPanic(const HostSet& host_set) { uint64_t global_panic_threshold = std::min( 100, runtime_.snapshot().getInteger(RuntimePanicThreshold, default_healthy_panic_percent_)); - double healthy_percent = host_set.warmedHostCount() == 0 - ? 0 - : 100.0 * host_set.healthyHosts().size() / host_set.warmedHostCount(); - - double degraded_percent = host_set.warmedHostCount() == 0 - ? 0 - : 100.0 * host_set.degradedHosts().size() / host_set.warmedHostCount(); + double healthy_percent = + host_set.warmedHostCount() == 0 + ? 0 + : 100.0 * host_set.healthyHosts().size() / host_set.warmedHostCount(); + + double degraded_percent = + host_set.warmedHostCount() == 0 + ? 0 + : 100.0 * host_set.degradedHosts().size() / host_set.warmedHostCount(); // If the % of healthy hosts in the cluster is less than our panic threshold, we use all hosts. if ((healthy_percent + degraded_percent) < global_panic_threshold) { return true; From 9de4c561b9440232229f6b121b519831383fc310 Mon Sep 17 00:00:00 2001 From: Snow Pettersen Date: Fri, 3 May 2019 13:37:45 -0400 Subject: [PATCH 09/28] spelling + fix load_balancer_benchmark build Signed-off-by: Snow Pettersen --- test/common/upstream/load_balancer_benchmark.cc | 2 +- test/common/upstream/upstream_impl_test.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/common/upstream/load_balancer_benchmark.cc b/test/common/upstream/load_balancer_benchmark.cc index 329e4eef53c4b..6985875cf5802 100644 --- a/test/common/upstream/load_balancer_benchmark.cc +++ b/test/common/upstream/load_balancer_benchmark.cc @@ -33,7 +33,7 @@ class BaseTester { HostSetImpl::updateHostsParams(updated_hosts, nullptr, std::make_shared(*updated_hosts), nullptr), - {}, hosts, {}, absl::nullopt); + {}, hosts, {}, hosts.size(), absl::nullopt); } PrioritySetImpl priority_set_; diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc index 00e7b7d30b1d6..350e48baf81e2 100644 --- a/test/common/upstream/upstream_impl_test.cc +++ b/test/common/upstream/upstream_impl_test.cc @@ -2310,7 +2310,7 @@ TEST_F(HostSetImplLocalityTest, AllUnhealthy) { EXPECT_FALSE(host_set_.chooseHealthyLocality().has_value()); } -// When a locality has endpoints that have not yet been warmed, weigth calculation should ignore +// When a locality has endpoints that have not yet been warmed, weight calculation should ignore // these hosts. TEST_F(HostSetImplLocalityTest, NotWarmedHostsLocality) { ON_CALL(*info_, warmHosts()).WillByDefault(Return(true)); From 0bc3d037aa24a5b21775c10de06685a98290cabf Mon Sep 17 00:00:00 2001 From: Snow Pettersen Date: Fri, 3 May 2019 17:53:01 -0400 Subject: [PATCH 10/28] pass whole list of hosts to faclitate subsetting Signed-off-by: Snow Pettersen --- include/envoy/upstream/types.h | 2 + include/envoy/upstream/upstream.h | 17 +++--- .../common/upstream/cluster_manager_impl.cc | 17 +++--- source/common/upstream/cluster_manager_impl.h | 1 - source/common/upstream/eds.cc | 9 +-- .../upstream/health_discovery_service.cc | 2 +- source/common/upstream/load_balancer_impl.cc | 17 +++--- .../common/upstream/original_dst_cluster.cc | 4 +- source/common/upstream/subset_lb.cc | 23 ++++--- source/common/upstream/subset_lb.h | 3 +- source/common/upstream/upstream_impl.cc | 60 ++++++++----------- source/common/upstream/upstream_impl.h | 27 +++++---- .../upstream/cluster_manager_impl_test.cc | 30 +++++----- .../upstream/load_balancer_impl_test.cc | 45 +++++++++++++- test/common/upstream/upstream_impl_test.cc | 41 ++++++------- test/mocks/upstream/mocks.cc | 2 +- test/mocks/upstream/mocks.h | 17 ++---- 17 files changed, 168 insertions(+), 149 deletions(-) diff --git a/include/envoy/upstream/types.h b/include/envoy/upstream/types.h index b73e66b220ea5..cde900e24b551 100644 --- a/include/envoy/upstream/types.h +++ b/include/envoy/upstream/types.h @@ -55,6 +55,8 @@ struct HealthyAvailability : PriorityAvailability { struct Healthy {}; // Phantom type indicating that the type is related to degraded hosts. struct Degraded {}; +// Phantom type indicating that the type is related to warmed hosts. +struct Warmed {}; } // namespace Upstream } // namespace Envoy diff --git a/include/envoy/upstream/upstream.h b/include/envoy/upstream/upstream.h index b47e6ae232735..8385252f3eb49 100644 --- a/include/envoy/upstream/upstream.h +++ b/include/envoy/upstream/upstream.h @@ -201,12 +201,14 @@ typedef std::shared_ptr HostConstSharedPtr; typedef std::vector HostVector; typedef Phantom HealthyHostVector; typedef Phantom DegradedHostVector; +typedef Phantom WarmedHostVector; typedef std::unordered_map HostMap; typedef std::shared_ptr HostVectorSharedPtr; typedef std::shared_ptr HostVectorConstSharedPtr; typedef std::shared_ptr HealthyHostVectorConstSharedPtr; typedef std::shared_ptr DegradedHostVectorConstSharedPtr; +typedef std::shared_ptr WarmedHostVectorConstSharedPtr; typedef std::unique_ptr HostListPtr; typedef std::unordered_map @@ -289,6 +291,11 @@ class HostSet { */ virtual const HostVector& degradedHosts() const PURE; + /* + * @return all warmed hosts contained in the set at the current time. + * */ + virtual const HostVector& warmedHosts() const PURE; + /** * @return hosts per locality. */ @@ -335,13 +342,6 @@ class HostSet { * @return uint32_t the overprovisioning factor of this host set. */ virtual uint32_t overprovisioningFactor() const PURE; - - /** - * @return uint32_t the number of warmed hosts in this host set. A host is considered warming - * if active health checking is enabled and the host has yet to be health checked for the first - * time. - */ - virtual uint32_t warmedHostCount() const PURE; }; typedef std::unique_ptr HostSetPtr; @@ -394,6 +394,7 @@ class PrioritySet { HostVectorConstSharedPtr hosts; HealthyHostVectorConstSharedPtr healthy_hosts; DegradedHostVectorConstSharedPtr degraded_hosts; + WarmedHostVectorConstSharedPtr warmed_hosts; HostsPerLocalityConstSharedPtr hosts_per_locality; HostsPerLocalityConstSharedPtr healthy_hosts_per_locality; HostsPerLocalityConstSharedPtr degraded_hosts_per_locality; @@ -413,7 +414,6 @@ class PrioritySet { virtual void updateHosts(uint32_t priority, UpdateHostsParams&& update_host_params, LocalityWeightsConstSharedPtr locality_weights, const HostVector& hosts_added, const HostVector& hosts_removed, - uint32_t warmed_host_count, absl::optional overprovisioning_factor) PURE; /** @@ -435,7 +435,6 @@ class PrioritySet { virtual void updateHosts(uint32_t priority, UpdateHostsParams&& update_host_params, LocalityWeightsConstSharedPtr locality_weights, const HostVector& hosts_added, const HostVector& hosts_removed, - uint32_t warmed_host_count, absl::optional overprovisioning_factor) PURE; }; diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index 1ca31a2636127..7e37e25e87ac5 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -687,24 +687,22 @@ void ClusterManagerImpl::postThreadLocalClusterUpdate(const Cluster& cluster, ui host_set->healthyHostsPerLocality().clone(); HostsPerLocalityConstSharedPtr degraded_hosts_per_locality_copy = host_set->degradedHostsPerLocality().clone(); - + WarmedHostVectorConstSharedPtr warmed_hosts_copy(new WarmedHostVector(host_set->warmedHosts())); auto warmed_hosts_per_locality_copy = host_set->warmedHostsPerLocality().clone(); tls_->runOnAllThreads([this, name = cluster.info()->name(), priority, hosts_copy, healthy_hosts_copy, degraded_hosts_copy, hosts_per_locality_copy, healthy_hosts_per_locality_copy, degraded_hosts_per_locality_copy, - warmed_hosts_per_locality_copy, + warmed_hosts_copy, warmed_hosts_per_locality_copy, locality_weights = host_set->localityWeights(), hosts_added, hosts_removed, - overprovisioning_factor = host_set->overprovisioningFactor(), - warmed_host_count = host_set->warmedHostCount()]() { + overprovisioning_factor = host_set->overprovisioningFactor()]() { ThreadLocalClusterManagerImpl::updateClusterMembership( name, priority, HostSetImpl::updateHostsParams(hosts_copy, hosts_per_locality_copy, healthy_hosts_copy, healthy_hosts_per_locality_copy, degraded_hosts_copy, - degraded_hosts_per_locality_copy, + degraded_hosts_per_locality_copy, warmed_hosts_copy, warmed_hosts_per_locality_copy), - locality_weights, hosts_added, hosts_removed, *tls_, warmed_host_count, - overprovisioning_factor); + locality_weights, hosts_added, hosts_removed, *tls_, overprovisioning_factor); }); } @@ -976,8 +974,7 @@ void ClusterManagerImpl::ThreadLocalClusterManagerImpl::updateClusterMembership( const std::string& name, uint32_t priority, PrioritySet::UpdateHostsParams&& update_hosts_params, LocalityWeightsConstSharedPtr locality_weights, const HostVector& hosts_added, - const HostVector& hosts_removed, ThreadLocal::Slot& tls, uint32_t warmed_host_count, - uint64_t overprovisioning_factor) { + const HostVector& hosts_removed, ThreadLocal::Slot& tls, uint64_t overprovisioning_factor) { ThreadLocalClusterManagerImpl& config = tls.getTyped(); @@ -987,7 +984,7 @@ void ClusterManagerImpl::ThreadLocalClusterManagerImpl::updateClusterMembership( hosts_added.size(), hosts_removed.size()); cluster_entry->priority_set_.updateHosts(priority, std::move(update_hosts_params), std::move(locality_weights), hosts_added, hosts_removed, - warmed_host_count, overprovisioning_factor); + overprovisioning_factor); // If an LB is thread aware, create a new worker local LB on membership changes. if (cluster_entry->lb_factory_ != nullptr) { diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index d3dba20b2f691..4cb0914ee20c4 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -330,7 +330,6 @@ class ClusterManagerImpl : public ClusterManager, Logger::LoggablehostsPerLocality().filter( {[&host_to_exclude](const Host& host) { return &host != host_to_exclude.get(); }})[0]; - // TODO(snowp): Do this inside the loop over hosts above. - const uint32_t warmed_hosts = std::count_if(hosts_copy->begin(), hosts_copy->end(), - [](const auto& host) { return host->warmed(); }); - prioritySet().updateHosts( - priority, HostSetImpl::partitionHosts(hosts_copy, hosts_per_locality_copy), - host_set->localityWeights(), {}, hosts_to_remove, warmed_hosts, absl::nullopt); + prioritySet().updateHosts(priority, + HostSetImpl::partitionHosts(hosts_copy, hosts_per_locality_copy), + host_set->localityWeights(), {}, hosts_to_remove, absl::nullopt); } if (host_to_exclude != nullptr) { diff --git a/source/common/upstream/health_discovery_service.cc b/source/common/upstream/health_discovery_service.cc index fbb72d4488251..d104759accdb8 100644 --- a/source/common/upstream/health_discovery_service.cc +++ b/source/common/upstream/health_discovery_service.cc @@ -258,7 +258,7 @@ void HdsCluster::initialize(std::function callback) { priority_set_.updateHosts( 0, HostSetImpl::partitionHosts(initial_hosts_, HostsPerLocalityImpl::empty()), {}, - *initial_hosts_, {}, initial_hosts_->size(), absl::nullopt); + *initial_hosts_, {}, absl::nullopt); } void HdsCluster::setOutlierDetector(const Outlier::DetectorSharedPtr&) { diff --git a/source/common/upstream/load_balancer_impl.cc b/source/common/upstream/load_balancer_impl.cc index 13a43e2adb32a..ac5c6526283ca 100644 --- a/source/common/upstream/load_balancer_impl.cc +++ b/source/common/upstream/load_balancer_impl.cc @@ -136,7 +136,7 @@ void LoadBalancerBase::recalculatePerPriorityState(uint32_t priority, // by the overprovisioning factor. HostSet& host_set = *priority_set.hostSetsPerPriority()[priority]; per_priority_health.get()[priority] = 0; - if (!host_set.hosts().empty() && host_set.warmedHostCount() > 0) { + if (!host_set.warmedHosts().empty()) { // Each priority level's health is ratio of healthy hosts to total number of hosts in a priority // multiplied by overprovisioning factor of 1.4 and capped at 100%. It means that if all // hosts are healthy that priority's health is 100%*1.4=140% and is capped at 100% which results @@ -144,12 +144,12 @@ void LoadBalancerBase::recalculatePerPriorityState(uint32_t priority, // capped at 100%). per_priority_health.get()[priority] = std::min(100, (host_set.overprovisioningFactor() * - host_set.healthyHosts().size() / host_set.warmedHostCount())); + host_set.healthyHosts().size() / host_set.warmedHosts().size())); // We perform the same computation for degraded hosts. per_priority_degraded.get()[priority] = std::min(100, (host_set.overprovisioningFactor() * - host_set.degradedHosts().size() / host_set.warmedHostCount())); + host_set.degradedHosts().size() / host_set.warmedHosts().size())); } // Now that we've updated health for the changed priority level, we need to calculate percentage @@ -442,15 +442,14 @@ HostConstSharedPtr LoadBalancerBase::chooseHost(LoadBalancerContext* context) { bool LoadBalancerBase::isGlobalPanic(const HostSet& host_set) { uint64_t global_panic_threshold = std::min( 100, runtime_.snapshot().getInteger(RuntimePanicThreshold, default_healthy_panic_percent_)); - double healthy_percent = - host_set.warmedHostCount() == 0 - ? 0 - : 100.0 * host_set.healthyHosts().size() / host_set.warmedHostCount(); + double healthy_percent = host_set.warmedHosts().empty() ? 0 + : 100.0 * host_set.healthyHosts().size() / + host_set.warmedHosts().size(); double degraded_percent = - host_set.warmedHostCount() == 0 + host_set.warmedHosts().empty() ? 0 - : 100.0 * host_set.degradedHosts().size() / host_set.warmedHostCount(); + : 100.0 * host_set.degradedHosts().size() / host_set.warmedHosts().size(); // If the % of healthy hosts in the cluster is less than our panic threshold, we use all hosts. if ((healthy_percent + degraded_percent) < global_panic_threshold) { return true; diff --git a/source/common/upstream/original_dst_cluster.cc b/source/common/upstream/original_dst_cluster.cc index 116d4234d09f9..f2a0af8911efc 100644 --- a/source/common/upstream/original_dst_cluster.cc +++ b/source/common/upstream/original_dst_cluster.cc @@ -153,7 +153,7 @@ void OriginalDstCluster::addHost(HostSharedPtr& host) { new_hosts->emplace_back(host); priority_set_.updateHosts(0, HostSetImpl::partitionHosts(new_hosts, HostsPerLocalityImpl::empty()), - {}, {std::move(host)}, {}, new_hosts->size(), absl::nullopt); + {}, {std::move(host)}, {}, absl::nullopt); } void OriginalDstCluster::cleanup() { @@ -180,7 +180,7 @@ void OriginalDstCluster::cleanup() { if (!to_be_removed.empty()) { priority_set_.updateHosts(0, HostSetImpl::partitionHosts(new_hosts, HostsPerLocalityImpl::empty()), - {}, {}, to_be_removed, new_hosts->size(), absl::nullopt); + {}, {}, to_be_removed, absl::nullopt); } cleanup_timer_->enableTimer(cleanup_interval_ms_); diff --git a/source/common/upstream/subset_lb.cc b/source/common/upstream/subset_lb.cc index 895dd08624569..9460ecb7885fb 100644 --- a/source/common/upstream/subset_lb.cc +++ b/source/common/upstream/subset_lb.cc @@ -551,6 +551,14 @@ void SubsetLoadBalancer::HostSubsetImpl::update(const HostVector& hosts_added, } } + auto warmed_hosts = std::make_shared(); + warmed_hosts->get().reserve(original_host_set_.degradedHosts().size()); + for (const auto& host : original_host_set_.warmedHosts()) { + if (cached_predicate(*host)) { + degraded_hosts->get().emplace_back(host); + } + } + // If we only have one locality we can avoid the first call to filter() by // just creating a new HostsPerLocality from the list of all hosts. // @@ -589,15 +597,12 @@ void SubsetLoadBalancer::HostSubsetImpl::update(const HostVector& hosts_added, } } - // TODO(snowp): Right now we just pass hosts->size(), really this need to be filtering down the - // list of warmed hosts provided by the original_host_set_. We cannot compute the list of warmed - // hosts directly as subset updates happen on the worker threads. - HostSetImpl::updateHosts( - HostSetImpl::updateHostsParams(hosts, hosts_per_locality, healthy_hosts, - healthy_hosts_per_locality, degraded_hosts, - degraded_hosts_per_locality, warmed_hosts_per_locality), - determineLocalityWeights(*hosts_per_locality), filtered_added, filtered_removed, - hosts->size(), absl::nullopt); + HostSetImpl::updateHosts(HostSetImpl::updateHostsParams( + hosts, hosts_per_locality, healthy_hosts, healthy_hosts_per_locality, + degraded_hosts, degraded_hosts_per_locality, warmed_hosts, + warmed_hosts_per_locality), + determineLocalityWeights(*hosts_per_locality), filtered_added, + filtered_removed, absl::nullopt); } LocalityWeightsConstSharedPtr SubsetLoadBalancer::HostSubsetImpl::determineLocalityWeights( diff --git a/source/common/upstream/subset_lb.h b/source/common/upstream/subset_lb.h index 45ac38e29f09d..3980cce1330a1 100644 --- a/source/common/upstream/subset_lb.h +++ b/source/common/upstream/subset_lb.h @@ -42,8 +42,7 @@ class SubsetLoadBalancer : public LoadBalancer, Logger::Loggable HostsPerLocalityImpl::filter( void HostSetImpl::updateHosts(PrioritySet::UpdateHostsParams&& update_hosts_params, LocalityWeightsConstSharedPtr locality_weights, const HostVector& hosts_added, const HostVector& hosts_removed, - uint32_t warmed_host_count, absl::optional overprovisioning_factor) { if (overprovisioning_factor.has_value()) { ASSERT(overprovisioning_factor.value() > 0); @@ -321,12 +320,12 @@ void HostSetImpl::updateHosts(PrioritySet::UpdateHostsParams&& update_hosts_para hosts_ = std::move(update_hosts_params.hosts); healthy_hosts_ = std::move(update_hosts_params.healthy_hosts); degraded_hosts_ = std::move(update_hosts_params.degraded_hosts); + warmed_hosts_ = std::move(update_hosts_params.warmed_hosts); hosts_per_locality_ = std::move(update_hosts_params.hosts_per_locality); healthy_hosts_per_locality_ = std::move(update_hosts_params.healthy_hosts_per_locality); degraded_hosts_per_locality_ = std::move(update_hosts_params.degraded_hosts_per_locality); warmed_hosts_per_locality_ = std::move(update_hosts_params.warmed_hosts_per_locality); locality_weights_ = std::move(locality_weights); - warmed_host_count_ = warmed_host_count; rebuildLocalityScheduler(healthy_locality_scheduler_, healthy_locality_entries_, *healthy_hosts_per_locality_, healthy_hosts_->get(), hosts_per_locality_, @@ -421,12 +420,10 @@ HostSetImpl::updateHostsParams(HostVectorConstSharedPtr hosts, // TODO(snowp): Move this function into test/ // This overload is only used as a convenience method in test, so it should live there. - auto warmed_per_locality = - hosts_per_locality->filter({[](const auto& host) { return host.warmed(); }})[0]; - return updateHostsParams(std::move(hosts), std::move(hosts_per_locality), - std::move(healthy_hosts), std::move(healthy_hosts_per_locality), - std::make_shared(), - HostsPerLocalityImpl::empty(), std::move(warmed_per_locality)); + return updateHostsParams( + hosts, hosts_per_locality, std::move(healthy_hosts), std::move(healthy_hosts_per_locality), + std::make_shared(), HostsPerLocalityImpl::empty(), + std::make_shared(*hosts), hosts_per_locality); } PrioritySet::UpdateHostsParams @@ -436,10 +433,12 @@ HostSetImpl::updateHostsParams(HostVectorConstSharedPtr hosts, HostsPerLocalityConstSharedPtr healthy_hosts_per_locality, DegradedHostVectorConstSharedPtr degraded_hosts, HostsPerLocalityConstSharedPtr degraded_hosts_per_locality, + WarmedHostVectorConstSharedPtr warmed_hosts, HostsPerLocalityConstSharedPtr warmed_counts_per_locality) { return PrioritySet::UpdateHostsParams{std::move(hosts), std::move(healthy_hosts), std::move(degraded_hosts), + std::move(warmed_hosts), std::move(hosts_per_locality), std::move(healthy_hosts_per_locality), std::move(degraded_hosts_per_locality), @@ -449,15 +448,16 @@ HostSetImpl::updateHostsParams(HostVectorConstSharedPtr hosts, PrioritySet::UpdateHostsParams HostSetImpl::partitionHosts(HostVectorConstSharedPtr hosts, HostsPerLocalityConstSharedPtr hosts_per_locality) { - auto healthy_and_degraded_hosts = ClusterImplBase::partitionHostList(*hosts); + auto partitioned_hosts = ClusterImplBase::partitionHostList(*hosts); auto healthy_degraded_warmed_hosts_per_locality = ClusterImplBase::partitionHostsPerLocality(*hosts_per_locality); return updateHostsParams(std::move(hosts), std::move(hosts_per_locality), - std::move(healthy_and_degraded_hosts.first), + std::move(std::get<0>(partitioned_hosts)), std::move(std::get<0>(healthy_degraded_warmed_hosts_per_locality)), - std::move(healthy_and_degraded_hosts.second), + std::move(std::get<1>(partitioned_hosts)), std::move(std::get<1>(healthy_degraded_warmed_hosts_per_locality)), + std::move(std::get<2>(partitioned_hosts)), std::move(std::get<2>(healthy_degraded_warmed_hosts_per_locality))); } @@ -499,13 +499,12 @@ PrioritySetImpl::getOrCreateHostSet(uint32_t priority, void PrioritySetImpl::updateHosts(uint32_t priority, UpdateHostsParams&& update_hosts_params, LocalityWeightsConstSharedPtr locality_weights, const HostVector& hosts_added, const HostVector& hosts_removed, - uint32_t warmed_host_count, absl::optional overprovisioning_factor) { // Ensure that we have a HostSet for the given priority. getOrCreateHostSet(priority, overprovisioning_factor); static_cast(host_sets_[priority].get()) ->updateHosts(std::move(update_hosts_params), std::move(locality_weights), hosts_added, - hosts_removed, warmed_host_count, overprovisioning_factor); + hosts_removed, overprovisioning_factor); if (!batch_update_) { runUpdateCallbacks(hosts_added, hosts_removed); @@ -528,8 +527,7 @@ void PrioritySetImpl::batchHostUpdate(BatchUpdateCb& callback) { void PrioritySetImpl::BatchUpdateScope::updateHosts( uint32_t priority, PrioritySet::UpdateHostsParams&& update_hosts_params, LocalityWeightsConstSharedPtr locality_weights, const HostVector& hosts_added, - const HostVector& hosts_removed, uint32_t warmed_host_count, - absl::optional overprovisioning_factor) { + const HostVector& hosts_removed, absl::optional overprovisioning_factor) { // We assume that each call updates a different priority. ASSERT(priorities_.find(priority) == priorities_.end()); priorities_.insert(priority); @@ -543,7 +541,7 @@ void PrioritySetImpl::BatchUpdateScope::updateHosts( } parent_.updateHosts(priority, std::move(update_hosts_params), locality_weights, hosts_added, - hosts_removed, warmed_host_count, overprovisioning_factor); + hosts_removed, overprovisioning_factor); } ClusterStats ClusterInfoImpl::generateStats(Stats::Scope& scope) { @@ -712,10 +710,12 @@ ClusterImplBase::ClusterImplBase( }); } -std::pair +std::tuple ClusterImplBase::partitionHostList(const HostVector& hosts) { auto healthy_list = std::make_shared(); auto degraded_list = std::make_shared(); + auto warmed_list = std::make_shared(); for (const auto& host : hosts) { if (host->health() == Host::Health::Healthy) { @@ -724,9 +724,12 @@ ClusterImplBase::partitionHostList(const HostVector& hosts) { if (host->health() == Host::Health::Degraded) { degraded_list->get().emplace_back(host); } + if (host->warmed()) { + warmed_list->get().emplace_back(host); + } } - return {healthy_list, degraded_list}; + return {healthy_list, degraded_list, warmed_list}; } std::tuplehosts())); - uint32_t warmed_hosts_count = host_set->hosts().size(); - if (health_checker_ != nullptr) { - warmed_hosts_count -= std::count_if(host_set->hosts().begin(), host_set->hosts().end(), - [](const auto& host) { return !host->warmed(); }); - } HostsPerLocalityConstSharedPtr hosts_per_locality_copy = host_set->hostsPerLocality().clone(); - prioritySet().updateHosts( - priority, HostSetImpl::partitionHosts(hosts_copy, hosts_per_locality_copy), - host_set->localityWeights(), {}, {}, warmed_hosts_count, absl::nullopt); + prioritySet().updateHosts(priority, + HostSetImpl::partitionHosts(hosts_copy, hosts_per_locality_copy), + host_set->localityWeights(), {}, {}, absl::nullopt); } } @@ -1017,12 +1015,7 @@ void PriorityStateManager::updateClusterPrioritySet( // We use std::map to guarantee a stable ordering for zone aware routing. std::map hosts_per_locality; - uint32_t warmed_host_count = hosts->size(); - for (const HostSharedPtr& host : *hosts) { - if (parent_.healthChecker() != nullptr && !host->warmed()) { - warmed_host_count--; - } // Take into consideration when a non-EDS cluster has active health checking, i.e. to mark all // the hosts unhealthy (host->healthFlagSet(Host::HealthFlag::FAILED_ACTIVE_HC)) and then fire // update callbacks to start the health checking process. @@ -1065,13 +1058,12 @@ void PriorityStateManager::updateClusterPrioritySet( if (update_cb_ != nullptr) { update_cb_->updateHosts(priority, HostSetImpl::partitionHosts(hosts, per_locality_shared), std::move(locality_weights), hosts_added.value_or(*hosts), - hosts_removed.value_or({}), warmed_host_count, - overprovisioning_factor); + hosts_removed.value_or({}), overprovisioning_factor); } else { parent_.prioritySet().updateHosts( priority, HostSetImpl::partitionHosts(hosts, per_locality_shared), std::move(locality_weights), hosts_added.value_or(*hosts), - hosts_removed.value_or({}), warmed_host_count, overprovisioning_factor); + hosts_removed.value_or({}), overprovisioning_factor); } } diff --git a/source/common/upstream/upstream_impl.h b/source/common/upstream/upstream_impl.h index 241ed1f581ef9..f2b35625a3c12 100644 --- a/source/common/upstream/upstream_impl.h +++ b/source/common/upstream/upstream_impl.h @@ -280,13 +280,12 @@ class HostsPerLocalityImpl : public HostsPerLocality { */ class HostSetImpl : public HostSet { public: - HostSetImpl(uint32_t priority, uint32_t warmed_host_count, - absl::optional overprovisioning_factor) + HostSetImpl(uint32_t priority, absl::optional overprovisioning_factor) : priority_(priority), overprovisioning_factor_(overprovisioning_factor.has_value() ? overprovisioning_factor.value() : kDefaultOverProvisioningFactor), - warmed_host_count_(warmed_host_count), hosts_(new HostVector()), - healthy_hosts_(new HealthyHostVector()), degraded_hosts_(new DegradedHostVector()) {} + hosts_(new HostVector()), healthy_hosts_(new HealthyHostVector()), + degraded_hosts_(new DegradedHostVector()), warmed_hosts_(new WarmedHostVector()) {} /** * Install a callback that will be invoked when the host set membership changes. @@ -301,6 +300,7 @@ class HostSetImpl : public HostSet { const HostVector& hosts() const override { return *hosts_; } const HostVector& healthyHosts() const override { return healthy_hosts_->get(); } const HostVector& degradedHosts() const override { return degraded_hosts_->get(); } + const HostVector& warmedHosts() const override { return warmed_hosts_->get(); } const HostsPerLocality& hostsPerLocality() const override { return *hosts_per_locality_; } const HostsPerLocality& healthyHostsPerLocality() const override { return *healthy_hosts_per_locality_; @@ -316,7 +316,6 @@ class HostSetImpl : public HostSet { absl::optional chooseDegradedLocality() override; uint32_t priority() const override { return priority_; } uint32_t overprovisioningFactor() const override { return overprovisioning_factor_; } - uint32_t warmedHostCount() const override { return warmed_host_count_; } // Utility methods for creating UpdateHostsParams. static PrioritySet::UpdateHostsParams @@ -334,13 +333,14 @@ class HostSetImpl : public HostSet { HostsPerLocalityConstSharedPtr healthy_hosts_per_locality, DegradedHostVectorConstSharedPtr degraded_hosts, HostsPerLocalityConstSharedPtr degraded_hosts_per_locality, + WarmedHostVectorConstSharedPtr warmed_hosts, HostsPerLocalityConstSharedPtr warmed_counts_per_locality); static PrioritySet::UpdateHostsParams partitionHosts(HostVectorConstSharedPtr hosts, HostsPerLocalityConstSharedPtr hosts_per_locality); void updateHosts(PrioritySet::UpdateHostsParams&& update_hosts_params, LocalityWeightsConstSharedPtr locality_weights, const HostVector& hosts_added, - const HostVector& hosts_removed, uint32_t warmed_host_count, + const HostVector& hosts_removed, absl::optional overprovisioning_factor = absl::nullopt); protected: @@ -359,10 +359,10 @@ class HostSetImpl : public HostSet { uint32_t priority_; uint32_t overprovisioning_factor_; - uint32_t warmed_host_count_; HostVectorConstSharedPtr hosts_; HealthyHostVectorConstSharedPtr healthy_hosts_; DegradedHostVectorConstSharedPtr degraded_hosts_; + WarmedHostVectorConstSharedPtr warmed_hosts_; HostsPerLocalityConstSharedPtr hosts_per_locality_{HostsPerLocalityImpl::empty()}; HostsPerLocalityConstSharedPtr healthy_hosts_per_locality_{HostsPerLocalityImpl::empty()}; HostsPerLocalityConstSharedPtr degraded_hosts_per_locality_{HostsPerLocalityImpl::empty()}; @@ -434,7 +434,7 @@ class PrioritySetImpl : public PrioritySet { void updateHosts(uint32_t priority, UpdateHostsParams&& update_hosts_params, LocalityWeightsConstSharedPtr locality_weights, const HostVector& hosts_added, - const HostVector& hosts_removed, uint32_t warmed_host_count, + const HostVector& hosts_removed, absl::optional overprovisioning_factor = absl::nullopt) override; void batchHostUpdate(BatchUpdateCb& callback) override; @@ -443,7 +443,7 @@ class PrioritySetImpl : public PrioritySet { // Allows subclasses of PrioritySetImpl to create their own type of HostSetImpl. virtual HostSetImplPtr createHostSet(uint32_t priority, absl::optional overprovisioning_factor) { - return std::make_unique(priority, 0, overprovisioning_factor); + return std::make_unique(priority, overprovisioning_factor); } protected: @@ -481,7 +481,6 @@ class PrioritySetImpl : public PrioritySet { PrioritySet::UpdateHostsParams&& update_hosts_params, LocalityWeightsConstSharedPtr locality_weights, const HostVector& hosts_added, const HostVector& hosts_removed, - uint32_t warmed_host_count, absl::optional overprovisioning_factor) override; std::unordered_set all_hosts_added_; @@ -652,10 +651,12 @@ class ClusterImplBase : public Cluster, protected Logger::Loggable + // Partitions the provided list of hosts into three new lists containing the healthy, degraded + // and warmed hosts respectively. + static std::tuple partitionHostList(const HostVector& hosts); + // Partitions the provided list of hosts per locality into three new lists containing the healthy, // degraded and warmed hosts respectively. static std::tuplepriority_set_.updateHosts( 0, HostSetImpl::partitionHosts(hosts_ptr, HostsPerLocalityImpl::empty()), nullptr, hosts, {}, - hosts.size(), 100); + 100); auto* tls_cluster = cluster_manager_->get(cluster1->info_->name()); @@ -2177,7 +2177,7 @@ TEST_F(ClusterManagerImplTest, MergedUpdates) { HostSetImpl::updateHostsParams(hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - {}, hosts_added, hosts_removed, hosts->size(), absl::nullopt); + {}, hosts_added, hosts_removed, absl::nullopt); EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.update_merge_cancelled").value()); @@ -2189,13 +2189,13 @@ TEST_F(ClusterManagerImplTest, MergedUpdates) { HostSetImpl::updateHostsParams(hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - {}, hosts_added, hosts_removed, hosts->size(), absl::nullopt); + {}, hosts_added, hosts_removed, absl::nullopt); cluster.prioritySet().updateHosts( 0, HostSetImpl::updateHostsParams(hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - {}, hosts_added, hosts_removed, hosts->size(), absl::nullopt); + {}, hosts_added, hosts_removed, absl::nullopt); EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.update_merge_cancelled").value()); @@ -2214,7 +2214,7 @@ TEST_F(ClusterManagerImplTest, MergedUpdates) { HostSetImpl::updateHostsParams(hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - {}, hosts_added, hosts_removed, hosts->size(), absl::nullopt); + {}, hosts_added, hosts_removed, absl::nullopt); EXPECT_EQ(2, factory_.stats_.counter("cluster_manager.cluster_updated").value()); EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.update_merge_cancelled").value()); @@ -2228,7 +2228,7 @@ TEST_F(ClusterManagerImplTest, MergedUpdates) { HostSetImpl::updateHostsParams(hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - {}, hosts_added, hosts_removed, hosts->size(), absl::nullopt); + {}, hosts_added, hosts_removed, absl::nullopt); (*hosts)[0]->healthFlagSet(Host::HealthFlag::FAILED_EDS_HEALTH); cluster.prioritySet().updateHosts( @@ -2236,7 +2236,7 @@ TEST_F(ClusterManagerImplTest, MergedUpdates) { HostSetImpl::updateHostsParams(hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - {}, hosts_added, hosts_removed, hosts->size(), absl::nullopt); + {}, hosts_added, hosts_removed, absl::nullopt); (*hosts)[0]->weight(100); cluster.prioritySet().updateHosts( @@ -2244,7 +2244,7 @@ TEST_F(ClusterManagerImplTest, MergedUpdates) { HostSetImpl::updateHostsParams(hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - {}, hosts_added, hosts_removed, hosts->size(), absl::nullopt); + {}, hosts_added, hosts_removed, absl::nullopt); // Updates not delivered yet. EXPECT_EQ(2, factory_.stats_.counter("cluster_manager.cluster_updated").value()); @@ -2258,7 +2258,7 @@ TEST_F(ClusterManagerImplTest, MergedUpdates) { HostSetImpl::updateHostsParams(hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - {}, hosts_added, hosts_removed, hosts->size(), absl::nullopt); + {}, hosts_added, hosts_removed, absl::nullopt); EXPECT_EQ(3, factory_.stats_.counter("cluster_manager.cluster_updated").value()); EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value()); @@ -2295,7 +2295,7 @@ TEST_F(ClusterManagerImplTest, MergedUpdatesOutOfWindow) { HostSetImpl::updateHostsParams(hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - {}, hosts_added, hosts_removed, hosts->size(), absl::nullopt); + {}, hosts_added, hosts_removed, absl::nullopt); EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value()); EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.update_out_of_merge_window").value()); @@ -2323,7 +2323,7 @@ TEST_F(ClusterManagerImplTest, MergedUpdatesInsideWindow) { HostSetImpl::updateHostsParams(hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - {}, hosts_added, hosts_removed, hosts->size(), absl::nullopt); + {}, hosts_added, hosts_removed, absl::nullopt); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.update_out_of_merge_window").value()); @@ -2359,7 +2359,7 @@ TEST_F(ClusterManagerImplTest, MergedUpdatesOutOfWindowDisabled) { HostSetImpl::updateHostsParams(hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - {}, hosts_added, hosts_removed, hosts->size(), absl::nullopt); + {}, hosts_added, hosts_removed, absl::nullopt); EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.update_out_of_merge_window").value()); @@ -2426,7 +2426,7 @@ TEST_F(ClusterManagerImplTest, MergedUpdatesDestroyedOnUpdate) { HostSetImpl::updateHostsParams(hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - {}, hosts_added, hosts_removed, hosts->size(), absl::nullopt); + {}, hosts_added, hosts_removed, absl::nullopt); EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.update_merge_cancelled").value()); @@ -2438,13 +2438,13 @@ TEST_F(ClusterManagerImplTest, MergedUpdatesDestroyedOnUpdate) { HostSetImpl::updateHostsParams(hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - {}, hosts_added, hosts_removed, hosts->size(), absl::nullopt); + {}, hosts_added, hosts_removed, absl::nullopt); cluster.prioritySet().updateHosts( 0, HostSetImpl::updateHostsParams(hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - {}, hosts_added, hosts_removed, hosts->size(), absl::nullopt); + {}, hosts_added, hosts_removed, absl::nullopt); EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.update_merge_cancelled").value()); diff --git a/test/common/upstream/load_balancer_impl_test.cc b/test/common/upstream/load_balancer_impl_test.cc index 0069b50ee4c5d..1337534c664ca 100644 --- a/test/common/upstream/load_balancer_impl_test.cc +++ b/test/common/upstream/load_balancer_impl_test.cc @@ -81,6 +81,7 @@ class LoadBalancerBaseTest : public LoadBalancerTestBase { for (; i < num_degraded_hosts; ++i) { host_set.degraded_hosts_.push_back(host_set.hosts_[i]); } + host_set.warmed_hosts_ = host_set.hosts_; host_set.runCallbacks({}, {}); } @@ -443,7 +444,7 @@ class RoundRobinLoadBalancerTest : public LoadBalancerTestBase { HostSetImpl::updateHostsParams(hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - {}, empty_host_vector_, empty_host_vector_, hosts->size(), absl::nullopt); + {}, empty_host_vector_, empty_host_vector_, absl::nullopt); } std::shared_ptr local_priority_set_; @@ -459,8 +460,10 @@ typedef RoundRobinLoadBalancerTest FailoverTest; // Ensure if all the hosts with priority 0 unhealthy, the next priority hosts are used. TEST_P(FailoverTest, BasicFailover) { host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80")}; + host_set_.warmed_hosts_ = host_set_.hosts_; failover_host_set_.healthy_hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:82")}; failover_host_set_.hosts_ = failover_host_set_.healthy_hosts_; + failover_host_set_.warmed_hosts_ = failover_host_set_.hosts_; init(false); EXPECT_EQ(failover_host_set_.healthy_hosts_[0], lb_->chooseHost(nullptr)); } @@ -469,6 +472,7 @@ TEST_P(FailoverTest, BasicFailover) { TEST_P(FailoverTest, BasicDegradedHosts) { host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80")}; host_set_.degraded_hosts_ = host_set_.hosts_; + host_set_.warmed_hosts_ = host_set_.hosts_; failover_host_set_.hosts_ = failover_host_set_.healthy_hosts_; init(false); EXPECT_EQ(host_set_.degraded_hosts_[0], lb_->chooseHost(nullptr)); @@ -479,8 +483,10 @@ TEST_P(FailoverTest, BasicDegradedHosts) { TEST_P(FailoverTest, BasicFailoverDegradedHosts) { host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80")}; host_set_.degraded_hosts_ = host_set_.hosts_; + host_set_.warmed_hosts_ = host_set_.hosts_; failover_host_set_.healthy_hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:82")}; failover_host_set_.hosts_ = failover_host_set_.healthy_hosts_; + failover_host_set_.warmed_hosts_ = failover_host_set_.hosts_; init(false); EXPECT_EQ(failover_host_set_.healthy_hosts_[0], lb_->chooseHost(nullptr)); } @@ -488,7 +494,9 @@ TEST_P(FailoverTest, BasicFailoverDegradedHosts) { // Test that extending the priority set with an existing LB causes the correct updates. TEST_P(FailoverTest, PriorityUpdatesWithLocalHostSet) { host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80")}; + host_set_.warmed_hosts_ = host_set_.hosts_; failover_host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:81")}; + failover_host_set_.warmed_hosts_ = failover_host_set_.hosts_; init(false); // With both the primary and failover hosts unhealthy, we should select an // unhealthy primary host. @@ -500,6 +508,7 @@ TEST_P(FailoverTest, PriorityUpdatesWithLocalHostSet) { HostVectorSharedPtr hosts(new HostVector({makeTestHost(info_, "tcp://127.0.0.1:82")})); tertiary_host_set_.hosts_ = *hosts; tertiary_host_set_.healthy_hosts_ = tertiary_host_set_.hosts_; + tertiary_host_set_.warmed_hosts_ = tertiary_host_set_.hosts_; HostVector add_hosts; add_hosts.push_back(tertiary_host_set_.hosts_[0]); tertiary_host_set_.runCallbacks(add_hosts, {}); @@ -519,7 +528,9 @@ TEST_P(FailoverTest, PriorityUpdatesWithLocalHostSet) { // Test extending the priority set. TEST_P(FailoverTest, ExtendPrioritiesUpdatingPrioritySet) { host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80")}; + host_set_.warmed_hosts_ = host_set_.hosts_; failover_host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:81")}; + failover_host_set_.warmed_hosts_ = failover_host_set_.hosts_; init(true); // With both the primary and failover hosts unhealthy, we should select an // unhealthy primary host. @@ -531,6 +542,7 @@ TEST_P(FailoverTest, ExtendPrioritiesUpdatingPrioritySet) { HostVectorSharedPtr hosts(new HostVector({makeTestHost(info_, "tcp://127.0.0.1:82")})); tertiary_host_set_.hosts_ = *hosts; tertiary_host_set_.healthy_hosts_ = tertiary_host_set_.hosts_; + tertiary_host_set_.warmed_hosts_ = tertiary_host_set_.hosts_; HostVector add_hosts; add_hosts.push_back(tertiary_host_set_.hosts_[0]); tertiary_host_set_.runCallbacks(add_hosts, {}); @@ -544,7 +556,9 @@ TEST_P(FailoverTest, ExtendPrioritiesUpdatingPrioritySet) { TEST_P(FailoverTest, ExtendPrioritiesWithLocalPrioritySet) { host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80")}; + host_set_.warmed_hosts_ = host_set_.hosts_; failover_host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:81")}; + failover_host_set_.warmed_hosts_ = failover_host_set_.hosts_; init(true); // With both the primary and failover hosts unhealthy, we should select an // unhealthy primary host. @@ -555,6 +569,7 @@ TEST_P(FailoverTest, ExtendPrioritiesWithLocalPrioritySet) { MockHostSet& tertiary_host_set_ = *priority_set_.getMockHostSet(2); HostVectorSharedPtr hosts2(new HostVector({makeTestHost(info_, "tcp://127.0.0.1:84")})); tertiary_host_set_.hosts_ = *hosts2; + tertiary_host_set_.warmed_hosts_ = *hosts2; tertiary_host_set_.healthy_hosts_ = tertiary_host_set_.hosts_; HostVector add_hosts; add_hosts.push_back(tertiary_host_set_.hosts_[0]); @@ -577,9 +592,9 @@ TEST_P(FailoverTest, PrioritiesWithNotAllWarmedHosts) { host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80"), makeTestHost(info_, "tcp://127.0.0.1:81")}; host_set_.healthy_hosts_ = {host_set_.hosts_[0]}; + host_set_.warmed_hosts_ = {host_set_.hosts_[0]}; failover_host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:82")}; failover_host_set_.healthy_hosts_ = failover_host_set_.hosts_; - ON_CALL(host_set_, warmedHostCount()).WillByDefault(Return(2)); init(true); EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(nullptr)); @@ -597,7 +612,8 @@ TEST_P(FailoverTest, PrioritiesWithZeroWarmedHosts) { makeTestHost(info_, "tcp://127.0.0.1:81")}; failover_host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:82")}; failover_host_set_.healthy_hosts_ = failover_host_set_.hosts_; - ON_CALL(host_set_, warmedHostCount()).WillByDefault(Return(0)); + failover_host_set_.warmed_hosts_ = failover_host_set_.hosts_; + init(true); EXPECT_EQ(failover_host_set_.hosts_[0], lb_->chooseHost(nullptr)); @@ -615,6 +631,7 @@ TEST_P(RoundRobinLoadBalancerTest, NoHosts) { TEST_P(RoundRobinLoadBalancerTest, SingleHost) { hostSet().healthy_hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80")}; hostSet().hosts_ = hostSet().healthy_hosts_; + hostSet().warmed_hosts_ = hostSet().healthy_hosts_; init(false); EXPECT_EQ(hostSet().healthy_hosts_[0], lb_->chooseHost(nullptr)); } @@ -623,6 +640,7 @@ TEST_P(RoundRobinLoadBalancerTest, Normal) { hostSet().healthy_hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80"), makeTestHost(info_, "tcp://127.0.0.1:81")}; hostSet().hosts_ = hostSet().healthy_hosts_; + hostSet().warmed_hosts_ = hostSet().healthy_hosts_; init(false); EXPECT_EQ(hostSet().healthy_hosts_[0], lb_->chooseHost(nullptr)); EXPECT_EQ(hostSet().healthy_hosts_[1], lb_->chooseHost(nullptr)); @@ -638,6 +656,7 @@ TEST_P(RoundRobinLoadBalancerTest, Seed) { makeTestHost(info_, "tcp://127.0.0.1:82"), }; hostSet().hosts_ = hostSet().healthy_hosts_; + hostSet().warmed_hosts_ = hostSet().healthy_hosts_; EXPECT_CALL(random_, random()).WillRepeatedly(Return(1)); init(false); EXPECT_EQ(hostSet().healthy_hosts_[1], lb_->chooseHost(nullptr)); @@ -653,6 +672,7 @@ TEST_P(RoundRobinLoadBalancerTest, Locality) { HostsPerLocalitySharedPtr hosts_per_locality = makeHostsPerLocality({{(*hosts)[1]}, {(*hosts)[0]}, {(*hosts)[2]}}); hostSet().hosts_ = *hosts; + hostSet().warmed_hosts_ = *hosts; hostSet().healthy_hosts_ = *hosts; hostSet().healthy_hosts_per_locality_ = hosts_per_locality; init(false); @@ -689,6 +709,7 @@ TEST_P(RoundRobinLoadBalancerTest, DegradedLocality) { makeHostsPerLocality({{}, {(*hosts)[1], (*hosts)[2]}}); hostSet().hosts_ = *hosts; + hostSet().warmed_hosts_ = *hosts; hostSet().healthy_hosts_ = *healthy_hosts; hostSet().degraded_hosts_ = *degraded_hosts; hostSet().hosts_per_locality_ = hosts_per_locality; @@ -709,6 +730,7 @@ TEST_P(RoundRobinLoadBalancerTest, Weighted) { hostSet().healthy_hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80", 1), makeTestHost(info_, "tcp://127.0.0.1:81", 2)}; hostSet().hosts_ = hostSet().healthy_hosts_; + hostSet().warmed_hosts_ = hostSet().healthy_hosts_; init(false); // Initial weights respected. EXPECT_EQ(hostSet().healthy_hosts_[1], lb_->chooseHost(nullptr)); @@ -771,6 +793,7 @@ TEST_P(RoundRobinLoadBalancerTest, WeightedSeed) { hostSet().healthy_hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80", 1), makeTestHost(info_, "tcp://127.0.0.1:81", 2)}; hostSet().hosts_ = hostSet().healthy_hosts_; + hostSet().warmed_hosts_ = hostSet().healthy_hosts_; EXPECT_CALL(random_, random()).WillRepeatedly(Return(1)); init(false); // Initial weights respected. @@ -789,6 +812,7 @@ TEST_P(RoundRobinLoadBalancerTest, MaxUnhealthyPanic) { makeTestHost(info_, "tcp://127.0.0.1:80"), makeTestHost(info_, "tcp://127.0.0.1:81"), makeTestHost(info_, "tcp://127.0.0.1:82"), makeTestHost(info_, "tcp://127.0.0.1:83"), makeTestHost(info_, "tcp://127.0.0.1:84"), makeTestHost(info_, "tcp://127.0.0.1:85")}; + hostSet().warmed_hosts_ = hostSet().hosts_; init(false); EXPECT_EQ(hostSet().hosts_[0], lb_->chooseHost(nullptr)); @@ -818,6 +842,7 @@ TEST_P(RoundRobinLoadBalancerTest, HostSelectionWithFilter) { hostSet().hosts_ = *hosts; hostSet().healthy_hosts_ = *hosts; + hostSet().warmed_hosts_ = *hosts; hostSet().healthy_hosts_per_locality_ = hosts_per_locality; init(false); @@ -863,6 +888,7 @@ TEST_P(RoundRobinLoadBalancerTest, ZoneAwareSmallCluster) { hostSet().hosts_ = *hosts; hostSet().healthy_hosts_ = *hosts; + hostSet().warmed_hosts_ = *hosts; hostSet().healthy_hosts_per_locality_ = hosts_per_locality; common_config_.mutable_healthy_panic_threshold()->set_value(0); common_config_.mutable_zone_aware_lb_config()->mutable_routing_enabled()->set_value(98); @@ -911,6 +937,7 @@ TEST_P(RoundRobinLoadBalancerTest, NoZoneAwareDifferentZoneSize) { hostSet().healthy_hosts_ = *hosts; hostSet().hosts_ = *hosts; + hostSet().warmed_hosts_ = *hosts; hostSet().healthy_hosts_per_locality_ = upstream_hosts_per_locality; common_config_.mutable_healthy_panic_threshold()->set_value(100); common_config_.mutable_zone_aware_lb_config()->mutable_routing_enabled()->set_value(98); @@ -950,6 +977,7 @@ TEST_P(RoundRobinLoadBalancerTest, ZoneAwareRoutingLargeZoneSwitchOnOff) { hostSet().healthy_hosts_ = *hosts; hostSet().hosts_ = *hosts; + hostSet().warmed_hosts_ = *hosts; hostSet().healthy_hosts_per_locality_ = hosts_per_locality; init(true); updateHosts(hosts, hosts_per_locality); @@ -997,6 +1025,7 @@ TEST_P(RoundRobinLoadBalancerTest, ZoneAwareRoutingSmallZone) { hostSet().healthy_hosts_ = *upstream_hosts; hostSet().hosts_ = *upstream_hosts; + hostSet().warmed_hosts_ = *upstream_hosts; hostSet().healthy_hosts_per_locality_ = upstream_hosts_per_locality; init(true); updateHosts(local_hosts, local_hosts_per_locality); @@ -1019,6 +1048,7 @@ TEST_P(RoundRobinLoadBalancerTest, LowPrecisionForDistribution) { // upstream_hosts and local_hosts do not matter, zone aware routing is based on per zone hosts. HostVectorSharedPtr upstream_hosts(new HostVector({makeTestHost(info_, "tcp://127.0.0.1:80")})); hostSet().healthy_hosts_ = *upstream_hosts; + hostSet().warmed_hosts_ = *upstream_hosts; hostSet().hosts_ = *upstream_hosts; HostVectorSharedPtr local_hosts(new HostVector({makeTestHost(info_, "tcp://127.0.0.1:0")})); @@ -1099,6 +1129,7 @@ TEST_P(RoundRobinLoadBalancerTest, NoZoneAwareRoutingNotHealthy) { {{}, {makeTestHost(info_, "tcp://127.0.0.1:80"), makeTestHost(info_, "tcp://127.0.0.2:80")}}); hostSet().healthy_hosts_ = *hosts; + hostSet().warmed_hosts_ = *hosts; hostSet().hosts_ = *hosts; hostSet().healthy_hosts_per_locality_ = hosts_per_locality; init(true); @@ -1130,6 +1161,7 @@ TEST_P(RoundRobinLoadBalancerTest, NoZoneAwareRoutingLocalEmpty) { hostSet().healthy_hosts_ = *upstream_hosts; hostSet().hosts_ = *upstream_hosts; + hostSet().warmed_hosts_ = *upstream_hosts; hostSet().healthy_hosts_per_locality_ = upstream_hosts_per_locality; init(true); updateHosts(local_hosts, local_hosts_per_locality); @@ -1157,6 +1189,7 @@ TEST_P(RoundRobinLoadBalancerTest, NoZoneAwareRoutingNoLocalLocality) { hostSet().healthy_hosts_ = *upstream_hosts; hostSet().hosts_ = *upstream_hosts; + hostSet().warmed_hosts_ = *upstream_hosts; hostSet().healthy_hosts_per_locality_ = upstream_hosts_per_locality; init(true); updateHosts(local_hosts, local_hosts_per_locality); @@ -1180,6 +1213,7 @@ TEST_P(LeastRequestLoadBalancerTest, NoHosts) { EXPECT_EQ(nullptr, lb_.chooseHos TEST_P(LeastRequestLoadBalancerTest, SingleHost) { hostSet().healthy_hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80")}; + hostSet().warmed_hosts_ = hostSet().healthy_hosts_; hostSet().hosts_ = hostSet().healthy_hosts_; hostSet().runCallbacks({}, {}); // Trigger callbacks. The added/removed lists are not relevant. @@ -1220,6 +1254,7 @@ TEST_P(LeastRequestLoadBalancerTest, Normal) { makeTestHost(info_, "tcp://127.0.0.1:81")}; stats_.max_host_weight_.set(1UL); hostSet().hosts_ = hostSet().healthy_hosts_; + hostSet().warmed_hosts_ = hostSet().healthy_hosts_; hostSet().runCallbacks({}, {}); // Trigger callbacks. The added/removed lists are not relevant. hostSet().healthy_hosts_[0]->stats().rq_active_.set(1); @@ -1239,6 +1274,7 @@ TEST_P(LeastRequestLoadBalancerTest, PNC) { makeTestHost(info_, "tcp://127.0.0.1:82"), makeTestHost(info_, "tcp://127.0.0.1:83")}; stats_.max_host_weight_.set(1UL); hostSet().hosts_ = hostSet().healthy_hosts_; + hostSet().warmed_hosts_ = hostSet().healthy_hosts_; hostSet().runCallbacks({}, {}); // Trigger callbacks. The added/removed lists are not relevant. hostSet().healthy_hosts_[0]->stats().rq_active_.set(4); @@ -1287,6 +1323,7 @@ TEST_P(LeastRequestLoadBalancerTest, WeightImbalance) { stats_.max_host_weight_.set(2UL); hostSet().hosts_ = hostSet().healthy_hosts_; + hostSet().warmed_hosts_ = hostSet().healthy_hosts_; hostSet().runCallbacks({}, {}); // Trigger callbacks. The added/removed lists are not relevant. EXPECT_CALL(random_, random()).WillRepeatedly(Return(0)); @@ -1329,6 +1366,7 @@ TEST_P(LeastRequestLoadBalancerTest, WeightImbalanceCallbacks) { stats_.max_host_weight_.set(2UL); hostSet().hosts_ = hostSet().healthy_hosts_; + hostSet().warmed_hosts_ = hostSet().healthy_hosts_; hostSet().runCallbacks({}, {}); // Trigger callbacks. The added/removed lists are not relevant. EXPECT_CALL(random_, random()).WillRepeatedly(Return(0)); @@ -1360,6 +1398,7 @@ TEST_P(RandomLoadBalancerTest, Normal) { hostSet().healthy_hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80"), makeTestHost(info_, "tcp://127.0.0.1:81")}; hostSet().hosts_ = hostSet().healthy_hosts_; + hostSet().warmed_hosts_ = hostSet().hosts_; hostSet().runCallbacks({}, {}); // Trigger callbacks. The added/removed lists are not relevant. EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(2)); EXPECT_EQ(hostSet().healthy_hosts_[0], lb_.chooseHost(nullptr)); diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc index 350e48baf81e2..52ad44b064b5f 100644 --- a/test/common/upstream/upstream_impl_test.cc +++ b/test/common/upstream/upstream_impl_test.cc @@ -1666,7 +1666,7 @@ class TestBatchUpdateCb : public PrioritySet::BatchUpdateCb { HostSetImpl::updateHostsParams(hosts_, hosts_per_locality_, std::make_shared(*hosts_), hosts_per_locality_), - {}, hosts_added, hosts_removed, hosts_->size(), absl::nullopt); + {}, hosts_added, hosts_removed, absl::nullopt); } // Remove the host from P1. @@ -1679,7 +1679,7 @@ class TestBatchUpdateCb : public PrioritySet::BatchUpdateCb { HostSetImpl::updateHostsParams(empty_hosts, HostsPerLocalityImpl::empty(), std::make_shared(*empty_hosts), HostsPerLocalityImpl::empty()), - {}, hosts_added, hosts_removed, hosts_->size(), absl::nullopt); + {}, hosts_added, hosts_removed, absl::nullopt); } } @@ -1734,7 +1734,7 @@ TEST(PrioritySet, Extend) { HostSetImpl::updateHostsParams(hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - {}, hosts_added, hosts_removed, hosts->size(), absl::nullopt); + {}, hosts_added, hosts_removed, absl::nullopt); } EXPECT_EQ(1, priority_changes); EXPECT_EQ(1, membership_changes); @@ -2285,7 +2285,7 @@ TEST(HostsPerLocalityImpl, Filter) { class HostSetImplLocalityTest : public testing::Test { public: LocalityWeightsConstSharedPtr locality_weights_; - HostSetImpl host_set_{0, 0, kDefaultOverProvisioningFactor}; + HostSetImpl host_set_{0, kDefaultOverProvisioningFactor}; std::shared_ptr info_{new NiceMock()}; HostVector hosts_{ makeTestHost(info_, "tcp://127.0.0.1:80"), makeTestHost(info_, "tcp://127.0.0.1:81"), @@ -2306,15 +2306,13 @@ TEST_F(HostSetImplLocalityTest, AllUnhealthy) { LocalityWeightsConstSharedPtr locality_weights{new LocalityWeights{1, 1, 1}}; auto hosts = makeHostsFromHostsPerLocality(hosts_per_locality); host_set_.updateHosts(HostSetImpl::updateHostsParams(hosts, hosts_per_locality), locality_weights, - {}, {}, hosts->size(), absl::nullopt); + {}, {}, absl::nullopt); EXPECT_FALSE(host_set_.chooseHealthyLocality().has_value()); } // When a locality has endpoints that have not yet been warmed, weight calculation should ignore // these hosts. TEST_F(HostSetImplLocalityTest, NotWarmedHostsLocality) { - ON_CALL(*info_, warmHosts()).WillByDefault(Return(true)); - // We have two localities with 3 hosts in L1, 2 hosts in L2. Two of the hosts in L1 are not warmed // yet, so even though they are unhealthy we should not adjust the locality weight. HostsPerLocalitySharedPtr hosts_per_locality = @@ -2324,18 +2322,15 @@ TEST_F(HostSetImplLocalityTest, NotWarmedHostsLocality) { HostsPerLocalitySharedPtr healthy_hosts_per_locality = makeHostsPerLocality({{hosts_[0]}, {hosts_[3], hosts_[4]}}); - // Mark hosts 1 and 2 as not warmed. - hosts_[1]->healthFlagSet(Host::HealthFlag::FAILED_ACTIVE_HC); - hosts_[1]->healthFlagSet(Host::HealthFlag::PENDING_ACTIVE_HC); - hosts_[2]->healthFlagSet(Host::HealthFlag::FAILED_ACTIVE_HC); - hosts_[2]->healthFlagSet(Host::HealthFlag::PENDING_ACTIVE_HC); - host_set_.updateHosts( HostSetImpl::updateHostsParams( hosts, hosts_per_locality, makeHostsFromHostsPerLocality(healthy_hosts_per_locality), + healthy_hosts_per_locality, std::make_shared(), + HostsPerLocalityImpl::empty(), + makeHostsFromHostsPerLocality(healthy_hosts_per_locality), healthy_hosts_per_locality), - locality_weights, {}, {}, hosts->size(), absl::nullopt); + locality_weights, {}, {}, absl::nullopt); // We should RR between localities with equal weight. EXPECT_EQ(0, host_set_.chooseHealthyLocality().value()); EXPECT_EQ(1, host_set_.chooseHealthyLocality().value()); @@ -2366,7 +2361,7 @@ TEST_F(HostSetImplLocalityTest, ZeroWarmedHostsLocality) { hosts, hosts_per_locality, makeHostsFromHostsPerLocality(healthy_hosts_per_locality), healthy_hosts_per_locality), - locality_weights, {}, {}, hosts->size(), absl::nullopt); + locality_weights, {}, {}, absl::nullopt); // We should only target the second locality since there are no healthy in the first. EXPECT_EQ(1, host_set_.chooseHealthyLocality().value()); EXPECT_EQ(1, host_set_.chooseHealthyLocality().value()); @@ -2381,7 +2376,7 @@ TEST_F(HostSetImplLocalityTest, EmptyLocality) { host_set_.updateHosts(HostSetImpl::updateHostsParams( hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - locality_weights, {}, {}, hosts->size(), absl::nullopt); + locality_weights, {}, {}, absl::nullopt); // Verify that we are not RRing between localities. EXPECT_EQ(0, host_set_.chooseHealthyLocality().value()); EXPECT_EQ(0, host_set_.chooseHealthyLocality().value()); @@ -2395,7 +2390,7 @@ TEST_F(HostSetImplLocalityTest, AllZeroWeights) { host_set_.updateHosts(HostSetImpl::updateHostsParams( hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - locality_weights, {}, {}, hosts->size()); + locality_weights, {}, {}); EXPECT_FALSE(host_set_.chooseHealthyLocality().has_value()); } @@ -2408,7 +2403,7 @@ TEST_F(HostSetImplLocalityTest, Unweighted) { host_set_.updateHosts(HostSetImpl::updateHostsParams( hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - locality_weights, {}, {}, hosts->size(), absl::nullopt); + locality_weights, {}, {}, absl::nullopt); EXPECT_EQ(0, host_set_.chooseHealthyLocality().value()); EXPECT_EQ(1, host_set_.chooseHealthyLocality().value()); EXPECT_EQ(2, host_set_.chooseHealthyLocality().value()); @@ -2425,7 +2420,7 @@ TEST_F(HostSetImplLocalityTest, Weighted) { host_set_.updateHosts(HostSetImpl::updateHostsParams( hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - locality_weights, {}, {}, hosts->size(), absl::nullopt); + locality_weights, {}, {}, absl::nullopt); EXPECT_EQ(1, host_set_.chooseHealthyLocality().value()); EXPECT_EQ(0, host_set_.chooseHealthyLocality().value()); EXPECT_EQ(1, host_set_.chooseHealthyLocality().value()); @@ -2443,7 +2438,7 @@ TEST_F(HostSetImplLocalityTest, MissingWeight) { host_set_.updateHosts(HostSetImpl::updateHostsParams( hosts, hosts_per_locality, std::make_shared(*hosts), hosts_per_locality), - locality_weights, {}, {}, hosts->size(), absl::nullopt); + locality_weights, {}, {}, absl::nullopt); EXPECT_EQ(0, host_set_.chooseHealthyLocality().value()); EXPECT_EQ(2, host_set_.chooseHealthyLocality().value()); EXPECT_EQ(0, host_set_.chooseHealthyLocality().value()); @@ -2471,7 +2466,7 @@ TEST_F(HostSetImplLocalityTest, UnhealthyFailover) { hosts, hosts_per_locality, makeHostsFromHostsPerLocality(healthy_hosts_per_locality), healthy_hosts_per_locality), - locality_weights, {}, {}, hosts->size(), absl::nullopt); + locality_weights, {}, {}, absl::nullopt); }; const auto expectPicks = [this](uint32_t locality_0_picks, uint32_t locality_1_picks) { @@ -2503,7 +2498,7 @@ TEST_F(HostSetImplLocalityTest, UnhealthyFailover) { TEST(OverProvisioningFactorTest, LocalityPickChanges) { auto setUpHostSetWithOPFAndTestPicks = [](const uint32_t overprovisioning_factor, const uint32_t pick_0, const uint32_t pick_1) { - HostSetImpl host_set(0, 0, overprovisioning_factor); + HostSetImpl host_set(0, overprovisioning_factor); std::shared_ptr cluster_info{new NiceMock()}; HostVector hosts{makeTestHost(cluster_info, "tcp://127.0.0.1:80"), makeTestHost(cluster_info, "tcp://127.0.0.1:81"), @@ -2519,7 +2514,7 @@ TEST(OverProvisioningFactorTest, LocalityPickChanges) { host_set.updateHosts(HostSetImpl::updateHostsParams(std::make_shared(hosts), hosts_per_locality, healthy_hosts, healthy_hosts_per_locality), - locality_weights, {}, {}, hosts.size(), absl::nullopt); + locality_weights, {}, {}, absl::nullopt); uint32_t cnts[] = {0, 0}; for (uint32_t i = 0; i < 100; ++i) { absl::optional locality_index = host_set.chooseHealthyLocality(); diff --git a/test/mocks/upstream/mocks.cc b/test/mocks/upstream/mocks.cc index 7ec848f4f77de..90d0356331ae0 100644 --- a/test/mocks/upstream/mocks.cc +++ b/test/mocks/upstream/mocks.cc @@ -24,6 +24,7 @@ MockHostSet::MockHostSet(uint32_t priority, uint32_t overprovisioning_factor) ON_CALL(*this, hosts()).WillByDefault(ReturnRef(hosts_)); ON_CALL(*this, healthyHosts()).WillByDefault(ReturnRef(healthy_hosts_)); ON_CALL(*this, degradedHosts()).WillByDefault(ReturnRef(degraded_hosts_)); + ON_CALL(*this, warmedHosts()).WillByDefault(ReturnRef(warmed_hosts_)); ON_CALL(*this, hostsPerLocality()).WillByDefault(Invoke([this]() -> const HostsPerLocality& { return *hosts_per_locality_; })); @@ -39,7 +40,6 @@ MockHostSet::MockHostSet(uint32_t priority, uint32_t overprovisioning_factor) ON_CALL(*this, localityWeights()).WillByDefault(Invoke([this]() -> LocalityWeightsConstSharedPtr { return locality_weights_; })); - ON_CALL(*this, warmedHostCount()).WillByDefault(Invoke([this]() { return hosts_.size(); })); } MockHostSet::~MockHostSet() = default; diff --git a/test/mocks/upstream/mocks.h b/test/mocks/upstream/mocks.h index 5795afa21f5ad..941b78ca62208 100644 --- a/test/mocks/upstream/mocks.h +++ b/test/mocks/upstream/mocks.h @@ -52,6 +52,7 @@ class MockHostSet : public HostSet { MOCK_CONST_METHOD0(hosts, const HostVector&()); MOCK_CONST_METHOD0(healthyHosts, const HostVector&()); MOCK_CONST_METHOD0(degradedHosts, const HostVector&()); + MOCK_CONST_METHOD0(warmedHosts, const HostVector&()); MOCK_CONST_METHOD0(hostsPerLocality, const HostsPerLocality&()); MOCK_CONST_METHOD0(healthyHostsPerLocality, const HostsPerLocality&()); MOCK_CONST_METHOD0(degradedHostsPerLocality, const HostsPerLocality&()); @@ -64,11 +65,11 @@ class MockHostSet : public HostSet { void setOverprovisioningFactor(const uint32_t overprovisioning_factor) { overprovisioning_factor_ = overprovisioning_factor; } - MOCK_CONST_METHOD0(warmedHostCount, uint32_t()); HostVector hosts_; HostVector healthy_hosts_; HostVector degraded_hosts_; + HostVector warmed_hosts_; HostsPerLocalitySharedPtr hosts_per_locality_{new HostsPerLocalityImpl()}; HostsPerLocalitySharedPtr healthy_hosts_per_locality_{new HostsPerLocalityImpl()}; HostsPerLocalitySharedPtr degraded_hosts_per_locality_{new HostsPerLocalityImpl()}; @@ -93,17 +94,11 @@ class MockPrioritySet : public PrioritySet { MOCK_CONST_METHOD1(addPriorityUpdateCb, Common::CallbackHandle*(PriorityUpdateCb callback)); MOCK_CONST_METHOD0(hostSetsPerPriority, const std::vector&()); MOCK_METHOD0(hostSetsPerPriority, std::vector&()); - MOCK_METHOD7(updateHosts, - void(uint32_t priority, UpdateHostsParams&& update_hosts_params, - LocalityWeightsConstSharedPtr locality_weights, const HostVector& hosts_added, - const HostVector& hosts_removed, uint32_t warmed_host_count, - absl::optional overprovisioning_factor)); + MOCK_METHOD6(updateHosts, void(uint32_t priority, UpdateHostsParams&& update_hosts_params, + LocalityWeightsConstSharedPtr locality_weights, + const HostVector& hosts_added, const HostVector& hosts_removed, + absl::optional overprovisioning_factor)); MOCK_METHOD1(batchHostUpdate, void(BatchUpdateCb&)); - MOCK_CONST_METHOD7(updateHosts, void(uint32_t priority, UpdateHostsParams&& update_hosts_params, - LocalityWeightsConstSharedPtr locality_weights, - const HostVector& hosts_added, - const HostVector& hosts_removed, uint32_t warmed_host_count, - absl::optional overprovisioning_factor)); MockHostSet* getMockHostSet(uint32_t priority) { getHostSet(priority); // Ensure the host set exists. From 1cb40f5dfb942d547d9710817a56f53763e828b2 Mon Sep 17 00:00:00 2001 From: Snow Pettersen Date: Fri, 3 May 2019 18:53:23 -0400 Subject: [PATCH 11/28] fix more build failures Signed-off-by: Snow Pettersen --- source/common/upstream/subset_lb.cc | 2 +- .../upstream/load_balancer_benchmark.cc | 2 +- .../upstream/load_balancer_simulation_test.cc | 4 ++-- .../upstream/original_dst_cluster_test.cc | 2 +- test/common/upstream/subset_lb_test.cc | 22 +++++++++++++------ .../clusters/custom_static_cluster.cc | 2 +- 6 files changed, 21 insertions(+), 13 deletions(-) diff --git a/source/common/upstream/subset_lb.cc b/source/common/upstream/subset_lb.cc index 9460ecb7885fb..b5c742c836c3f 100644 --- a/source/common/upstream/subset_lb.cc +++ b/source/common/upstream/subset_lb.cc @@ -555,7 +555,7 @@ void SubsetLoadBalancer::HostSubsetImpl::update(const HostVector& hosts_added, warmed_hosts->get().reserve(original_host_set_.degradedHosts().size()); for (const auto& host : original_host_set_.warmedHosts()) { if (cached_predicate(*host)) { - degraded_hosts->get().emplace_back(host); + warmed_hosts->get().emplace_back(host); } } diff --git a/test/common/upstream/load_balancer_benchmark.cc b/test/common/upstream/load_balancer_benchmark.cc index 6985875cf5802..329e4eef53c4b 100644 --- a/test/common/upstream/load_balancer_benchmark.cc +++ b/test/common/upstream/load_balancer_benchmark.cc @@ -33,7 +33,7 @@ class BaseTester { HostSetImpl::updateHostsParams(updated_hosts, nullptr, std::make_shared(*updated_hosts), nullptr), - {}, hosts, {}, hosts.size(), absl::nullopt); + {}, hosts, {}, absl::nullopt); } PrioritySetImpl priority_set_; diff --git a/test/common/upstream/load_balancer_simulation_test.cc b/test/common/upstream/load_balancer_simulation_test.cc index a5fc2c406e3fa..76f820a161bdd 100644 --- a/test/common/upstream/load_balancer_simulation_test.cc +++ b/test/common/upstream/load_balancer_simulation_test.cc @@ -59,7 +59,7 @@ TEST(DISABLED_LeastRequestLoadBalancerWeightTest, Weight) { HostSetImpl::updateHostsParams(updated_hosts, updated_locality_hosts, std::make_shared(*updated_hosts), updated_locality_hosts), - {}, hosts, {}, updated_hosts->size(), absl::nullopt); + {}, hosts, {}, absl::nullopt); Stats::IsolatedStoreImpl stats_store; ClusterStats stats{ClusterInfoImpl::generateStats(stats_store)}; @@ -165,7 +165,7 @@ class DISABLED_SimulationTest : public testing::Test { HostSetImpl::updateHostsParams( originating_hosts, per_zone_local_shared, std::make_shared(*originating_hosts), per_zone_local_shared), - {}, empty_vector_, empty_vector_, originating_hosts->size(), absl::nullopt); + {}, empty_vector_, empty_vector_, absl::nullopt); HostConstSharedPtr selected = lb.chooseHost(nullptr); hits[selected->address()->asString()]++; diff --git a/test/common/upstream/original_dst_cluster_test.cc b/test/common/upstream/original_dst_cluster_test.cc index 3a121bc8e1e9c..23314f2f25fb3 100644 --- a/test/common/upstream/original_dst_cluster_test.cc +++ b/test/common/upstream/original_dst_cluster_test.cc @@ -468,7 +468,7 @@ TEST_F(OriginalDstClusterTest, MultipleClusters) { second.updateHosts(0, HostSetImpl::updateHostsParams(new_hosts, empty_hosts_per_locality, healthy_hosts, empty_hosts_per_locality), - {}, added, removed, new_hosts->size(), absl::nullopt); + {}, added, removed, absl::nullopt); }); EXPECT_CALL(membership_updated_, ready()); diff --git a/test/common/upstream/subset_lb_test.cc b/test/common/upstream/subset_lb_test.cc index 8bbd6520ad42d..68ea7ae94cd08 100644 --- a/test/common/upstream/subset_lb_test.cc +++ b/test/common/upstream/subset_lb_test.cc @@ -132,6 +132,7 @@ class SubsetLoadBalancerTest : public testing::TestWithParam { host_set.hosts_ = hosts; host_set.hosts_per_locality_ = makeHostsPerLocality({hosts}); host_set.healthy_hosts_ = host_set.hosts_; + host_set.warmed_hosts_ = host_set.hosts_; host_set.healthy_hosts_per_locality_ = host_set.hosts_per_locality_; } @@ -157,6 +158,7 @@ class SubsetLoadBalancerTest : public testing::TestWithParam { host_set.hosts_per_locality_ = makeHostsPerLocality({first_locality, second_locality}); host_set.warmed_hosts_per_locality_ = makeHostsPerLocality({first_locality, second_locality}); host_set.healthy_hosts_ = host_set.hosts_; + host_set.warmed_hosts_ = host_set.hosts_; host_set.healthy_hosts_per_locality_ = host_set.hosts_per_locality_; host_set.locality_weights_ = std::make_shared(locality_weights); } @@ -201,6 +203,8 @@ class SubsetLoadBalancerTest : public testing::TestWithParam { host_set_.healthy_hosts_ = host_set_.hosts_; host_set_.healthy_hosts_per_locality_ = host_set_.hosts_per_locality_; + host_set_.warmed_hosts_ = host_set_.hosts_; + host_set_.warmed_hosts_per_locality_ = host_set_.hosts_per_locality_; local_hosts_.reset(new HostVector()); std::vector local_hosts_per_locality_vector; @@ -217,10 +221,12 @@ class SubsetLoadBalancerTest : public testing::TestWithParam { local_priority_set_.updateHosts( 0, - HostSetImpl::updateHostsParams(local_hosts_, local_hosts_per_locality_, - std::make_shared(*local_hosts_), - local_hosts_per_locality_), - {}, {}, {}, local_hosts_->size(), absl::nullopt); + HostSetImpl::updateHostsParams( + local_hosts_, local_hosts_per_locality_, + std::make_shared(*local_hosts_), local_hosts_per_locality_, + std::make_shared(), HostsPerLocalityImpl::empty(), + std::make_shared(*local_hosts_), local_hosts_per_locality_), + {}, {}, {}, absl::nullopt); lb_.reset(new SubsetLoadBalancer( lb_type_, priority_set_, &local_priority_set_, stats_, stats_store_, runtime_, random_, @@ -269,6 +275,7 @@ class SubsetLoadBalancerTest : public testing::TestWithParam { } host_set.hosts_per_locality_ = makeHostsPerLocality(std::move(locality_hosts_copy)); host_set.healthy_hosts_per_locality_ = host_set.hosts_per_locality_; + host_set.warmed_hosts_per_locality_ = host_set.hosts_per_locality_; } if (GetParam() == REMOVES_FIRST && !remove.empty()) { @@ -278,6 +285,7 @@ class SubsetLoadBalancerTest : public testing::TestWithParam { for (const auto& host : add) { host_set.hosts_.emplace_back(host); host_set.healthy_hosts_ = host_set.hosts_; + host_set.warmed_hosts_ = host_set.hosts_; if (add_in_locality) { std::vector locality_hosts_copy = host_set.hosts_per_locality_->get(); @@ -319,7 +327,7 @@ class SubsetLoadBalancerTest : public testing::TestWithParam { HostSetImpl::updateHostsParams(local_hosts_, local_hosts_per_locality_, std::make_shared(*local_hosts_), local_hosts_per_locality_), - {}, {}, remove, local_hosts_->size(), absl::nullopt); + {}, {}, remove, absl::nullopt); } for (const auto& host : add) { @@ -336,7 +344,7 @@ class SubsetLoadBalancerTest : public testing::TestWithParam { HostSetImpl::updateHostsParams(local_hosts_, local_hosts_per_locality_, std::make_shared(*local_hosts_), local_hosts_per_locality_), - {}, add, {}, local_hosts_->size(), absl::nullopt); + {}, add, {}, absl::nullopt); } } else if (!add.empty() || !remove.empty()) { local_priority_set_.updateHosts( @@ -344,7 +352,7 @@ class SubsetLoadBalancerTest : public testing::TestWithParam { HostSetImpl::updateHostsParams(local_hosts_, local_hosts_per_locality_, std::make_shared(*local_hosts_), local_hosts_per_locality_), - {}, add, remove, local_hosts_->size(), absl::nullopt); + {}, add, remove, absl::nullopt); } } diff --git a/test/integration/clusters/custom_static_cluster.cc b/test/integration/clusters/custom_static_cluster.cc index 7003315801875..d4dae7cce4da9 100644 --- a/test/integration/clusters/custom_static_cluster.cc +++ b/test/integration/clusters/custom_static_cluster.cc @@ -12,7 +12,7 @@ void CustomStaticCluster::startPreInit() { this->priority_set_.updateHosts( priority_, Upstream::HostSetImpl::partitionHosts(hosts_ptr, Upstream::HostsPerLocalityImpl::empty()), {}, - hosts, {}, hosts.size(), absl::nullopt); + hosts, {}, absl::nullopt); onPreInitComplete(); } From a4a4f1000b6eced8c2c4a9afb5e2677b24d2ef50 Mon Sep 17 00:00:00 2001 From: Snow Pettersen Date: Sat, 4 May 2019 12:09:39 -0400 Subject: [PATCH 12/28] fix filter bug Signed-off-by: Snow Pettersen --- source/common/upstream/upstream_impl.cc | 2 +- source/common/upstream/upstream_impl.h | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index 497740a742991..f65b13888d95a 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -302,7 +302,7 @@ std::vector HostsPerLocalityImpl::filter( } for (size_t i = 0; i < predicates.size(); ++i) { - mutable_clones[i]->hosts_per_locality_.push_back(std::move(current_locality_hosts[0])); + mutable_clones[i]->hosts_per_locality_.push_back(std::move(current_locality_hosts[i])); } } diff --git a/source/common/upstream/upstream_impl.h b/source/common/upstream/upstream_impl.h index f2b35625a3c12..0441a71117ed8 100644 --- a/source/common/upstream/upstream_impl.h +++ b/source/common/upstream/upstream_impl.h @@ -225,7 +225,11 @@ class HostImpl : public HostDescriptionImpl, bool used() const override { return used_; } void used(bool new_used) override { used_ = new_used; } bool warmed() const override { - return !cluster_->warmHosts() || !healthFlagGet(HealthFlag::PENDING_ACTIVE_HC); + if (cluster_->warmHosts()) { + return !healthFlagGet(HealthFlag::PENDING_ACTIVE_HC); + } + + return true; } protected: From ed4eaa7daf6694408e138f86149a7970381031be Mon Sep 17 00:00:00 2001 From: Snow Pettersen Date: Sat, 4 May 2019 12:58:22 -0400 Subject: [PATCH 13/28] fix test failures Signed-off-by: Snow Pettersen --- source/common/upstream/upstream_impl.h | 2 +- test/common/upstream/ring_hash_lb_test.cc | 17 +++++++++++++++++ .../priority/previous_priorities/config_test.cc | 1 + 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/source/common/upstream/upstream_impl.h b/source/common/upstream/upstream_impl.h index 0441a71117ed8..318138cf58a5d 100644 --- a/source/common/upstream/upstream_impl.h +++ b/source/common/upstream/upstream_impl.h @@ -225,7 +225,7 @@ class HostImpl : public HostDescriptionImpl, bool used() const override { return used_; } void used(bool new_used) override { used_ = new_used; } bool warmed() const override { - if (cluster_->warmHosts()) { + if (cluster_ != nullptr && cluster_->warmHosts()) { return !healthFlagGet(HealthFlag::PENDING_ACTIVE_HC); } diff --git a/test/common/upstream/ring_hash_lb_test.cc b/test/common/upstream/ring_hash_lb_test.cc index 35d7942ac60f2..71d34add835ba 100644 --- a/test/common/upstream/ring_hash_lb_test.cc +++ b/test/common/upstream/ring_hash_lb_test.cc @@ -90,6 +90,7 @@ TEST_P(RingHashLoadBalancerTest, Basic) { makeTestHost(info_, "tcp://127.0.0.1:92"), makeTestHost(info_, "tcp://127.0.0.1:93"), makeTestHost(info_, "tcp://127.0.0.1:94"), makeTestHost(info_, "tcp://127.0.0.1:95")}; hostSet().healthy_hosts_ = hostSet().hosts_; + hostSet().warmed_hosts_ = hostSet().hosts_; hostSet().runCallbacks({}, {}); config_ = envoy::api::v2::Cluster::RingHashLbConfig(); @@ -161,8 +162,10 @@ TEST_P(RingHashLoadBalancerTest, Basic) { // Ensure if all the hosts with priority 0 unhealthy, the next priority hosts are used. TEST_P(RingHashFailoverTest, BasicFailover) { host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80")}; + host_set_.warmed_hosts_ = host_set_.hosts_; failover_host_set_.healthy_hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:82")}; failover_host_set_.hosts_ = failover_host_set_.healthy_hosts_; + failover_host_set_.warmed_hosts_ = failover_host_set_.hosts_; config_ = envoy::api::v2::Cluster::RingHashLbConfig(); config_.value().mutable_minimum_ring_size()->set_value(12); @@ -190,6 +193,7 @@ TEST_P(RingHashFailoverTest, BasicFailover) { host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80"), makeTestHost(info_, "tcp://127.0.0.1:81")}; host_set_.healthy_hosts_ = {host_set_.hosts_[0]}; + host_set_.warmed_hosts_ = host_set_.hosts_; host_set_.runCallbacks({}, {}); lb = lb_->factory()->create(); EXPECT_CALL(random_, random()).WillOnce(Return(69)); @@ -205,6 +209,7 @@ TEST_P(RingHashLoadBalancerTest, BasicWithMurmur2) { makeTestHost(info_, "tcp://127.0.0.1:82"), makeTestHost(info_, "tcp://127.0.0.1:83"), makeTestHost(info_, "tcp://127.0.0.1:84"), makeTestHost(info_, "tcp://127.0.0.1:85")}; hostSet().healthy_hosts_ = hostSet().hosts_; + hostSet().warmed_hosts_ = hostSet().hosts_; hostSet().runCallbacks({}, {}); config_ = envoy::api::v2::Cluster::RingHashLbConfig(); @@ -258,6 +263,7 @@ TEST_P(RingHashLoadBalancerTest, UnevenHosts) { hostSet().hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80"), makeTestHost(info_, "tcp://127.0.0.1:81")}; hostSet().healthy_hosts_ = hostSet().hosts_; + hostSet().warmed_hosts_ = hostSet().hosts_; hostSet().runCallbacks({}, {}); config_ = envoy::api::v2::Cluster::RingHashLbConfig(); @@ -284,6 +290,7 @@ TEST_P(RingHashLoadBalancerTest, UnevenHosts) { hostSet().hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:81"), makeTestHost(info_, "tcp://127.0.0.1:82")}; hostSet().healthy_hosts_ = hostSet().hosts_; + hostSet().warmed_hosts_ = hostSet().hosts_; hostSet().runCallbacks({}, {}); // hash ring: @@ -308,6 +315,7 @@ TEST_P(RingHashLoadBalancerTest, HostWeightedTinyRing) { makeTestHost(info_, "tcp://127.0.0.1:91", 2), makeTestHost(info_, "tcp://127.0.0.1:92", 3)}; hostSet().healthy_hosts_ = hostSet().hosts_; + hostSet().warmed_hosts_ = hostSet().hosts_; hostSet().runCallbacks({}, {}); // enforce a ring size of exactly six entries @@ -337,6 +345,7 @@ TEST_P(RingHashLoadBalancerTest, HostWeightedLargeRing) { makeTestHost(info_, "tcp://127.0.0.1:91", 2), makeTestHost(info_, "tcp://127.0.0.1:92", 3)}; hostSet().healthy_hosts_ = hostSet().hosts_; + hostSet().warmed_hosts_ = hostSet().hosts_; hostSet().runCallbacks({}, {}); config_ = envoy::api::v2::Cluster::RingHashLbConfig(); @@ -365,6 +374,7 @@ TEST_P(RingHashLoadBalancerTest, ZeroLocalityWeights) { hostSet().hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:90"), makeTestHost(info_, "tcp://127.0.0.1:91")}; hostSet().healthy_hosts_ = hostSet().hosts_; + hostSet().warmed_hosts_ = hostSet().hosts_; hostSet().hosts_per_locality_ = makeHostsPerLocality({{hostSet().hosts_[0]}, {hostSet().hosts_[1]}}); hostSet().healthy_hosts_per_locality_ = hostSet().hosts_per_locality_; @@ -382,6 +392,7 @@ TEST_P(RingHashLoadBalancerTest, LocalityWeightedTinyRing) { makeTestHost(info_, "tcp://127.0.0.1:90"), makeTestHost(info_, "tcp://127.0.0.1:91"), makeTestHost(info_, "tcp://127.0.0.1:92"), makeTestHost(info_, "tcp://127.0.0.1:93")}; hostSet().healthy_hosts_ = hostSet().hosts_; + hostSet().warmed_hosts_ = hostSet().hosts_; hostSet().hosts_per_locality_ = makeHostsPerLocality( {{hostSet().hosts_[0]}, {hostSet().hosts_[1]}, {hostSet().hosts_[2]}, {hostSet().hosts_[3]}}); hostSet().healthy_hosts_per_locality_ = hostSet().hosts_per_locality_; @@ -416,6 +427,7 @@ TEST_P(RingHashLoadBalancerTest, LocalityWeightedLargeRing) { makeTestHost(info_, "tcp://127.0.0.1:90"), makeTestHost(info_, "tcp://127.0.0.1:91"), makeTestHost(info_, "tcp://127.0.0.1:92"), makeTestHost(info_, "tcp://127.0.0.1:93")}; hostSet().healthy_hosts_ = hostSet().hosts_; + hostSet().warmed_hosts_ = hostSet().hosts_; hostSet().hosts_per_locality_ = makeHostsPerLocality( {{hostSet().hosts_[0]}, {hostSet().hosts_[1]}, {hostSet().hosts_[2]}, {hostSet().hosts_[3]}}); hostSet().healthy_hosts_per_locality_ = hostSet().hosts_per_locality_; @@ -452,6 +464,7 @@ TEST_P(RingHashLoadBalancerTest, HostAndLocalityWeightedTinyRing) { makeTestHost(info_, "tcp://127.0.0.1:90", 1), makeTestHost(info_, "tcp://127.0.0.1:91", 2), makeTestHost(info_, "tcp://127.0.0.1:92", 1), makeTestHost(info_, "tcp://127.0.0.1:93", 2)}; hostSet().healthy_hosts_ = hostSet().hosts_; + hostSet().warmed_hosts_ = hostSet().hosts_; hostSet().hosts_per_locality_ = makeHostsPerLocality( {{hostSet().hosts_[0], hostSet().hosts_[1]}, {hostSet().hosts_[2], hostSet().hosts_[3]}}); hostSet().healthy_hosts_per_locality_ = hostSet().hosts_per_locality_; @@ -489,6 +502,7 @@ TEST_P(RingHashLoadBalancerTest, HostAndLocalityWeightedLargeRing) { makeTestHost(info_, "tcp://127.0.0.1:90", 1), makeTestHost(info_, "tcp://127.0.0.1:91", 2), makeTestHost(info_, "tcp://127.0.0.1:92", 1), makeTestHost(info_, "tcp://127.0.0.1:93", 2)}; hostSet().healthy_hosts_ = hostSet().hosts_; + hostSet().warmed_hosts_ = hostSet().hosts_; hostSet().hosts_per_locality_ = makeHostsPerLocality( {{hostSet().hosts_[0], hostSet().hosts_[1]}, {hostSet().hosts_[2], hostSet().hosts_[3]}}); hostSet().healthy_hosts_per_locality_ = hostSet().hosts_per_locality_; @@ -524,6 +538,7 @@ TEST_P(RingHashLoadBalancerTest, SmallFractionalScale) { makeTestHost(info_, "tcp://127.0.0.1:90"), makeTestHost(info_, "tcp://127.0.0.1:91"), makeTestHost(info_, "tcp://127.0.0.1:92"), makeTestHost(info_, "tcp://127.0.0.1:93")}; hostSet().healthy_hosts_ = hostSet().hosts_; + hostSet().warmed_hosts_ = hostSet().hosts_; hostSet().runCallbacks({}, {}); config_ = envoy::api::v2::Cluster::RingHashLbConfig(); @@ -564,6 +579,7 @@ TEST_P(RingHashLoadBalancerTest, LargeFractionalScale) { hostSet().hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:90"), makeTestHost(info_, "tcp://127.0.0.1:91")}; hostSet().healthy_hosts_ = hostSet().hosts_; + hostSet().warmed_hosts_ = hostSet().hosts_; hostSet().runCallbacks({}, {}); config_ = envoy::api::v2::Cluster::RingHashLbConfig(); @@ -598,6 +614,7 @@ TEST_P(RingHashLoadBalancerTest, LopsidedWeightSmallScale) { (i == 0 ? heavy_but_sparse : light_but_dense).push_back(host); } hostSet().healthy_hosts_ = hostSet().hosts_; + hostSet().warmed_hosts_ = hostSet().hosts_; hostSet().hosts_per_locality_ = makeHostsPerLocality({heavy_but_sparse, light_but_dense}); hostSet().healthy_hosts_per_locality_ = hostSet().hosts_per_locality_; hostSet().locality_weights_ = makeLocalityWeights({127, 1}); diff --git a/test/extensions/retry/priority/previous_priorities/config_test.cc b/test/extensions/retry/priority/previous_priorities/config_test.cc index 9b2df93460202..f6f77e8fded7d 100644 --- a/test/extensions/retry/priority/previous_priorities/config_test.cc +++ b/test/extensions/retry/priority/previous_priorities/config_test.cc @@ -42,6 +42,7 @@ class RetryPriorityTest : public testing::Test { ASSERT(count >= healthy_count + degraded_count); host_set->hosts_.resize(count); + host_set->warmed_hosts_.resize(count); host_set->healthy_hosts_.resize(healthy_count); host_set->degraded_hosts_.resize(degraded_count); host_set->runCallbacks({}, {}); From d881b2772b3277378dfc11bdafb924ec407635fb Mon Sep 17 00:00:00 2001 From: Snow Pettersen Date: Sun, 5 May 2019 12:28:43 -0400 Subject: [PATCH 14/28] track excluded hosts instead of warmed hosts to reduce memory usage Signed-off-by: Snow Pettersen --- include/envoy/upstream/types.h | 4 +- include/envoy/upstream/upstream.h | 17 +++--- source/common/upstream/load_balancer_impl.cc | 26 ++++---- source/common/upstream/subset_lb.cc | 16 ++--- source/common/upstream/upstream_impl.cc | 61 ++++++++++--------- source/common/upstream/upstream_impl.h | 27 ++++---- .../upstream/load_balancer_impl_test.cc | 40 ------------ test/common/upstream/ring_hash_lb_test.cc | 17 ------ test/common/upstream/subset_lb_test.cc | 9 +-- test/common/upstream/upstream_impl_test.cc | 35 ++--------- .../previous_priorities/config_test.cc | 1 - test/mocks/upstream/mocks.cc | 6 +- test/mocks/upstream/mocks.h | 8 +-- 13 files changed, 91 insertions(+), 176 deletions(-) diff --git a/include/envoy/upstream/types.h b/include/envoy/upstream/types.h index cde900e24b551..202a4d136a17c 100644 --- a/include/envoy/upstream/types.h +++ b/include/envoy/upstream/types.h @@ -55,8 +55,8 @@ struct HealthyAvailability : PriorityAvailability { struct Healthy {}; // Phantom type indicating that the type is related to degraded hosts. struct Degraded {}; -// Phantom type indicating that the type is related to warmed hosts. -struct Warmed {}; +// Phantom type indicating that the type is related to excluded hosts. +struct Excluded {}; } // namespace Upstream } // namespace Envoy diff --git a/include/envoy/upstream/upstream.h b/include/envoy/upstream/upstream.h index 8385252f3eb49..ea70a3ada8978 100644 --- a/include/envoy/upstream/upstream.h +++ b/include/envoy/upstream/upstream.h @@ -201,14 +201,14 @@ typedef std::shared_ptr HostConstSharedPtr; typedef std::vector HostVector; typedef Phantom HealthyHostVector; typedef Phantom DegradedHostVector; -typedef Phantom WarmedHostVector; +typedef Phantom ExcludedHostVector; typedef std::unordered_map HostMap; typedef std::shared_ptr HostVectorSharedPtr; typedef std::shared_ptr HostVectorConstSharedPtr; typedef std::shared_ptr HealthyHostVectorConstSharedPtr; typedef std::shared_ptr DegradedHostVectorConstSharedPtr; -typedef std::shared_ptr WarmedHostVectorConstSharedPtr; +typedef std::shared_ptr ExcludedHostVectorConstSharedPtr; typedef std::unique_ptr HostListPtr; typedef std::unordered_map @@ -292,9 +292,10 @@ class HostSet { virtual const HostVector& degradedHosts() const PURE; /* - * @return all warmed hosts contained in the set at the current time. + * @return all excluded hosts contained in the set at the current time. Excluded hosts should be + * ignored when computing load balancing weights, but may overlap with hosts in hosts(). * */ - virtual const HostVector& warmedHosts() const PURE; + virtual const HostVector& excludedHosts() const PURE; /** * @return hosts per locality. @@ -312,9 +313,9 @@ class HostSet { virtual const HostsPerLocality& degradedHostsPerLocality() const PURE; /** - * @return same as hostsPerLocality but only contains warmed hosts. + * @return same as hostsPerLocality but only contains excluded hosts. */ - virtual const HostsPerLocality& warmedHostsPerLocality() const PURE; + virtual const HostsPerLocality& excludedHostsPerLocality() const PURE; /** * @return weights for each locality in the host set. @@ -394,11 +395,11 @@ class PrioritySet { HostVectorConstSharedPtr hosts; HealthyHostVectorConstSharedPtr healthy_hosts; DegradedHostVectorConstSharedPtr degraded_hosts; - WarmedHostVectorConstSharedPtr warmed_hosts; + ExcludedHostVectorConstSharedPtr excluded_hosts; HostsPerLocalityConstSharedPtr hosts_per_locality; HostsPerLocalityConstSharedPtr healthy_hosts_per_locality; HostsPerLocalityConstSharedPtr degraded_hosts_per_locality; - HostsPerLocalityConstSharedPtr warmed_hosts_per_locality; + HostsPerLocalityConstSharedPtr excluded_hosts_per_locality; }; /** diff --git a/source/common/upstream/load_balancer_impl.cc b/source/common/upstream/load_balancer_impl.cc index ac5c6526283ca..9c89f994dcdc6 100644 --- a/source/common/upstream/load_balancer_impl.cc +++ b/source/common/upstream/load_balancer_impl.cc @@ -136,20 +136,20 @@ void LoadBalancerBase::recalculatePerPriorityState(uint32_t priority, // by the overprovisioning factor. HostSet& host_set = *priority_set.hostSetsPerPriority()[priority]; per_priority_health.get()[priority] = 0; - if (!host_set.warmedHosts().empty()) { + const auto host_count = host_set.hosts().size() - host_set.excludedHosts().size(); + + if (host_count > 0) { // Each priority level's health is ratio of healthy hosts to total number of hosts in a priority // multiplied by overprovisioning factor of 1.4 and capped at 100%. It means that if all // hosts are healthy that priority's health is 100%*1.4=140% and is capped at 100% which results // in 100%. If 80% of hosts are healthy, that priority's health is still 100% (80%*1.4=112% and // capped at 100%). - per_priority_health.get()[priority] = - std::min(100, (host_set.overprovisioningFactor() * - host_set.healthyHosts().size() / host_set.warmedHosts().size())); + per_priority_health.get()[priority] = std::min( + 100, (host_set.overprovisioningFactor() * host_set.healthyHosts().size() / host_count)); // We perform the same computation for degraded hosts. - per_priority_degraded.get()[priority] = - std::min(100, (host_set.overprovisioningFactor() * - host_set.degradedHosts().size() / host_set.warmedHosts().size())); + per_priority_degraded.get()[priority] = std::min( + 100, (host_set.overprovisioningFactor() * host_set.degradedHosts().size() / host_count)); } // Now that we've updated health for the changed priority level, we need to calculate percentage @@ -442,14 +442,12 @@ HostConstSharedPtr LoadBalancerBase::chooseHost(LoadBalancerContext* context) { bool LoadBalancerBase::isGlobalPanic(const HostSet& host_set) { uint64_t global_panic_threshold = std::min( 100, runtime_.snapshot().getInteger(RuntimePanicThreshold, default_healthy_panic_percent_)); - double healthy_percent = host_set.warmedHosts().empty() ? 0 - : 100.0 * host_set.healthyHosts().size() / - host_set.warmedHosts().size(); + const auto host_count = host_set.hosts().size() - host_set.excludedHosts().size(); + double healthy_percent = + host_count == 0 ? 0.0 : 100.0 * host_set.healthyHosts().size() / host_count; double degraded_percent = - host_set.warmedHosts().empty() - ? 0 - : 100.0 * host_set.degradedHosts().size() / host_set.warmedHosts().size(); + host_count == 0 ? 0.0 : 100.0 * host_set.degradedHosts().size() / host_count; // If the % of healthy hosts in the cluster is less than our panic threshold, we use all hosts. if ((healthy_percent + degraded_percent) < global_panic_threshold) { return true; @@ -465,6 +463,8 @@ void ZoneAwareLoadBalancerBase::calculateLocalityPercentage( total_hosts += locality_hosts.size(); } + // TODO(snowp): Should we ignore excluded hosts here too? + size_t i = 0; for (const auto& locality_hosts : hosts_per_locality.get()) { ret[i++] = total_hosts > 0 ? 10000ULL * locality_hosts.size() / total_hosts : 0; diff --git a/source/common/upstream/subset_lb.cc b/source/common/upstream/subset_lb.cc index b5c742c836c3f..56e048d265fe7 100644 --- a/source/common/upstream/subset_lb.cc +++ b/source/common/upstream/subset_lb.cc @@ -551,11 +551,11 @@ void SubsetLoadBalancer::HostSubsetImpl::update(const HostVector& hosts_added, } } - auto warmed_hosts = std::make_shared(); - warmed_hosts->get().reserve(original_host_set_.degradedHosts().size()); - for (const auto& host : original_host_set_.warmedHosts()) { + auto excluded_hosts = std::make_shared(); + excluded_hosts->get().reserve(original_host_set_.excludedHosts().size()); + for (const auto& host : original_host_set_.excludedHosts()) { if (cached_predicate(*host)) { - warmed_hosts->get().emplace_back(host); + excluded_hosts->get().emplace_back(host); } } @@ -576,8 +576,8 @@ void SubsetLoadBalancer::HostSubsetImpl::update(const HostVector& hosts_added, original_host_set_.healthyHostsPerLocality().filter({cached_predicate})[0]; HostsPerLocalityConstSharedPtr degraded_hosts_per_locality = original_host_set_.degradedHostsPerLocality().filter({cached_predicate})[0]; - auto warmed_hosts_per_locality = - original_host_set_.warmedHostsPerLocality().filter({cached_predicate})[0]; + auto excluded_hosts_per_locality = + original_host_set_.excludedHostsPerLocality().filter({cached_predicate})[0]; // We can use the cached predicate here, since we trust that the hosts in hosts_added were also // present in the list of all hosts. @@ -599,8 +599,8 @@ void SubsetLoadBalancer::HostSubsetImpl::update(const HostVector& hosts_added, HostSetImpl::updateHosts(HostSetImpl::updateHostsParams( hosts, hosts_per_locality, healthy_hosts, healthy_hosts_per_locality, - degraded_hosts, degraded_hosts_per_locality, warmed_hosts, - warmed_hosts_per_locality), + degraded_hosts, degraded_hosts_per_locality, excluded_hosts, + excluded_hosts_per_locality), determineLocalityWeights(*hosts_per_locality), filtered_added, filtered_removed, absl::nullopt); } diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index f65b13888d95a..d309eb8e71e0e 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -320,19 +320,20 @@ void HostSetImpl::updateHosts(PrioritySet::UpdateHostsParams&& update_hosts_para hosts_ = std::move(update_hosts_params.hosts); healthy_hosts_ = std::move(update_hosts_params.healthy_hosts); degraded_hosts_ = std::move(update_hosts_params.degraded_hosts); - warmed_hosts_ = std::move(update_hosts_params.warmed_hosts); + excluded_hosts_ = std::move(update_hosts_params.excluded_hosts); hosts_per_locality_ = std::move(update_hosts_params.hosts_per_locality); healthy_hosts_per_locality_ = std::move(update_hosts_params.healthy_hosts_per_locality); degraded_hosts_per_locality_ = std::move(update_hosts_params.degraded_hosts_per_locality); - warmed_hosts_per_locality_ = std::move(update_hosts_params.warmed_hosts_per_locality); + excluded_hosts_per_locality_ = std::move(update_hosts_params.excluded_hosts_per_locality); locality_weights_ = std::move(locality_weights); rebuildLocalityScheduler(healthy_locality_scheduler_, healthy_locality_entries_, *healthy_hosts_per_locality_, healthy_hosts_->get(), hosts_per_locality_, - warmed_hosts_per_locality_, locality_weights_, overprovisioning_factor_); + excluded_hosts_per_locality_, locality_weights_, + overprovisioning_factor_); rebuildLocalityScheduler(degraded_locality_scheduler_, degraded_locality_entries_, *degraded_hosts_per_locality_, degraded_hosts_->get(), - hosts_per_locality_, warmed_hosts_per_locality_, locality_weights_, + hosts_per_locality_, excluded_hosts_per_locality_, locality_weights_, overprovisioning_factor_); runUpdateCallbacks(hosts_added, hosts_removed); @@ -343,9 +344,8 @@ void HostSetImpl::rebuildLocalityScheduler( std::vector>& locality_entries, const HostsPerLocality& eligible_hosts_per_locality, const HostVector& eligible_hosts, HostsPerLocalityConstSharedPtr all_hosts_per_locality, - HostsPerLocalityConstSharedPtr warmed_counts_per_locality, + HostsPerLocalityConstSharedPtr excluded_hosts_per_locality, LocalityWeightsConstSharedPtr locality_weights, uint32_t overprovisioning_factor) { - ASSERT((all_hosts_per_locality == nullptr) == (warmed_counts_per_locality == nullptr)); // Rebuild the locality scheduler by computing the effective weight of each // locality in this priority. The scheduler is reset by default, and is rebuilt only if we have // locality weights (i.e. using EDS) and there is at least one eligible host in this priority. @@ -367,9 +367,9 @@ void HostSetImpl::rebuildLocalityScheduler( locality_scheduler = std::make_unique>(); locality_entries.clear(); for (uint32_t i = 0; i < all_hosts_per_locality->get().size(); ++i) { - const double effective_weight = - effectiveLocalityWeight(i, eligible_hosts_per_locality, *warmed_counts_per_locality, - *locality_weights, overprovisioning_factor); + const double effective_weight = effectiveLocalityWeight( + i, eligible_hosts_per_locality, *excluded_hosts_per_locality, *all_hosts_per_locality, + *locality_weights, overprovisioning_factor); if (effective_weight > 0) { locality_entries.emplace_back(std::make_shared(i, effective_weight)); locality_scheduler->add(effective_weight, locality_entries.back()); @@ -423,7 +423,7 @@ HostSetImpl::updateHostsParams(HostVectorConstSharedPtr hosts, return updateHostsParams( hosts, hosts_per_locality, std::move(healthy_hosts), std::move(healthy_hosts_per_locality), std::make_shared(), HostsPerLocalityImpl::empty(), - std::make_shared(*hosts), hosts_per_locality); + std::make_shared(), HostsPerLocalityImpl::empty()); } PrioritySet::UpdateHostsParams @@ -433,45 +433,49 @@ HostSetImpl::updateHostsParams(HostVectorConstSharedPtr hosts, HostsPerLocalityConstSharedPtr healthy_hosts_per_locality, DegradedHostVectorConstSharedPtr degraded_hosts, HostsPerLocalityConstSharedPtr degraded_hosts_per_locality, - WarmedHostVectorConstSharedPtr warmed_hosts, - HostsPerLocalityConstSharedPtr warmed_counts_per_locality) { + ExcludedHostVectorConstSharedPtr excluded_hosts, + HostsPerLocalityConstSharedPtr excluded_hosts_per_locality) { return PrioritySet::UpdateHostsParams{std::move(hosts), std::move(healthy_hosts), std::move(degraded_hosts), - std::move(warmed_hosts), + std::move(excluded_hosts), std::move(hosts_per_locality), std::move(healthy_hosts_per_locality), std::move(degraded_hosts_per_locality), - std::move(warmed_counts_per_locality)}; + std::move(excluded_hosts_per_locality)}; } PrioritySet::UpdateHostsParams HostSetImpl::partitionHosts(HostVectorConstSharedPtr hosts, HostsPerLocalityConstSharedPtr hosts_per_locality) { auto partitioned_hosts = ClusterImplBase::partitionHostList(*hosts); - auto healthy_degraded_warmed_hosts_per_locality = + auto healthy_degraded_excluded_hosts_per_locality = ClusterImplBase::partitionHostsPerLocality(*hosts_per_locality); return updateHostsParams(std::move(hosts), std::move(hosts_per_locality), std::move(std::get<0>(partitioned_hosts)), - std::move(std::get<0>(healthy_degraded_warmed_hosts_per_locality)), + std::move(std::get<0>(healthy_degraded_excluded_hosts_per_locality)), std::move(std::get<1>(partitioned_hosts)), - std::move(std::get<1>(healthy_degraded_warmed_hosts_per_locality)), + std::move(std::get<1>(healthy_degraded_excluded_hosts_per_locality)), std::move(std::get<2>(partitioned_hosts)), - std::move(std::get<2>(healthy_degraded_warmed_hosts_per_locality))); + std::move(std::get<2>(healthy_degraded_excluded_hosts_per_locality))); } double HostSetImpl::effectiveLocalityWeight(uint32_t index, const HostsPerLocality& eligible_hosts_per_locality, - const HostsPerLocality& warmed_count_per_locality, + const HostsPerLocality& excluded_hosts_per_locality, + const HostsPerLocality& all_hosts_per_locality, const LocalityWeights& locality_weights, uint32_t overprovisioning_factor) { const auto& locality_eligible_hosts = eligible_hosts_per_locality.get()[index]; - const auto warmed_count = warmed_count_per_locality.get()[index].size(); - if (warmed_count == 0) { + const uint32_t excluded_count = excluded_hosts_per_locality.get().size() > index + ? excluded_hosts_per_locality.get()[index].size() + : 0; + const auto host_count = all_hosts_per_locality.get()[index].size() - excluded_count; + if (host_count == 0) { return 0.0; } - const double locality_availability_ratio = 1.0 * locality_eligible_hosts.size() / warmed_count; + const double locality_availability_ratio = 1.0 * locality_eligible_hosts.size() / host_count; const uint32_t weight = locality_weights[index]; // Availability ranges from 0-1.0, and is the ratio of eligible hosts to total hosts, modified by // the overprovisioning factor. @@ -704,6 +708,7 @@ ClusterImplBase::ClusterImplBase( healthy_hosts += host_set->healthyHosts().size(); degraded_hosts += host_set->degradedHosts().size(); } + // TODO(snowp): Stats for excluded hosts? info_->stats().membership_total_.set(hosts); info_->stats().membership_healthy_.set(healthy_hosts); info_->stats().membership_degraded_.set(degraded_hosts); @@ -711,11 +716,11 @@ ClusterImplBase::ClusterImplBase( } std::tuple + ExcludedHostVectorConstSharedPtr> ClusterImplBase::partitionHostList(const HostVector& hosts) { auto healthy_list = std::make_shared(); auto degraded_list = std::make_shared(); - auto warmed_list = std::make_shared(); + auto excluded_list = std::make_shared(); for (const auto& host : hosts) { if (host->health() == Host::Health::Healthy) { @@ -724,12 +729,12 @@ ClusterImplBase::partitionHostList(const HostVector& hosts) { if (host->health() == Host::Health::Degraded) { degraded_list->get().emplace_back(host); } - if (host->warmed()) { - warmed_list->get().emplace_back(host); + if (!host->warmed()) { + excluded_list->get().emplace_back(host); } } - return {healthy_list, degraded_list, warmed_list}; + return {healthy_list, degraded_list, excluded_list}; } std::tupleget(); } const HostVector& degradedHosts() const override { return degraded_hosts_->get(); } - const HostVector& warmedHosts() const override { return warmed_hosts_->get(); } + const HostVector& excludedHosts() const override { return excluded_hosts_->get(); } const HostsPerLocality& hostsPerLocality() const override { return *hosts_per_locality_; } const HostsPerLocality& healthyHostsPerLocality() const override { return *healthy_hosts_per_locality_; @@ -312,8 +312,8 @@ class HostSetImpl : public HostSet { const HostsPerLocality& degradedHostsPerLocality() const override { return *degraded_hosts_per_locality_; } - const HostsPerLocality& warmedHostsPerLocality() const override { - return *warmed_hosts_per_locality_; + const HostsPerLocality& excludedHostsPerLocality() const override { + return *excluded_hosts_per_locality_; } LocalityWeightsConstSharedPtr localityWeights() const override { return locality_weights_; } absl::optional chooseHealthyLocality() override; @@ -337,8 +337,8 @@ class HostSetImpl : public HostSet { HostsPerLocalityConstSharedPtr healthy_hosts_per_locality, DegradedHostVectorConstSharedPtr degraded_hosts, HostsPerLocalityConstSharedPtr degraded_hosts_per_locality, - WarmedHostVectorConstSharedPtr warmed_hosts, - HostsPerLocalityConstSharedPtr warmed_counts_per_locality); + ExcludedHostVectorConstSharedPtr excluded_hosts, + HostsPerLocalityConstSharedPtr excluded_hosts_per_locality); static PrioritySet::UpdateHostsParams partitionHosts(HostVectorConstSharedPtr hosts, HostsPerLocalityConstSharedPtr hosts_per_locality); @@ -357,7 +357,8 @@ class HostSetImpl : public HostSet { // locality. static double effectiveLocalityWeight(uint32_t index, const HostsPerLocality& eligible_hosts_per_locality, - const HostsPerLocality& warmed_counts_per_locality, + const HostsPerLocality& excluded_hosts_per_locality, + const HostsPerLocality& all_hosts_per_locality, const LocalityWeights& locality_weights, uint32_t overprovisioning_factor); @@ -366,11 +367,11 @@ class HostSetImpl : public HostSet { HostVectorConstSharedPtr hosts_; HealthyHostVectorConstSharedPtr healthy_hosts_; DegradedHostVectorConstSharedPtr degraded_hosts_; - WarmedHostVectorConstSharedPtr warmed_hosts_; + ExcludedHostVectorConstSharedPtr excluded_hosts_; HostsPerLocalityConstSharedPtr hosts_per_locality_{HostsPerLocalityImpl::empty()}; HostsPerLocalityConstSharedPtr healthy_hosts_per_locality_{HostsPerLocalityImpl::empty()}; HostsPerLocalityConstSharedPtr degraded_hosts_per_locality_{HostsPerLocalityImpl::empty()}; - HostsPerLocalityConstSharedPtr warmed_hosts_per_locality_{HostsPerLocalityImpl::empty()}; + HostsPerLocalityConstSharedPtr excluded_hosts_per_locality_{HostsPerLocalityImpl::empty()}; // TODO(mattklein123): Remove mutable. mutable Common::CallbackManager member_update_cb_helper_; @@ -402,7 +403,7 @@ class HostSetImpl : public HostSet { std::vector>& locality_entries, const HostsPerLocality& eligible_hosts_per_locality, const HostVector& eligible_hosts, HostsPerLocalityConstSharedPtr all_hosts_per_locality, - HostsPerLocalityConstSharedPtr warmed_counts_per_locality, + HostsPerLocalityConstSharedPtr excluded_hosts_per_locality, LocalityWeightsConstSharedPtr locality_weights, uint32_t overprovisioning_factor); static absl::optional chooseLocality(EdfScheduler* locality_scheduler); @@ -656,13 +657,13 @@ class ClusterImplBase : public Cluster, protected Logger::Loggable + ExcludedHostVectorConstSharedPtr> partitionHostList(const HostVector& hosts); // Partitions the provided list of hosts per locality into three new lists containing the healthy, - // degraded and warmed hosts respectively. + // degraded and excluded hosts respectively. static std::tuple partitionHostsPerLocality(const HostsPerLocality& hosts); diff --git a/test/common/upstream/load_balancer_impl_test.cc b/test/common/upstream/load_balancer_impl_test.cc index 1337534c664ca..2558fbdfbbc80 100644 --- a/test/common/upstream/load_balancer_impl_test.cc +++ b/test/common/upstream/load_balancer_impl_test.cc @@ -81,7 +81,6 @@ class LoadBalancerBaseTest : public LoadBalancerTestBase { for (; i < num_degraded_hosts; ++i) { host_set.degraded_hosts_.push_back(host_set.hosts_[i]); } - host_set.warmed_hosts_ = host_set.hosts_; host_set.runCallbacks({}, {}); } @@ -460,10 +459,8 @@ typedef RoundRobinLoadBalancerTest FailoverTest; // Ensure if all the hosts with priority 0 unhealthy, the next priority hosts are used. TEST_P(FailoverTest, BasicFailover) { host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80")}; - host_set_.warmed_hosts_ = host_set_.hosts_; failover_host_set_.healthy_hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:82")}; failover_host_set_.hosts_ = failover_host_set_.healthy_hosts_; - failover_host_set_.warmed_hosts_ = failover_host_set_.hosts_; init(false); EXPECT_EQ(failover_host_set_.healthy_hosts_[0], lb_->chooseHost(nullptr)); } @@ -472,7 +469,6 @@ TEST_P(FailoverTest, BasicFailover) { TEST_P(FailoverTest, BasicDegradedHosts) { host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80")}; host_set_.degraded_hosts_ = host_set_.hosts_; - host_set_.warmed_hosts_ = host_set_.hosts_; failover_host_set_.hosts_ = failover_host_set_.healthy_hosts_; init(false); EXPECT_EQ(host_set_.degraded_hosts_[0], lb_->chooseHost(nullptr)); @@ -483,10 +479,8 @@ TEST_P(FailoverTest, BasicDegradedHosts) { TEST_P(FailoverTest, BasicFailoverDegradedHosts) { host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80")}; host_set_.degraded_hosts_ = host_set_.hosts_; - host_set_.warmed_hosts_ = host_set_.hosts_; failover_host_set_.healthy_hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:82")}; failover_host_set_.hosts_ = failover_host_set_.healthy_hosts_; - failover_host_set_.warmed_hosts_ = failover_host_set_.hosts_; init(false); EXPECT_EQ(failover_host_set_.healthy_hosts_[0], lb_->chooseHost(nullptr)); } @@ -494,9 +488,7 @@ TEST_P(FailoverTest, BasicFailoverDegradedHosts) { // Test that extending the priority set with an existing LB causes the correct updates. TEST_P(FailoverTest, PriorityUpdatesWithLocalHostSet) { host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80")}; - host_set_.warmed_hosts_ = host_set_.hosts_; failover_host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:81")}; - failover_host_set_.warmed_hosts_ = failover_host_set_.hosts_; init(false); // With both the primary and failover hosts unhealthy, we should select an // unhealthy primary host. @@ -508,7 +500,6 @@ TEST_P(FailoverTest, PriorityUpdatesWithLocalHostSet) { HostVectorSharedPtr hosts(new HostVector({makeTestHost(info_, "tcp://127.0.0.1:82")})); tertiary_host_set_.hosts_ = *hosts; tertiary_host_set_.healthy_hosts_ = tertiary_host_set_.hosts_; - tertiary_host_set_.warmed_hosts_ = tertiary_host_set_.hosts_; HostVector add_hosts; add_hosts.push_back(tertiary_host_set_.hosts_[0]); tertiary_host_set_.runCallbacks(add_hosts, {}); @@ -528,9 +519,7 @@ TEST_P(FailoverTest, PriorityUpdatesWithLocalHostSet) { // Test extending the priority set. TEST_P(FailoverTest, ExtendPrioritiesUpdatingPrioritySet) { host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80")}; - host_set_.warmed_hosts_ = host_set_.hosts_; failover_host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:81")}; - failover_host_set_.warmed_hosts_ = failover_host_set_.hosts_; init(true); // With both the primary and failover hosts unhealthy, we should select an // unhealthy primary host. @@ -542,7 +531,6 @@ TEST_P(FailoverTest, ExtendPrioritiesUpdatingPrioritySet) { HostVectorSharedPtr hosts(new HostVector({makeTestHost(info_, "tcp://127.0.0.1:82")})); tertiary_host_set_.hosts_ = *hosts; tertiary_host_set_.healthy_hosts_ = tertiary_host_set_.hosts_; - tertiary_host_set_.warmed_hosts_ = tertiary_host_set_.hosts_; HostVector add_hosts; add_hosts.push_back(tertiary_host_set_.hosts_[0]); tertiary_host_set_.runCallbacks(add_hosts, {}); @@ -556,9 +544,7 @@ TEST_P(FailoverTest, ExtendPrioritiesUpdatingPrioritySet) { TEST_P(FailoverTest, ExtendPrioritiesWithLocalPrioritySet) { host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80")}; - host_set_.warmed_hosts_ = host_set_.hosts_; failover_host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:81")}; - failover_host_set_.warmed_hosts_ = failover_host_set_.hosts_; init(true); // With both the primary and failover hosts unhealthy, we should select an // unhealthy primary host. @@ -569,7 +555,6 @@ TEST_P(FailoverTest, ExtendPrioritiesWithLocalPrioritySet) { MockHostSet& tertiary_host_set_ = *priority_set_.getMockHostSet(2); HostVectorSharedPtr hosts2(new HostVector({makeTestHost(info_, "tcp://127.0.0.1:84")})); tertiary_host_set_.hosts_ = *hosts2; - tertiary_host_set_.warmed_hosts_ = *hosts2; tertiary_host_set_.healthy_hosts_ = tertiary_host_set_.hosts_; HostVector add_hosts; add_hosts.push_back(tertiary_host_set_.hosts_[0]); @@ -592,7 +577,6 @@ TEST_P(FailoverTest, PrioritiesWithNotAllWarmedHosts) { host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80"), makeTestHost(info_, "tcp://127.0.0.1:81")}; host_set_.healthy_hosts_ = {host_set_.hosts_[0]}; - host_set_.warmed_hosts_ = {host_set_.hosts_[0]}; failover_host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:82")}; failover_host_set_.healthy_hosts_ = failover_host_set_.hosts_; init(true); @@ -612,7 +596,6 @@ TEST_P(FailoverTest, PrioritiesWithZeroWarmedHosts) { makeTestHost(info_, "tcp://127.0.0.1:81")}; failover_host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:82")}; failover_host_set_.healthy_hosts_ = failover_host_set_.hosts_; - failover_host_set_.warmed_hosts_ = failover_host_set_.hosts_; init(true); @@ -631,7 +614,6 @@ TEST_P(RoundRobinLoadBalancerTest, NoHosts) { TEST_P(RoundRobinLoadBalancerTest, SingleHost) { hostSet().healthy_hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80")}; hostSet().hosts_ = hostSet().healthy_hosts_; - hostSet().warmed_hosts_ = hostSet().healthy_hosts_; init(false); EXPECT_EQ(hostSet().healthy_hosts_[0], lb_->chooseHost(nullptr)); } @@ -640,7 +622,6 @@ TEST_P(RoundRobinLoadBalancerTest, Normal) { hostSet().healthy_hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80"), makeTestHost(info_, "tcp://127.0.0.1:81")}; hostSet().hosts_ = hostSet().healthy_hosts_; - hostSet().warmed_hosts_ = hostSet().healthy_hosts_; init(false); EXPECT_EQ(hostSet().healthy_hosts_[0], lb_->chooseHost(nullptr)); EXPECT_EQ(hostSet().healthy_hosts_[1], lb_->chooseHost(nullptr)); @@ -656,7 +637,6 @@ TEST_P(RoundRobinLoadBalancerTest, Seed) { makeTestHost(info_, "tcp://127.0.0.1:82"), }; hostSet().hosts_ = hostSet().healthy_hosts_; - hostSet().warmed_hosts_ = hostSet().healthy_hosts_; EXPECT_CALL(random_, random()).WillRepeatedly(Return(1)); init(false); EXPECT_EQ(hostSet().healthy_hosts_[1], lb_->chooseHost(nullptr)); @@ -672,7 +652,6 @@ TEST_P(RoundRobinLoadBalancerTest, Locality) { HostsPerLocalitySharedPtr hosts_per_locality = makeHostsPerLocality({{(*hosts)[1]}, {(*hosts)[0]}, {(*hosts)[2]}}); hostSet().hosts_ = *hosts; - hostSet().warmed_hosts_ = *hosts; hostSet().healthy_hosts_ = *hosts; hostSet().healthy_hosts_per_locality_ = hosts_per_locality; init(false); @@ -709,7 +688,6 @@ TEST_P(RoundRobinLoadBalancerTest, DegradedLocality) { makeHostsPerLocality({{}, {(*hosts)[1], (*hosts)[2]}}); hostSet().hosts_ = *hosts; - hostSet().warmed_hosts_ = *hosts; hostSet().healthy_hosts_ = *healthy_hosts; hostSet().degraded_hosts_ = *degraded_hosts; hostSet().hosts_per_locality_ = hosts_per_locality; @@ -730,7 +708,6 @@ TEST_P(RoundRobinLoadBalancerTest, Weighted) { hostSet().healthy_hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80", 1), makeTestHost(info_, "tcp://127.0.0.1:81", 2)}; hostSet().hosts_ = hostSet().healthy_hosts_; - hostSet().warmed_hosts_ = hostSet().healthy_hosts_; init(false); // Initial weights respected. EXPECT_EQ(hostSet().healthy_hosts_[1], lb_->chooseHost(nullptr)); @@ -793,7 +770,6 @@ TEST_P(RoundRobinLoadBalancerTest, WeightedSeed) { hostSet().healthy_hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80", 1), makeTestHost(info_, "tcp://127.0.0.1:81", 2)}; hostSet().hosts_ = hostSet().healthy_hosts_; - hostSet().warmed_hosts_ = hostSet().healthy_hosts_; EXPECT_CALL(random_, random()).WillRepeatedly(Return(1)); init(false); // Initial weights respected. @@ -812,7 +788,6 @@ TEST_P(RoundRobinLoadBalancerTest, MaxUnhealthyPanic) { makeTestHost(info_, "tcp://127.0.0.1:80"), makeTestHost(info_, "tcp://127.0.0.1:81"), makeTestHost(info_, "tcp://127.0.0.1:82"), makeTestHost(info_, "tcp://127.0.0.1:83"), makeTestHost(info_, "tcp://127.0.0.1:84"), makeTestHost(info_, "tcp://127.0.0.1:85")}; - hostSet().warmed_hosts_ = hostSet().hosts_; init(false); EXPECT_EQ(hostSet().hosts_[0], lb_->chooseHost(nullptr)); @@ -842,7 +817,6 @@ TEST_P(RoundRobinLoadBalancerTest, HostSelectionWithFilter) { hostSet().hosts_ = *hosts; hostSet().healthy_hosts_ = *hosts; - hostSet().warmed_hosts_ = *hosts; hostSet().healthy_hosts_per_locality_ = hosts_per_locality; init(false); @@ -888,7 +862,6 @@ TEST_P(RoundRobinLoadBalancerTest, ZoneAwareSmallCluster) { hostSet().hosts_ = *hosts; hostSet().healthy_hosts_ = *hosts; - hostSet().warmed_hosts_ = *hosts; hostSet().healthy_hosts_per_locality_ = hosts_per_locality; common_config_.mutable_healthy_panic_threshold()->set_value(0); common_config_.mutable_zone_aware_lb_config()->mutable_routing_enabled()->set_value(98); @@ -937,7 +910,6 @@ TEST_P(RoundRobinLoadBalancerTest, NoZoneAwareDifferentZoneSize) { hostSet().healthy_hosts_ = *hosts; hostSet().hosts_ = *hosts; - hostSet().warmed_hosts_ = *hosts; hostSet().healthy_hosts_per_locality_ = upstream_hosts_per_locality; common_config_.mutable_healthy_panic_threshold()->set_value(100); common_config_.mutable_zone_aware_lb_config()->mutable_routing_enabled()->set_value(98); @@ -977,7 +949,6 @@ TEST_P(RoundRobinLoadBalancerTest, ZoneAwareRoutingLargeZoneSwitchOnOff) { hostSet().healthy_hosts_ = *hosts; hostSet().hosts_ = *hosts; - hostSet().warmed_hosts_ = *hosts; hostSet().healthy_hosts_per_locality_ = hosts_per_locality; init(true); updateHosts(hosts, hosts_per_locality); @@ -1025,7 +996,6 @@ TEST_P(RoundRobinLoadBalancerTest, ZoneAwareRoutingSmallZone) { hostSet().healthy_hosts_ = *upstream_hosts; hostSet().hosts_ = *upstream_hosts; - hostSet().warmed_hosts_ = *upstream_hosts; hostSet().healthy_hosts_per_locality_ = upstream_hosts_per_locality; init(true); updateHosts(local_hosts, local_hosts_per_locality); @@ -1048,7 +1018,6 @@ TEST_P(RoundRobinLoadBalancerTest, LowPrecisionForDistribution) { // upstream_hosts and local_hosts do not matter, zone aware routing is based on per zone hosts. HostVectorSharedPtr upstream_hosts(new HostVector({makeTestHost(info_, "tcp://127.0.0.1:80")})); hostSet().healthy_hosts_ = *upstream_hosts; - hostSet().warmed_hosts_ = *upstream_hosts; hostSet().hosts_ = *upstream_hosts; HostVectorSharedPtr local_hosts(new HostVector({makeTestHost(info_, "tcp://127.0.0.1:0")})); @@ -1129,7 +1098,6 @@ TEST_P(RoundRobinLoadBalancerTest, NoZoneAwareRoutingNotHealthy) { {{}, {makeTestHost(info_, "tcp://127.0.0.1:80"), makeTestHost(info_, "tcp://127.0.0.2:80")}}); hostSet().healthy_hosts_ = *hosts; - hostSet().warmed_hosts_ = *hosts; hostSet().hosts_ = *hosts; hostSet().healthy_hosts_per_locality_ = hosts_per_locality; init(true); @@ -1161,7 +1129,6 @@ TEST_P(RoundRobinLoadBalancerTest, NoZoneAwareRoutingLocalEmpty) { hostSet().healthy_hosts_ = *upstream_hosts; hostSet().hosts_ = *upstream_hosts; - hostSet().warmed_hosts_ = *upstream_hosts; hostSet().healthy_hosts_per_locality_ = upstream_hosts_per_locality; init(true); updateHosts(local_hosts, local_hosts_per_locality); @@ -1189,7 +1156,6 @@ TEST_P(RoundRobinLoadBalancerTest, NoZoneAwareRoutingNoLocalLocality) { hostSet().healthy_hosts_ = *upstream_hosts; hostSet().hosts_ = *upstream_hosts; - hostSet().warmed_hosts_ = *upstream_hosts; hostSet().healthy_hosts_per_locality_ = upstream_hosts_per_locality; init(true); updateHosts(local_hosts, local_hosts_per_locality); @@ -1213,7 +1179,6 @@ TEST_P(LeastRequestLoadBalancerTest, NoHosts) { EXPECT_EQ(nullptr, lb_.chooseHos TEST_P(LeastRequestLoadBalancerTest, SingleHost) { hostSet().healthy_hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80")}; - hostSet().warmed_hosts_ = hostSet().healthy_hosts_; hostSet().hosts_ = hostSet().healthy_hosts_; hostSet().runCallbacks({}, {}); // Trigger callbacks. The added/removed lists are not relevant. @@ -1254,7 +1219,6 @@ TEST_P(LeastRequestLoadBalancerTest, Normal) { makeTestHost(info_, "tcp://127.0.0.1:81")}; stats_.max_host_weight_.set(1UL); hostSet().hosts_ = hostSet().healthy_hosts_; - hostSet().warmed_hosts_ = hostSet().healthy_hosts_; hostSet().runCallbacks({}, {}); // Trigger callbacks. The added/removed lists are not relevant. hostSet().healthy_hosts_[0]->stats().rq_active_.set(1); @@ -1274,7 +1238,6 @@ TEST_P(LeastRequestLoadBalancerTest, PNC) { makeTestHost(info_, "tcp://127.0.0.1:82"), makeTestHost(info_, "tcp://127.0.0.1:83")}; stats_.max_host_weight_.set(1UL); hostSet().hosts_ = hostSet().healthy_hosts_; - hostSet().warmed_hosts_ = hostSet().healthy_hosts_; hostSet().runCallbacks({}, {}); // Trigger callbacks. The added/removed lists are not relevant. hostSet().healthy_hosts_[0]->stats().rq_active_.set(4); @@ -1323,7 +1286,6 @@ TEST_P(LeastRequestLoadBalancerTest, WeightImbalance) { stats_.max_host_weight_.set(2UL); hostSet().hosts_ = hostSet().healthy_hosts_; - hostSet().warmed_hosts_ = hostSet().healthy_hosts_; hostSet().runCallbacks({}, {}); // Trigger callbacks. The added/removed lists are not relevant. EXPECT_CALL(random_, random()).WillRepeatedly(Return(0)); @@ -1366,7 +1328,6 @@ TEST_P(LeastRequestLoadBalancerTest, WeightImbalanceCallbacks) { stats_.max_host_weight_.set(2UL); hostSet().hosts_ = hostSet().healthy_hosts_; - hostSet().warmed_hosts_ = hostSet().healthy_hosts_; hostSet().runCallbacks({}, {}); // Trigger callbacks. The added/removed lists are not relevant. EXPECT_CALL(random_, random()).WillRepeatedly(Return(0)); @@ -1398,7 +1359,6 @@ TEST_P(RandomLoadBalancerTest, Normal) { hostSet().healthy_hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80"), makeTestHost(info_, "tcp://127.0.0.1:81")}; hostSet().hosts_ = hostSet().healthy_hosts_; - hostSet().warmed_hosts_ = hostSet().hosts_; hostSet().runCallbacks({}, {}); // Trigger callbacks. The added/removed lists are not relevant. EXPECT_CALL(random_, random()).WillOnce(Return(0)).WillOnce(Return(2)); EXPECT_EQ(hostSet().healthy_hosts_[0], lb_.chooseHost(nullptr)); diff --git a/test/common/upstream/ring_hash_lb_test.cc b/test/common/upstream/ring_hash_lb_test.cc index 71d34add835ba..35d7942ac60f2 100644 --- a/test/common/upstream/ring_hash_lb_test.cc +++ b/test/common/upstream/ring_hash_lb_test.cc @@ -90,7 +90,6 @@ TEST_P(RingHashLoadBalancerTest, Basic) { makeTestHost(info_, "tcp://127.0.0.1:92"), makeTestHost(info_, "tcp://127.0.0.1:93"), makeTestHost(info_, "tcp://127.0.0.1:94"), makeTestHost(info_, "tcp://127.0.0.1:95")}; hostSet().healthy_hosts_ = hostSet().hosts_; - hostSet().warmed_hosts_ = hostSet().hosts_; hostSet().runCallbacks({}, {}); config_ = envoy::api::v2::Cluster::RingHashLbConfig(); @@ -162,10 +161,8 @@ TEST_P(RingHashLoadBalancerTest, Basic) { // Ensure if all the hosts with priority 0 unhealthy, the next priority hosts are used. TEST_P(RingHashFailoverTest, BasicFailover) { host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80")}; - host_set_.warmed_hosts_ = host_set_.hosts_; failover_host_set_.healthy_hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:82")}; failover_host_set_.hosts_ = failover_host_set_.healthy_hosts_; - failover_host_set_.warmed_hosts_ = failover_host_set_.hosts_; config_ = envoy::api::v2::Cluster::RingHashLbConfig(); config_.value().mutable_minimum_ring_size()->set_value(12); @@ -193,7 +190,6 @@ TEST_P(RingHashFailoverTest, BasicFailover) { host_set_.hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80"), makeTestHost(info_, "tcp://127.0.0.1:81")}; host_set_.healthy_hosts_ = {host_set_.hosts_[0]}; - host_set_.warmed_hosts_ = host_set_.hosts_; host_set_.runCallbacks({}, {}); lb = lb_->factory()->create(); EXPECT_CALL(random_, random()).WillOnce(Return(69)); @@ -209,7 +205,6 @@ TEST_P(RingHashLoadBalancerTest, BasicWithMurmur2) { makeTestHost(info_, "tcp://127.0.0.1:82"), makeTestHost(info_, "tcp://127.0.0.1:83"), makeTestHost(info_, "tcp://127.0.0.1:84"), makeTestHost(info_, "tcp://127.0.0.1:85")}; hostSet().healthy_hosts_ = hostSet().hosts_; - hostSet().warmed_hosts_ = hostSet().hosts_; hostSet().runCallbacks({}, {}); config_ = envoy::api::v2::Cluster::RingHashLbConfig(); @@ -263,7 +258,6 @@ TEST_P(RingHashLoadBalancerTest, UnevenHosts) { hostSet().hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80"), makeTestHost(info_, "tcp://127.0.0.1:81")}; hostSet().healthy_hosts_ = hostSet().hosts_; - hostSet().warmed_hosts_ = hostSet().hosts_; hostSet().runCallbacks({}, {}); config_ = envoy::api::v2::Cluster::RingHashLbConfig(); @@ -290,7 +284,6 @@ TEST_P(RingHashLoadBalancerTest, UnevenHosts) { hostSet().hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:81"), makeTestHost(info_, "tcp://127.0.0.1:82")}; hostSet().healthy_hosts_ = hostSet().hosts_; - hostSet().warmed_hosts_ = hostSet().hosts_; hostSet().runCallbacks({}, {}); // hash ring: @@ -315,7 +308,6 @@ TEST_P(RingHashLoadBalancerTest, HostWeightedTinyRing) { makeTestHost(info_, "tcp://127.0.0.1:91", 2), makeTestHost(info_, "tcp://127.0.0.1:92", 3)}; hostSet().healthy_hosts_ = hostSet().hosts_; - hostSet().warmed_hosts_ = hostSet().hosts_; hostSet().runCallbacks({}, {}); // enforce a ring size of exactly six entries @@ -345,7 +337,6 @@ TEST_P(RingHashLoadBalancerTest, HostWeightedLargeRing) { makeTestHost(info_, "tcp://127.0.0.1:91", 2), makeTestHost(info_, "tcp://127.0.0.1:92", 3)}; hostSet().healthy_hosts_ = hostSet().hosts_; - hostSet().warmed_hosts_ = hostSet().hosts_; hostSet().runCallbacks({}, {}); config_ = envoy::api::v2::Cluster::RingHashLbConfig(); @@ -374,7 +365,6 @@ TEST_P(RingHashLoadBalancerTest, ZeroLocalityWeights) { hostSet().hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:90"), makeTestHost(info_, "tcp://127.0.0.1:91")}; hostSet().healthy_hosts_ = hostSet().hosts_; - hostSet().warmed_hosts_ = hostSet().hosts_; hostSet().hosts_per_locality_ = makeHostsPerLocality({{hostSet().hosts_[0]}, {hostSet().hosts_[1]}}); hostSet().healthy_hosts_per_locality_ = hostSet().hosts_per_locality_; @@ -392,7 +382,6 @@ TEST_P(RingHashLoadBalancerTest, LocalityWeightedTinyRing) { makeTestHost(info_, "tcp://127.0.0.1:90"), makeTestHost(info_, "tcp://127.0.0.1:91"), makeTestHost(info_, "tcp://127.0.0.1:92"), makeTestHost(info_, "tcp://127.0.0.1:93")}; hostSet().healthy_hosts_ = hostSet().hosts_; - hostSet().warmed_hosts_ = hostSet().hosts_; hostSet().hosts_per_locality_ = makeHostsPerLocality( {{hostSet().hosts_[0]}, {hostSet().hosts_[1]}, {hostSet().hosts_[2]}, {hostSet().hosts_[3]}}); hostSet().healthy_hosts_per_locality_ = hostSet().hosts_per_locality_; @@ -427,7 +416,6 @@ TEST_P(RingHashLoadBalancerTest, LocalityWeightedLargeRing) { makeTestHost(info_, "tcp://127.0.0.1:90"), makeTestHost(info_, "tcp://127.0.0.1:91"), makeTestHost(info_, "tcp://127.0.0.1:92"), makeTestHost(info_, "tcp://127.0.0.1:93")}; hostSet().healthy_hosts_ = hostSet().hosts_; - hostSet().warmed_hosts_ = hostSet().hosts_; hostSet().hosts_per_locality_ = makeHostsPerLocality( {{hostSet().hosts_[0]}, {hostSet().hosts_[1]}, {hostSet().hosts_[2]}, {hostSet().hosts_[3]}}); hostSet().healthy_hosts_per_locality_ = hostSet().hosts_per_locality_; @@ -464,7 +452,6 @@ TEST_P(RingHashLoadBalancerTest, HostAndLocalityWeightedTinyRing) { makeTestHost(info_, "tcp://127.0.0.1:90", 1), makeTestHost(info_, "tcp://127.0.0.1:91", 2), makeTestHost(info_, "tcp://127.0.0.1:92", 1), makeTestHost(info_, "tcp://127.0.0.1:93", 2)}; hostSet().healthy_hosts_ = hostSet().hosts_; - hostSet().warmed_hosts_ = hostSet().hosts_; hostSet().hosts_per_locality_ = makeHostsPerLocality( {{hostSet().hosts_[0], hostSet().hosts_[1]}, {hostSet().hosts_[2], hostSet().hosts_[3]}}); hostSet().healthy_hosts_per_locality_ = hostSet().hosts_per_locality_; @@ -502,7 +489,6 @@ TEST_P(RingHashLoadBalancerTest, HostAndLocalityWeightedLargeRing) { makeTestHost(info_, "tcp://127.0.0.1:90", 1), makeTestHost(info_, "tcp://127.0.0.1:91", 2), makeTestHost(info_, "tcp://127.0.0.1:92", 1), makeTestHost(info_, "tcp://127.0.0.1:93", 2)}; hostSet().healthy_hosts_ = hostSet().hosts_; - hostSet().warmed_hosts_ = hostSet().hosts_; hostSet().hosts_per_locality_ = makeHostsPerLocality( {{hostSet().hosts_[0], hostSet().hosts_[1]}, {hostSet().hosts_[2], hostSet().hosts_[3]}}); hostSet().healthy_hosts_per_locality_ = hostSet().hosts_per_locality_; @@ -538,7 +524,6 @@ TEST_P(RingHashLoadBalancerTest, SmallFractionalScale) { makeTestHost(info_, "tcp://127.0.0.1:90"), makeTestHost(info_, "tcp://127.0.0.1:91"), makeTestHost(info_, "tcp://127.0.0.1:92"), makeTestHost(info_, "tcp://127.0.0.1:93")}; hostSet().healthy_hosts_ = hostSet().hosts_; - hostSet().warmed_hosts_ = hostSet().hosts_; hostSet().runCallbacks({}, {}); config_ = envoy::api::v2::Cluster::RingHashLbConfig(); @@ -579,7 +564,6 @@ TEST_P(RingHashLoadBalancerTest, LargeFractionalScale) { hostSet().hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:90"), makeTestHost(info_, "tcp://127.0.0.1:91")}; hostSet().healthy_hosts_ = hostSet().hosts_; - hostSet().warmed_hosts_ = hostSet().hosts_; hostSet().runCallbacks({}, {}); config_ = envoy::api::v2::Cluster::RingHashLbConfig(); @@ -614,7 +598,6 @@ TEST_P(RingHashLoadBalancerTest, LopsidedWeightSmallScale) { (i == 0 ? heavy_but_sparse : light_but_dense).push_back(host); } hostSet().healthy_hosts_ = hostSet().hosts_; - hostSet().warmed_hosts_ = hostSet().hosts_; hostSet().hosts_per_locality_ = makeHostsPerLocality({heavy_but_sparse, light_but_dense}); hostSet().healthy_hosts_per_locality_ = hostSet().hosts_per_locality_; hostSet().locality_weights_ = makeLocalityWeights({127, 1}); diff --git a/test/common/upstream/subset_lb_test.cc b/test/common/upstream/subset_lb_test.cc index 68ea7ae94cd08..a18cff8abadc9 100644 --- a/test/common/upstream/subset_lb_test.cc +++ b/test/common/upstream/subset_lb_test.cc @@ -132,7 +132,6 @@ class SubsetLoadBalancerTest : public testing::TestWithParam { host_set.hosts_ = hosts; host_set.hosts_per_locality_ = makeHostsPerLocality({hosts}); host_set.healthy_hosts_ = host_set.hosts_; - host_set.warmed_hosts_ = host_set.hosts_; host_set.healthy_hosts_per_locality_ = host_set.hosts_per_locality_; } @@ -156,9 +155,7 @@ class SubsetLoadBalancerTest : public testing::TestWithParam { host_set.hosts_ = all_hosts; host_set.hosts_per_locality_ = makeHostsPerLocality({first_locality, second_locality}); - host_set.warmed_hosts_per_locality_ = makeHostsPerLocality({first_locality, second_locality}); host_set.healthy_hosts_ = host_set.hosts_; - host_set.warmed_hosts_ = host_set.hosts_; host_set.healthy_hosts_per_locality_ = host_set.hosts_per_locality_; host_set.locality_weights_ = std::make_shared(locality_weights); } @@ -203,8 +200,6 @@ class SubsetLoadBalancerTest : public testing::TestWithParam { host_set_.healthy_hosts_ = host_set_.hosts_; host_set_.healthy_hosts_per_locality_ = host_set_.hosts_per_locality_; - host_set_.warmed_hosts_ = host_set_.hosts_; - host_set_.warmed_hosts_per_locality_ = host_set_.hosts_per_locality_; local_hosts_.reset(new HostVector()); std::vector local_hosts_per_locality_vector; @@ -225,7 +220,7 @@ class SubsetLoadBalancerTest : public testing::TestWithParam { local_hosts_, local_hosts_per_locality_, std::make_shared(*local_hosts_), local_hosts_per_locality_, std::make_shared(), HostsPerLocalityImpl::empty(), - std::make_shared(*local_hosts_), local_hosts_per_locality_), + std::make_shared(), HostsPerLocalityImpl::empty()), {}, {}, {}, absl::nullopt); lb_.reset(new SubsetLoadBalancer( @@ -275,7 +270,6 @@ class SubsetLoadBalancerTest : public testing::TestWithParam { } host_set.hosts_per_locality_ = makeHostsPerLocality(std::move(locality_hosts_copy)); host_set.healthy_hosts_per_locality_ = host_set.hosts_per_locality_; - host_set.warmed_hosts_per_locality_ = host_set.hosts_per_locality_; } if (GetParam() == REMOVES_FIRST && !remove.empty()) { @@ -285,7 +279,6 @@ class SubsetLoadBalancerTest : public testing::TestWithParam { for (const auto& host : add) { host_set.hosts_.emplace_back(host); host_set.healthy_hosts_ = host_set.hosts_; - host_set.warmed_hosts_ = host_set.hosts_; if (add_in_locality) { std::vector locality_hosts_copy = host_set.hosts_per_locality_->get(); diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc index 52ad44b064b5f..0aa858c200876 100644 --- a/test/common/upstream/upstream_impl_test.cc +++ b/test/common/upstream/upstream_impl_test.cc @@ -2321,6 +2321,8 @@ TEST_F(HostSetImplLocalityTest, NotWarmedHostsLocality) { auto hosts = makeHostsFromHostsPerLocality(hosts_per_locality); HostsPerLocalitySharedPtr healthy_hosts_per_locality = makeHostsPerLocality({{hosts_[0]}, {hosts_[3], hosts_[4]}}); + HostsPerLocalitySharedPtr excluded_hosts_per_locality = + makeHostsPerLocality({{hosts_[1], hosts_[2]}, {}}); host_set_.updateHosts( HostSetImpl::updateHostsParams( @@ -2328,8 +2330,8 @@ TEST_F(HostSetImplLocalityTest, NotWarmedHostsLocality) { makeHostsFromHostsPerLocality(healthy_hosts_per_locality), healthy_hosts_per_locality, std::make_shared(), HostsPerLocalityImpl::empty(), - makeHostsFromHostsPerLocality(healthy_hosts_per_locality), - healthy_hosts_per_locality), + makeHostsFromHostsPerLocality(excluded_hosts_per_locality), + excluded_hosts_per_locality), locality_weights, {}, {}, absl::nullopt); // We should RR between localities with equal weight. EXPECT_EQ(0, host_set_.chooseHealthyLocality().value()); @@ -2338,35 +2340,6 @@ TEST_F(HostSetImplLocalityTest, NotWarmedHostsLocality) { EXPECT_EQ(1, host_set_.chooseHealthyLocality().value()); } -// Verifies that we handle the case when there are no warmed hosts in a set with unhealthy hosts. -TEST_F(HostSetImplLocalityTest, ZeroWarmedHostsLocality) { - ON_CALL(*info_, warmHosts()).WillByDefault(Return(true)); - - // We have two localities with 2 hosts in L1, 2 hosts in L2. The two hosts in L1 are not warmed. - HostsPerLocalitySharedPtr hosts_per_locality = - makeHostsPerLocality({{hosts_[1], hosts_[2]}, {hosts_[3], hosts_[4]}}); - LocalityWeightsConstSharedPtr locality_weights{new LocalityWeights{1, 1}}; - auto hosts = makeHostsFromHostsPerLocality(hosts_per_locality); - HostsPerLocalitySharedPtr healthy_hosts_per_locality = - makeHostsPerLocality({{}, {hosts_[3], hosts_[4]}}); - - // Mark hosts 1 and 2 as not warmed. - hosts_[1]->healthFlagSet(Host::HealthFlag::FAILED_ACTIVE_HC); - hosts_[1]->setActiveHealthFailureType(Host::ActiveHealthFailureType::UNKNOWN); - hosts_[2]->healthFlagSet(Host::HealthFlag::FAILED_ACTIVE_HC); - hosts_[2]->setActiveHealthFailureType(Host::ActiveHealthFailureType::UNKNOWN); - - host_set_.updateHosts( - HostSetImpl::updateHostsParams( - hosts, hosts_per_locality, - makeHostsFromHostsPerLocality(healthy_hosts_per_locality), - healthy_hosts_per_locality), - locality_weights, {}, {}, absl::nullopt); - // We should only target the second locality since there are no healthy in the first. - EXPECT_EQ(1, host_set_.chooseHealthyLocality().value()); - EXPECT_EQ(1, host_set_.chooseHealthyLocality().value()); -} - // When a locality has zero hosts, it should be treated as if it has zero healthy. TEST_F(HostSetImplLocalityTest, EmptyLocality) { HostsPerLocalitySharedPtr hosts_per_locality = diff --git a/test/extensions/retry/priority/previous_priorities/config_test.cc b/test/extensions/retry/priority/previous_priorities/config_test.cc index f6f77e8fded7d..9b2df93460202 100644 --- a/test/extensions/retry/priority/previous_priorities/config_test.cc +++ b/test/extensions/retry/priority/previous_priorities/config_test.cc @@ -42,7 +42,6 @@ class RetryPriorityTest : public testing::Test { ASSERT(count >= healthy_count + degraded_count); host_set->hosts_.resize(count); - host_set->warmed_hosts_.resize(count); host_set->healthy_hosts_.resize(healthy_count); host_set->degraded_hosts_.resize(degraded_count); host_set->runCallbacks({}, {}); diff --git a/test/mocks/upstream/mocks.cc b/test/mocks/upstream/mocks.cc index 90d0356331ae0..973c3d53983dc 100644 --- a/test/mocks/upstream/mocks.cc +++ b/test/mocks/upstream/mocks.cc @@ -24,7 +24,7 @@ MockHostSet::MockHostSet(uint32_t priority, uint32_t overprovisioning_factor) ON_CALL(*this, hosts()).WillByDefault(ReturnRef(hosts_)); ON_CALL(*this, healthyHosts()).WillByDefault(ReturnRef(healthy_hosts_)); ON_CALL(*this, degradedHosts()).WillByDefault(ReturnRef(degraded_hosts_)); - ON_CALL(*this, warmedHosts()).WillByDefault(ReturnRef(warmed_hosts_)); + ON_CALL(*this, excludedHosts()).WillByDefault(ReturnRef(excluded_hosts_)); ON_CALL(*this, hostsPerLocality()).WillByDefault(Invoke([this]() -> const HostsPerLocality& { return *hosts_per_locality_; })); @@ -34,9 +34,9 @@ MockHostSet::MockHostSet(uint32_t priority, uint32_t overprovisioning_factor) ON_CALL(*this, degradedHostsPerLocality()) .WillByDefault( Invoke([this]() -> const HostsPerLocality& { return *degraded_hosts_per_locality_; })); - ON_CALL(*this, warmedHostsPerLocality()) + ON_CALL(*this, excludedHostsPerLocality()) .WillByDefault( - Invoke([this]() -> const HostsPerLocality& { return *warmed_hosts_per_locality_; })); + Invoke([this]() -> const HostsPerLocality& { return *excluded_hosts_per_locality_; })); ON_CALL(*this, localityWeights()).WillByDefault(Invoke([this]() -> LocalityWeightsConstSharedPtr { return locality_weights_; })); diff --git a/test/mocks/upstream/mocks.h b/test/mocks/upstream/mocks.h index 941b78ca62208..3bf8873ea8e97 100644 --- a/test/mocks/upstream/mocks.h +++ b/test/mocks/upstream/mocks.h @@ -52,11 +52,11 @@ class MockHostSet : public HostSet { MOCK_CONST_METHOD0(hosts, const HostVector&()); MOCK_CONST_METHOD0(healthyHosts, const HostVector&()); MOCK_CONST_METHOD0(degradedHosts, const HostVector&()); - MOCK_CONST_METHOD0(warmedHosts, const HostVector&()); + MOCK_CONST_METHOD0(excludedHosts, const HostVector&()); MOCK_CONST_METHOD0(hostsPerLocality, const HostsPerLocality&()); MOCK_CONST_METHOD0(healthyHostsPerLocality, const HostsPerLocality&()); MOCK_CONST_METHOD0(degradedHostsPerLocality, const HostsPerLocality&()); - MOCK_CONST_METHOD0(warmedHostsPerLocality, const HostsPerLocality&()); + MOCK_CONST_METHOD0(excludedHostsPerLocality, const HostsPerLocality&()); MOCK_CONST_METHOD0(localityWeights, LocalityWeightsConstSharedPtr()); MOCK_METHOD0(chooseHealthyLocality, absl::optional()); MOCK_METHOD0(chooseDegradedLocality, absl::optional()); @@ -69,11 +69,11 @@ class MockHostSet : public HostSet { HostVector hosts_; HostVector healthy_hosts_; HostVector degraded_hosts_; - HostVector warmed_hosts_; + HostVector excluded_hosts_; HostsPerLocalitySharedPtr hosts_per_locality_{new HostsPerLocalityImpl()}; HostsPerLocalitySharedPtr healthy_hosts_per_locality_{new HostsPerLocalityImpl()}; HostsPerLocalitySharedPtr degraded_hosts_per_locality_{new HostsPerLocalityImpl()}; - HostsPerLocalitySharedPtr warmed_hosts_per_locality_{new HostsPerLocalityImpl()}; + HostsPerLocalitySharedPtr excluded_hosts_per_locality_{new HostsPerLocalityImpl()}; LocalityWeightsConstSharedPtr locality_weights_{{}}; Common::CallbackManager member_update_cb_helper_; uint32_t priority_{}; From ce7d5b42869b2590ff231886c0113733448e6670 Mon Sep 17 00:00:00 2001 From: Snow Pettersen Date: Sun, 5 May 2019 12:37:19 -0400 Subject: [PATCH 15/28] fix cm tests Signed-off-by: Snow Pettersen --- source/common/upstream/cluster_manager_impl.cc | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index 7e37e25e87ac5..2f14631e113a5 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -687,21 +687,22 @@ void ClusterManagerImpl::postThreadLocalClusterUpdate(const Cluster& cluster, ui host_set->healthyHostsPerLocality().clone(); HostsPerLocalityConstSharedPtr degraded_hosts_per_locality_copy = host_set->degradedHostsPerLocality().clone(); - WarmedHostVectorConstSharedPtr warmed_hosts_copy(new WarmedHostVector(host_set->warmedHosts())); - auto warmed_hosts_per_locality_copy = host_set->warmedHostsPerLocality().clone(); + ExcludedHostVectorConstSharedPtr excluded_hosts_copy( + new ExcludedHostVector(host_set->excludedHosts())); + auto excluded_hosts_per_locality_copy = host_set->excludedHostsPerLocality().clone(); tls_->runOnAllThreads([this, name = cluster.info()->name(), priority, hosts_copy, healthy_hosts_copy, degraded_hosts_copy, hosts_per_locality_copy, healthy_hosts_per_locality_copy, degraded_hosts_per_locality_copy, - warmed_hosts_copy, warmed_hosts_per_locality_copy, + excluded_hosts_copy, excluded_hosts_per_locality_copy, locality_weights = host_set->localityWeights(), hosts_added, hosts_removed, overprovisioning_factor = host_set->overprovisioningFactor()]() { ThreadLocalClusterManagerImpl::updateClusterMembership( name, priority, HostSetImpl::updateHostsParams(hosts_copy, hosts_per_locality_copy, healthy_hosts_copy, healthy_hosts_per_locality_copy, degraded_hosts_copy, - degraded_hosts_per_locality_copy, warmed_hosts_copy, - warmed_hosts_per_locality_copy), + degraded_hosts_per_locality_copy, excluded_hosts_copy, + excluded_hosts_per_locality_copy), locality_weights, hosts_added, hosts_removed, *tls_, overprovisioning_factor); }); } From f017af1710cdfe81341699bfc582900b907fa231 Mon Sep 17 00:00:00 2001 From: Snow Pettersen Date: Mon, 6 May 2019 12:29:52 -0400 Subject: [PATCH 16/28] expose const shared ptr, use UpdateHostsParams Signed-off-by: Snow Pettersen --- include/envoy/upstream/upstream.h | 40 +++++++++++++++++++ .../common/upstream/cluster_manager_impl.cc | 32 +++------------ source/common/upstream/cluster_manager_impl.h | 2 +- source/common/upstream/upstream_impl.cc | 6 +++ source/common/upstream/upstream_impl.h | 17 ++++++++ test/mocks/upstream/mocks.cc | 16 ++++++++ test/mocks/upstream/mocks.h | 8 ++++ 7 files changed, 93 insertions(+), 28 deletions(-) diff --git a/include/envoy/upstream/upstream.h b/include/envoy/upstream/upstream.h index ea70a3ada8978..21991fc8cf014 100644 --- a/include/envoy/upstream/upstream.h +++ b/include/envoy/upstream/upstream.h @@ -275,6 +275,11 @@ class HostSet { */ virtual const HostVector& hosts() const PURE; + /** + * @return a shared ptr to the vector returned by hosts(). + */ + virtual HostVectorConstSharedPtr hostsPtr() const PURE; + /** * @return all healthy hosts contained in the set at the current time. NOTE: This set is * eventually consistent. There is a time window where a host in this set may become @@ -283,6 +288,11 @@ class HostSet { */ virtual const HostVector& healthyHosts() const PURE; + /** + * @return a shared ptr to the vector returned by healthyHosts(). + */ + virtual HealthyHostVectorConstSharedPtr healthyHostsPtr() const PURE; + /** * @return all degraded hosts contained in the set at the current time. NOTE: This set is * eventually consistent. There is a time window where a host in this set may become @@ -291,32 +301,62 @@ class HostSet { */ virtual const HostVector& degradedHosts() const PURE; + /** + * @return a shared ptr to the vector returned by degradedHosts(). + */ + virtual DegradedHostVectorConstSharedPtr degradedHostsPtr() const PURE; + /* * @return all excluded hosts contained in the set at the current time. Excluded hosts should be * ignored when computing load balancing weights, but may overlap with hosts in hosts(). * */ virtual const HostVector& excludedHosts() const PURE; + /** + * @return a shared ptr to the vector returned by excludedHosts(). + */ + virtual ExcludedHostVectorConstSharedPtr excludedHostsPtr() const PURE; + /** * @return hosts per locality. */ virtual const HostsPerLocality& hostsPerLocality() const PURE; + /** + * @return a shared ptr to the HostsPerLocality returned by hostsPerLocality(). + */ + virtual HostsPerLocalityConstSharedPtr hostsPerLocalityPtr() const PURE; + /** * @return same as hostsPerLocality but only contains healthy hosts. */ virtual const HostsPerLocality& healthyHostsPerLocality() const PURE; + /** + * @return a shared ptr to the HostsPerLocality returned by healthyHostsPerLocality(). + */ + virtual HostsPerLocalityConstSharedPtr healthyHostsPerLocalityPtr() const PURE; + /** * @return same as hostsPerLocality but only contains degraded hosts. */ virtual const HostsPerLocality& degradedHostsPerLocality() const PURE; + /** + * @return a shared ptr to the HostsPerLocality returned by degradedHostsPerLocality(). + */ + virtual HostsPerLocalityConstSharedPtr degradedHostsPerLocalityPtr() const PURE; + /** * @return same as hostsPerLocality but only contains excluded hosts. */ virtual const HostsPerLocality& excludedHostsPerLocality() const PURE; + /** + * @return a shared ptr to the HostsPerLocality returned by excludedHostsPerLocality(). + */ + virtual HostsPerLocalityConstSharedPtr excludedHostsPerLocalityPtr() const PURE; + /** * @return weights for each locality in the host set. */ diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index 2f14631e113a5..1386ae3fad8d3 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -676,34 +676,13 @@ void ClusterManagerImpl::postThreadLocalClusterUpdate(const Cluster& cluster, ui const HostVector& hosts_removed) { const auto& host_set = cluster.prioritySet().hostSetsPerPriority()[priority]; - // TODO(htuch): Can we skip these copies by exporting out const shared_ptr from HostSet? - HostVectorConstSharedPtr hosts_copy(new HostVector(host_set->hosts())); - HealthyHostVectorConstSharedPtr healthy_hosts_copy( - new HealthyHostVector(host_set->healthyHosts())); - DegradedHostVectorConstSharedPtr degraded_hosts_copy( - new DegradedHostVector(host_set->degradedHosts())); - HostsPerLocalityConstSharedPtr hosts_per_locality_copy = host_set->hostsPerLocality().clone(); - HostsPerLocalityConstSharedPtr healthy_hosts_per_locality_copy = - host_set->healthyHostsPerLocality().clone(); - HostsPerLocalityConstSharedPtr degraded_hosts_per_locality_copy = - host_set->degradedHostsPerLocality().clone(); - ExcludedHostVectorConstSharedPtr excluded_hosts_copy( - new ExcludedHostVector(host_set->excludedHosts())); - auto excluded_hosts_per_locality_copy = host_set->excludedHostsPerLocality().clone(); - - tls_->runOnAllThreads([this, name = cluster.info()->name(), priority, hosts_copy, - healthy_hosts_copy, degraded_hosts_copy, hosts_per_locality_copy, - healthy_hosts_per_locality_copy, degraded_hosts_per_locality_copy, - excluded_hosts_copy, excluded_hosts_per_locality_copy, + tls_->runOnAllThreads([this, name = cluster.info()->name(), priority, + update_params = HostSetImpl::updateHostsParams(*host_set), locality_weights = host_set->localityWeights(), hosts_added, hosts_removed, overprovisioning_factor = host_set->overprovisioningFactor()]() { ThreadLocalClusterManagerImpl::updateClusterMembership( - name, priority, - HostSetImpl::updateHostsParams(hosts_copy, hosts_per_locality_copy, healthy_hosts_copy, - healthy_hosts_per_locality_copy, degraded_hosts_copy, - degraded_hosts_per_locality_copy, excluded_hosts_copy, - excluded_hosts_per_locality_copy), - locality_weights, hosts_added, hosts_removed, *tls_, overprovisioning_factor); + name, priority, update_params, locality_weights, hosts_added, hosts_removed, *tls_, + overprovisioning_factor); }); } @@ -972,8 +951,7 @@ void ClusterManagerImpl::ThreadLocalClusterManagerImpl::removeHosts(const std::s } void ClusterManagerImpl::ThreadLocalClusterManagerImpl::updateClusterMembership( - const std::string& name, uint32_t priority, - PrioritySet::UpdateHostsParams&& update_hosts_params, + const std::string& name, uint32_t priority, PrioritySet::UpdateHostsParams update_hosts_params, LocalityWeightsConstSharedPtr locality_weights, const HostVector& hosts_added, const HostVector& hosts_removed, ThreadLocal::Slot& tls, uint64_t overprovisioning_factor) { diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index 4cb0914ee20c4..2195538576250 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -326,7 +326,7 @@ class ClusterManagerImpl : public ClusterManager, Logger::Loggableget(); } + HealthyHostVectorConstSharedPtr healthyHostsPtr() const override { return healthy_hosts_; } const HostVector& degradedHosts() const override { return degraded_hosts_->get(); } + DegradedHostVectorConstSharedPtr degradedHostsPtr() const override { return degraded_hosts_; } const HostVector& excludedHosts() const override { return excluded_hosts_->get(); } + ExcludedHostVectorConstSharedPtr excludedHostsPtr() const override { return excluded_hosts_; } const HostsPerLocality& hostsPerLocality() const override { return *hosts_per_locality_; } + HostsPerLocalityConstSharedPtr hostsPerLocalityPtr() const override { + return hosts_per_locality_; + } const HostsPerLocality& healthyHostsPerLocality() const override { return *healthy_hosts_per_locality_; } + HostsPerLocalityConstSharedPtr healthyHostsPerLocalityPtr() const override { + return healthy_hosts_per_locality_; + } const HostsPerLocality& degradedHostsPerLocality() const override { return *degraded_hosts_per_locality_; } + HostsPerLocalityConstSharedPtr degradedHostsPerLocalityPtr() const override { + return degraded_hosts_per_locality_; + } const HostsPerLocality& excludedHostsPerLocality() const override { return *excluded_hosts_per_locality_; } + HostsPerLocalityConstSharedPtr excludedHostsPerLocalityPtr() const override { + return excluded_hosts_per_locality_; + } LocalityWeightsConstSharedPtr localityWeights() const override { return locality_weights_; } absl::optional chooseHealthyLocality() override; absl::optional chooseDegradedLocality() override; @@ -339,6 +355,7 @@ class HostSetImpl : public HostSet { HostsPerLocalityConstSharedPtr degraded_hosts_per_locality, ExcludedHostVectorConstSharedPtr excluded_hosts, HostsPerLocalityConstSharedPtr excluded_hosts_per_locality); + static PrioritySet::UpdateHostsParams updateHostsParams(const HostSet& host_set); static PrioritySet::UpdateHostsParams partitionHosts(HostVectorConstSharedPtr hosts, HostsPerLocalityConstSharedPtr hosts_per_locality); diff --git a/test/mocks/upstream/mocks.cc b/test/mocks/upstream/mocks.cc index 973c3d53983dc..f8c426e856c92 100644 --- a/test/mocks/upstream/mocks.cc +++ b/test/mocks/upstream/mocks.cc @@ -22,21 +22,37 @@ MockHostSet::MockHostSet(uint32_t priority, uint32_t overprovisioning_factor) : priority_(priority), overprovisioning_factor_(overprovisioning_factor) { ON_CALL(*this, priority()).WillByDefault(Return(priority_)); ON_CALL(*this, hosts()).WillByDefault(ReturnRef(hosts_)); + ON_CALL(*this, hostsPtr()).WillByDefault(Invoke([this]() { + return std::make_shared(hosts_); + })); ON_CALL(*this, healthyHosts()).WillByDefault(ReturnRef(healthy_hosts_)); + ON_CALL(*this, healthyHostsPtr()).WillByDefault(Invoke([this]() { + return std::make_shared(healthy_hosts_); + })); ON_CALL(*this, degradedHosts()).WillByDefault(ReturnRef(degraded_hosts_)); + ON_CALL(*this, degradedHostsPtr()).WillByDefault(Invoke([this]() { + return std::make_shared(degraded_hosts_); + })); ON_CALL(*this, excludedHosts()).WillByDefault(ReturnRef(excluded_hosts_)); + ON_CALL(*this, excludedHostsPtr()).WillByDefault(Invoke([this]() { + return std::make_shared(excluded_hosts_); + })); ON_CALL(*this, hostsPerLocality()).WillByDefault(Invoke([this]() -> const HostsPerLocality& { return *hosts_per_locality_; })); + ON_CALL(*this, hostsPerLocalityPtr()).WillByDefault(Return(hosts_per_locality_)); ON_CALL(*this, healthyHostsPerLocality()) .WillByDefault( Invoke([this]() -> const HostsPerLocality& { return *healthy_hosts_per_locality_; })); + ON_CALL(*this, healthyHostsPerLocalityPtr()).WillByDefault(Return(healthy_hosts_per_locality_)); ON_CALL(*this, degradedHostsPerLocality()) .WillByDefault( Invoke([this]() -> const HostsPerLocality& { return *degraded_hosts_per_locality_; })); + ON_CALL(*this, degradedHostsPerLocalityPtr()).WillByDefault(Return(degraded_hosts_per_locality_)); ON_CALL(*this, excludedHostsPerLocality()) .WillByDefault( Invoke([this]() -> const HostsPerLocality& { return *excluded_hosts_per_locality_; })); + ON_CALL(*this, excludedHostsPerLocalityPtr()).WillByDefault(Return(excluded_hosts_per_locality_)); ON_CALL(*this, localityWeights()).WillByDefault(Invoke([this]() -> LocalityWeightsConstSharedPtr { return locality_weights_; })); diff --git a/test/mocks/upstream/mocks.h b/test/mocks/upstream/mocks.h index 3bf8873ea8e97..a93e90b237728 100644 --- a/test/mocks/upstream/mocks.h +++ b/test/mocks/upstream/mocks.h @@ -50,13 +50,21 @@ class MockHostSet : public HostSet { // Upstream::HostSet MOCK_CONST_METHOD0(hosts, const HostVector&()); + MOCK_CONST_METHOD0(hostsPtr, HostVectorConstSharedPtr()); MOCK_CONST_METHOD0(healthyHosts, const HostVector&()); + MOCK_CONST_METHOD0(healthyHostsPtr, HealthyHostVectorConstSharedPtr()); MOCK_CONST_METHOD0(degradedHosts, const HostVector&()); + MOCK_CONST_METHOD0(degradedHostsPtr, DegradedHostVectorConstSharedPtr()); MOCK_CONST_METHOD0(excludedHosts, const HostVector&()); + MOCK_CONST_METHOD0(excludedHostsPtr, ExcludedHostVectorConstSharedPtr()); MOCK_CONST_METHOD0(hostsPerLocality, const HostsPerLocality&()); + MOCK_CONST_METHOD0(hostsPerLocalityPtr, HostsPerLocalityConstSharedPtr()); MOCK_CONST_METHOD0(healthyHostsPerLocality, const HostsPerLocality&()); + MOCK_CONST_METHOD0(healthyHostsPerLocalityPtr, HostsPerLocalityConstSharedPtr()); MOCK_CONST_METHOD0(degradedHostsPerLocality, const HostsPerLocality&()); + MOCK_CONST_METHOD0(degradedHostsPerLocalityPtr, HostsPerLocalityConstSharedPtr()); MOCK_CONST_METHOD0(excludedHostsPerLocality, const HostsPerLocality&()); + MOCK_CONST_METHOD0(excludedHostsPerLocalityPtr, HostsPerLocalityConstSharedPtr()); MOCK_CONST_METHOD0(localityWeights, LocalityWeightsConstSharedPtr()); MOCK_METHOD0(chooseHealthyLocality, absl::optional()); MOCK_METHOD0(chooseDegradedLocality, absl::optional()); From 5e688e22cdebdcb1f2ef108895eb677c856ad183 Mon Sep 17 00:00:00 2001 From: Snow Pettersen Date: Mon, 6 May 2019 12:43:24 -0400 Subject: [PATCH 17/28] remove warmed() function, check flag directly Signed-off-by: Snow Pettersen --- include/envoy/upstream/upstream.h | 2 -- source/common/upstream/upstream_impl.cc | 14 +++++++++----- source/common/upstream/upstream_impl.h | 7 ------- 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/include/envoy/upstream/upstream.h b/include/envoy/upstream/upstream.h index 21991fc8cf014..589b3432d1299 100644 --- a/include/envoy/upstream/upstream.h +++ b/include/envoy/upstream/upstream.h @@ -192,8 +192,6 @@ class Host : virtual public HostDescription { * @param new_used supplies the new value of host being in use to be stored. */ virtual void used(bool new_used) PURE; - - virtual bool warmed() const PURE; }; typedef std::shared_ptr HostConstSharedPtr; diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index 68cab959e4d42..564cfa5e9b5ee 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -735,7 +735,7 @@ ClusterImplBase::partitionHostList(const HostVector& hosts) { if (host->health() == Host::Health::Degraded) { degraded_list->get().emplace_back(host); } - if (!host->warmed()) { + if (host->healthFlagGet(Host::HealthFlag::PENDING_ACTIVE_HC)) { excluded_list->get().emplace_back(host); } } @@ -749,7 +749,7 @@ ClusterImplBase::partitionHostsPerLocality(const HostsPerLocality& hosts) { auto filtered_clones = hosts.filter({[](const Host& host) { return host.health() == Host::Health::Healthy; }, [](const Host& host) { return host.health() == Host::Health::Degraded; }, - [](const Host& host) { return !host.warmed(); }}); + [](const Host& host) { return host.healthFlagGet(Host::HealthFlag::PENDING_ACTIVE_HC); }}); return {std::move(filtered_clones[0]), std::move(filtered_clones[1]), std::move(filtered_clones[2])}; @@ -1230,11 +1230,15 @@ bool BaseDynamicClusterImpl::updateDynamicHostList(const HostVector& new_hosts, max_host_weight = host->weight(); } - // If we are depending on a health checker, we initialize to unhealthy and set the - // flag indicating that we have not performed any active health checks yet. + // If we are depending on a health checker, we initialize to unhealthy. if (health_checker_ != nullptr) { host->healthFlagSet(Host::HealthFlag::FAILED_ACTIVE_HC); - host->healthFlagSet(Host::HealthFlag::PENDING_ACTIVE_HC); + + // If we want to exclude hosts until they have been health checked, mark them with + // a flag to indicate that they have not been health checked yet. + if (info_->warmHosts()) { + host->healthFlagSet(Host::HealthFlag::PENDING_ACTIVE_HC); + } } updated_hosts[host->address()->asString()] = host; diff --git a/source/common/upstream/upstream_impl.h b/source/common/upstream/upstream_impl.h index ff2ddd3426612..36e87e82b197b 100644 --- a/source/common/upstream/upstream_impl.h +++ b/source/common/upstream/upstream_impl.h @@ -224,13 +224,6 @@ class HostImpl : public HostDescriptionImpl, void weight(uint32_t new_weight) override; bool used() const override { return used_; } void used(bool new_used) override { used_ = new_used; } - bool warmed() const override { - if (cluster_ != nullptr && cluster_->warmHosts()) { - return !healthFlagGet(HealthFlag::PENDING_ACTIVE_HC); - } - - return true; - } protected: static Network::ClientConnectionPtr From 8c332164afab47a646aef4c08475388d149527eb Mon Sep 17 00:00:00 2001 From: Snow Pettersen Date: Mon, 6 May 2019 14:05:31 -0400 Subject: [PATCH 18/28] add integration test + stats Signed-off-by: Snow Pettersen --- include/envoy/upstream/upstream.h | 1 + .../upstream/health_checker_base_impl.cc | 2 + source/common/upstream/upstream_impl.cc | 12 +++-- test/integration/eds_integration_test.cc | 53 +++++++++++++++++++ 4 files changed, 63 insertions(+), 5 deletions(-) diff --git a/include/envoy/upstream/upstream.h b/include/envoy/upstream/upstream.h index 589b3432d1299..e9f197f4209ce 100644 --- a/include/envoy/upstream/upstream.h +++ b/include/envoy/upstream/upstream.h @@ -575,6 +575,7 @@ class PrioritySet { COUNTER (membership_change) \ GAUGE (membership_healthy) \ GAUGE (membership_degraded) \ + GAUGE (membership_excluded) \ GAUGE (membership_total) \ COUNTER (retry_or_shadow_abandoned) \ COUNTER (update_attempt) \ diff --git a/source/common/upstream/health_checker_base_impl.cc b/source/common/upstream/health_checker_base_impl.cc index fd239241904e8..0d24e04f8d7d3 100644 --- a/source/common/upstream/health_checker_base_impl.cc +++ b/source/common/upstream/health_checker_base_impl.cc @@ -310,6 +310,8 @@ HealthTransition HealthCheckerImplBase::ActiveHealthCheckSession::setUnhealthy( if (host_->healthFlagGet(Host::HealthFlag::PENDING_ACTIVE_HC)) { host_->healthFlagClear(Host::HealthFlag::PENDING_ACTIVE_HC); + // Even though the health value of the host changed, we set this to Changed to that the cluster + // can update its list of excluded hosts. changed_state = HealthTransition::Changed; } diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index 8a909fdf989d8..a0b221e4e627b 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -708,16 +708,18 @@ ClusterImplBase::ClusterImplBase( uint32_t healthy_hosts = 0; uint32_t degraded_hosts = 0; + uint32_t excluded_hosts = 0; uint32_t hosts = 0; for (const auto& host_set : prioritySet().hostSetsPerPriority()) { hosts += host_set->hosts().size(); healthy_hosts += host_set->healthyHosts().size(); degraded_hosts += host_set->degradedHosts().size(); + excluded_hosts += host_set->excludedHosts().size(); } - // TODO(snowp): Stats for excluded hosts? info_->stats().membership_total_.set(hosts); info_->stats().membership_healthy_.set(healthy_hosts); info_->stats().membership_degraded_.set(degraded_hosts); + info_->stats().membership_excluded_.set(excluded_hosts); }); } @@ -746,10 +748,10 @@ ClusterImplBase::partitionHostList(const HostVector& hosts) { std::tuple ClusterImplBase::partitionHostsPerLocality(const HostsPerLocality& hosts) { - auto filtered_clones = - hosts.filter({[](const Host& host) { return host.health() == Host::Health::Healthy; }, - [](const Host& host) { return host.health() == Host::Health::Degraded; }, - [](const Host& host) { return host.healthFlagGet(Host::HealthFlag::PENDING_ACTIVE_HC); }}); + auto filtered_clones = hosts.filter( + {[](const Host& host) { return host.health() == Host::Health::Healthy; }, + [](const Host& host) { return host.health() == Host::Health::Degraded; }, + [](const Host& host) { return host.healthFlagGet(Host::HealthFlag::PENDING_ACTIVE_HC); }}); return {std::move(filtered_clones[0]), std::move(filtered_clones[1]), std::move(filtered_clones[2])}; diff --git a/test/integration/eds_integration_test.cc b/test/integration/eds_integration_test.cc index 4bcd9c2bf7c50..3a9ece0396ae9 100644 --- a/test/integration/eds_integration_test.cc +++ b/test/integration/eds_integration_test.cc @@ -108,6 +108,59 @@ TEST_P(EdsIntegrationTest, RemoveAfterHcFail) { EXPECT_EQ(0, test_server_->gauge("cluster.cluster_0.membership_total")->value()); } +// Verifies that endpoints are ignored until health checked when configured to. +TEST_P(EdsIntegrationTest, EndpointWarmingSuccessfulHc) { + config_helper_.addConfigModifier([this](envoy::config::bootstrap::v2::Bootstrap& bootstrap) { + // Switch predefined cluster_0 to EDS filesystem sourcing. + auto* cluster_0 = bootstrap.mutable_static_resources()->mutable_clusters(0); + cluster_0->mutable_common_lb_config()->set_ignore_new_hosts_until_first_hc(true); + }); + + // Endpoints are initially excluded. + initializeTest(true); + fake_upstreams_[0]->set_allow_unexpected_disconnects(true); + setEndpoints(1, 0, 0, false); + + EXPECT_EQ(1, test_server_->gauge("cluster.cluster_0.membership_total")->value()); + EXPECT_EQ(1, test_server_->gauge("cluster.cluster_0.membership_excluded")->value()); + EXPECT_EQ(0, test_server_->gauge("cluster.cluster_0.membership_healthy")->value()); + + // Wait for the first HC and verify the host is healthy and that it is no longer being excluded. + // The other endpoint should stil be excluded. + waitForNextUpstreamRequest(0); + upstream_request_->encodeHeaders(Http::TestHeaderMapImpl{{":status", "200"}}, true); + test_server_->waitForGaugeEq("cluster.cluster_0.membership_excluded", 0); + EXPECT_EQ(1, test_server_->gauge("cluster.cluster_0.membership_total")->value()); + EXPECT_EQ(1, test_server_->gauge("cluster.cluster_0.membership_healthy")->value()); +} + +// Verifies that endpoints are ignored until health checked when configured to when the first +// health check fails. +TEST_P(EdsIntegrationTest, EndpointWarmingFailedHc) { + config_helper_.addConfigModifier([this](envoy::config::bootstrap::v2::Bootstrap& bootstrap) { + // Switch predefined cluster_0 to EDS filesystem sourcing. + auto* cluster_0 = bootstrap.mutable_static_resources()->mutable_clusters(0); + cluster_0->mutable_common_lb_config()->set_ignore_new_hosts_until_first_hc(true); + }); + + // Endpoints are initially excluded. + initializeTest(true); + fake_upstreams_[0]->set_allow_unexpected_disconnects(true); + setEndpoints(1, 0, 0, false); + + EXPECT_EQ(1, test_server_->gauge("cluster.cluster_0.membership_total")->value()); + EXPECT_EQ(1, test_server_->gauge("cluster.cluster_0.membership_excluded")->value()); + EXPECT_EQ(0, test_server_->gauge("cluster.cluster_0.membership_healthy")->value()); + + // Wait for the first HC and verify the host is healthy and that it is no longer being excluded. + // The other endpoint should stil be excluded. + waitForNextUpstreamRequest(0); + upstream_request_->encodeHeaders(Http::TestHeaderMapImpl{{":status", "503"}}, true); + test_server_->waitForGaugeEq("cluster.cluster_0.membership_excluded", 0); + EXPECT_EQ(1, test_server_->gauge("cluster.cluster_0.membership_total")->value()); + EXPECT_EQ(0, test_server_->gauge("cluster.cluster_0.membership_healthy")->value()); +} + // Validate that health status updates are consumed from EDS. TEST_P(EdsIntegrationTest, HealthUpdate) { initializeTest(false); From ab918c3779b639fc872dd61dae081301fe883c66 Mon Sep 17 00:00:00 2001 From: Snow Pettersen Date: Mon, 6 May 2019 16:13:19 -0400 Subject: [PATCH 19/28] add UTs for panic Signed-off-by: Snow Pettersen --- source/common/upstream/load_balancer_impl.cc | 1 + .../upstream/load_balancer_impl_test.cc | 30 +++++++++++++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/source/common/upstream/load_balancer_impl.cc b/source/common/upstream/load_balancer_impl.cc index 9c89f994dcdc6..ea3dad58b5fb5 100644 --- a/source/common/upstream/load_balancer_impl.cc +++ b/source/common/upstream/load_balancer_impl.cc @@ -136,6 +136,7 @@ void LoadBalancerBase::recalculatePerPriorityState(uint32_t priority, // by the overprovisioning factor. HostSet& host_set = *priority_set.hostSetsPerPriority()[priority]; per_priority_health.get()[priority] = 0; + per_priority_degraded.get()[priority] = 0; const auto host_count = host_set.hosts().size() - host_set.excludedHosts().size(); if (host_count > 0) { diff --git a/test/common/upstream/load_balancer_impl_test.cc b/test/common/upstream/load_balancer_impl_test.cc index 2558fbdfbbc80..ec85b87d307f1 100644 --- a/test/common/upstream/load_balancer_impl_test.cc +++ b/test/common/upstream/load_balancer_impl_test.cc @@ -65,12 +65,13 @@ class TestLb : public LoadBalancerBase { class LoadBalancerBaseTest : public LoadBalancerTestBase { public: void updateHostSet(MockHostSet& host_set, uint32_t num_hosts, uint32_t num_healthy_hosts, - uint32_t num_degraded_hosts = 0) { - ASSERT(num_healthy_hosts + num_degraded_hosts <= num_hosts); + uint32_t num_degraded_hosts = 0, uint32_t num_excluded_hosts = 0) { + ASSERT(num_healthy_hosts + num_degraded_hosts + num_excluded_hosts <= num_hosts); host_set.hosts_.clear(); host_set.healthy_hosts_.clear(); host_set.degraded_hosts_.clear(); + host_set.excluded_hosts_.clear(); for (uint32_t i = 0; i < num_hosts; ++i) { host_set.hosts_.push_back(makeTestHost(info_, "tcp://127.0.0.1:80")); } @@ -78,9 +79,14 @@ class LoadBalancerBaseTest : public LoadBalancerTestBase { for (; i < num_healthy_hosts; ++i) { host_set.healthy_hosts_.push_back(host_set.hosts_[i]); } - for (; i < num_degraded_hosts; ++i) { + for (; i < (num_healthy_hosts + num_degraded_hosts); ++i) { host_set.degraded_hosts_.push_back(host_set.hosts_[i]); } + + for (; i < (num_healthy_hosts + num_degraded_hosts + num_excluded_hosts); ++i) { + host_set.degraded_hosts_.push_back(host_set.hosts_[i]); + host_set.excluded_hosts_.push_back(host_set.hosts_[i]); + } host_set.runCallbacks({}, {}); } @@ -283,6 +289,24 @@ TEST_P(LoadBalancerBaseTest, GentleFailover) { updateHostSet(failover_host_set_, 4 /* num_hosts */, 1 /* num_healthy_hosts */); ASSERT_THAT(getLoadPercentage(), ElementsAre(50, 50)); ASSERT_THAT(getPanic(), ElementsAre(true, true)); + + // Health P=0 == 100*1.4 == 35 P=1 == 35 + // Since 3 hosts are excluded, P=0 should be considered fully healthy. + // Total health = 100% + 35% is greater than 100%. Panic should not trigger. + updateHostSet(host_set_, 4 /* num_hosts */, 1 /* num_healthy_hosts */, 0 /* num_degraded_hosts */, + 3 /* num_excluded_hosts */); + updateHostSet(failover_host_set_, 5 /* num_hosts */, 1 /* num_healthy_hosts */); + ASSERT_THAT(getLoadPercentage(), ElementsAre(100, 0)); + ASSERT_THAT(getPanic(), ElementsAre(false, false)); + + // Health P=0 == 100*1.4 == 35 P=1 == 35 + // Since 4 hosts are excluded and are unhealthy, P=0 should be considered fully unavailable. + // Total health = 35% is less than 100%. Panic should trigger. + updateHostSet(host_set_, 4 /* num_hosts */, 0 /* num_healthy_hosts */, 0 /* num_degraded_hosts */, + 4 /* num_excluded_hosts */); + updateHostSet(failover_host_set_, 4 /* num_hosts */, 1 /* num_healthy_hosts */); + ASSERT_THAT(getLoadPercentage(), ElementsAre(0, 100)); + ASSERT_THAT(getPanic(), ElementsAre(true, true)); } TEST_P(LoadBalancerBaseTest, GentleFailoverWithExtraLevels) { From 7950e0b57e5cead104c6d534202b6022710c1453 Mon Sep 17 00:00:00 2001 From: Snow Pettersen Date: Mon, 6 May 2019 17:08:27 -0400 Subject: [PATCH 20/28] move params helpers to test + other nits Signed-off-by: Snow Pettersen --- api/envoy/api/v2/cds.proto | 4 +- include/envoy/upstream/upstream.h | 2 +- source/common/upstream/upstream_impl.cc | 22 ------ source/common/upstream/upstream_impl.h | 9 --- .../upstream/cluster_manager_impl_test.cc | 70 +++++++----------- .../upstream/load_balancer_impl_test.cc | 5 +- test/common/upstream/upstream_impl_test.cc | 74 +++++++++---------- test/common/upstream/utility.h | 18 +++++ test/integration/eds_integration_test.cc | 2 +- 9 files changed, 88 insertions(+), 118 deletions(-) diff --git a/api/envoy/api/v2/cds.proto b/api/envoy/api/v2/cds.proto index 18096506bf37b..74ae29fb11b64 100644 --- a/api/envoy/api/v2/cds.proto +++ b/api/envoy/api/v2/cds.proto @@ -532,8 +532,8 @@ message Cluster { // active health checking is also configured. // // Ignoring a host means that for any load balancing calculations that adjust weights based - // on the ratio of eligible hosts and total hosts (priority spillover, locality weighting, etc.) - // will exclude these hosts in the denominator. + // on the ratio of eligible hosts and total hosts (priority spillover, locality weighting and + // panic mode) Envoy will exclude these hosts in the denominator. // // For example, with hosts in two priorities P0 and P1, where P0 looks like // {healthy, unhealthy (new), unhealthy (new)} diff --git a/include/envoy/upstream/upstream.h b/include/envoy/upstream/upstream.h index e9f197f4209ce..0c503187a097c 100644 --- a/include/envoy/upstream/upstream.h +++ b/include/envoy/upstream/upstream.h @@ -307,7 +307,7 @@ class HostSet { /* * @return all excluded hosts contained in the set at the current time. Excluded hosts should be * ignored when computing load balancing weights, but may overlap with hosts in hosts(). - * */ + */ virtual const HostVector& excludedHosts() const PURE; /** diff --git a/source/common/upstream/upstream_impl.cc b/source/common/upstream/upstream_impl.cc index a0b221e4e627b..9cf1c843bd11c 100644 --- a/source/common/upstream/upstream_impl.cc +++ b/source/common/upstream/upstream_impl.cc @@ -404,28 +404,6 @@ HostSetImpl::chooseLocality(EdfScheduler* locality_scheduler) { return locality->index_; } -PrioritySet::UpdateHostsParams -HostSetImpl::updateHostsParams(HostVectorConstSharedPtr hosts, - HostsPerLocalityConstSharedPtr hosts_per_locality) { - return updateHostsParams(std::move(hosts), std::move(hosts_per_locality), - std::make_shared(), - HostsPerLocalityImpl::empty()); -} - -PrioritySet::UpdateHostsParams -HostSetImpl::updateHostsParams(HostVectorConstSharedPtr hosts, - HostsPerLocalityConstSharedPtr hosts_per_locality, - HealthyHostVectorConstSharedPtr healthy_hosts, - HostsPerLocalityConstSharedPtr healthy_hosts_per_locality) { - // TODO(snowp): Move this function into test/ - // This overload is only used as a convenience method in test, so it should live there. - - return updateHostsParams( - hosts, hosts_per_locality, std::move(healthy_hosts), std::move(healthy_hosts_per_locality), - std::make_shared(), HostsPerLocalityImpl::empty(), - std::make_shared(), HostsPerLocalityImpl::empty()); -} - PrioritySet::UpdateHostsParams HostSetImpl::updateHostsParams(HostVectorConstSharedPtr hosts, HostsPerLocalityConstSharedPtr hosts_per_locality, diff --git a/source/common/upstream/upstream_impl.h b/source/common/upstream/upstream_impl.h index aeabbf921ec34..5766c835aae65 100644 --- a/source/common/upstream/upstream_impl.h +++ b/source/common/upstream/upstream_impl.h @@ -330,15 +330,6 @@ class HostSetImpl : public HostSet { uint32_t priority() const override { return priority_; } uint32_t overprovisioningFactor() const override { return overprovisioning_factor_; } - // Utility methods for creating UpdateHostsParams. - static PrioritySet::UpdateHostsParams - updateHostsParams(HostVectorConstSharedPtr hosts, - HostsPerLocalityConstSharedPtr hosts_per_locality); - static PrioritySet::UpdateHostsParams - updateHostsParams(HostVectorConstSharedPtr hosts, - HostsPerLocalityConstSharedPtr hosts_per_locality, - HealthyHostVectorConstSharedPtr healthy_hosts, - HostsPerLocalityConstSharedPtr healthy_hosts_per_locality); static PrioritySet::UpdateHostsParams updateHostsParams(HostVectorConstSharedPtr hosts, HostsPerLocalityConstSharedPtr hosts_per_locality, diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index 5647c8d8a6e24..58edddda177db 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -2174,9 +2174,8 @@ TEST_F(ClusterManagerImplTest, MergedUpdates) { hosts_removed.push_back((*hosts)[0]); cluster.prioritySet().updateHosts( 0, - HostSetImpl::updateHostsParams(hosts, hosts_per_locality, - std::make_shared(*hosts), - hosts_per_locality), + updateHostsParams(hosts, hosts_per_locality, + std::make_shared(*hosts), hosts_per_locality), {}, hosts_added, hosts_removed, absl::nullopt); EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value()); @@ -2186,15 +2185,13 @@ TEST_F(ClusterManagerImplTest, MergedUpdates) { hosts_removed.clear(); cluster.prioritySet().updateHosts( 0, - HostSetImpl::updateHostsParams(hosts, hosts_per_locality, - std::make_shared(*hosts), - hosts_per_locality), + updateHostsParams(hosts, hosts_per_locality, + std::make_shared(*hosts), hosts_per_locality), {}, hosts_added, hosts_removed, absl::nullopt); cluster.prioritySet().updateHosts( 0, - HostSetImpl::updateHostsParams(hosts, hosts_per_locality, - std::make_shared(*hosts), - hosts_per_locality), + updateHostsParams(hosts, hosts_per_locality, + std::make_shared(*hosts), hosts_per_locality), {}, hosts_added, hosts_removed, absl::nullopt); EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value()); @@ -2211,9 +2208,8 @@ TEST_F(ClusterManagerImplTest, MergedUpdates) { hosts_added.push_back((*hosts)[0]); cluster.prioritySet().updateHosts( 0, - HostSetImpl::updateHostsParams(hosts, hosts_per_locality, - std::make_shared(*hosts), - hosts_per_locality), + updateHostsParams(hosts, hosts_per_locality, + std::make_shared(*hosts), hosts_per_locality), {}, hosts_added, hosts_removed, absl::nullopt); EXPECT_EQ(2, factory_.stats_.counter("cluster_manager.cluster_updated").value()); EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value()); @@ -2225,25 +2221,22 @@ TEST_F(ClusterManagerImplTest, MergedUpdates) { (*hosts)[0]->metadata(buildMetadata("v1")); cluster.prioritySet().updateHosts( 0, - HostSetImpl::updateHostsParams(hosts, hosts_per_locality, - std::make_shared(*hosts), - hosts_per_locality), + updateHostsParams(hosts, hosts_per_locality, + std::make_shared(*hosts), hosts_per_locality), {}, hosts_added, hosts_removed, absl::nullopt); (*hosts)[0]->healthFlagSet(Host::HealthFlag::FAILED_EDS_HEALTH); cluster.prioritySet().updateHosts( 0, - HostSetImpl::updateHostsParams(hosts, hosts_per_locality, - std::make_shared(*hosts), - hosts_per_locality), + updateHostsParams(hosts, hosts_per_locality, + std::make_shared(*hosts), hosts_per_locality), {}, hosts_added, hosts_removed, absl::nullopt); (*hosts)[0]->weight(100); cluster.prioritySet().updateHosts( 0, - HostSetImpl::updateHostsParams(hosts, hosts_per_locality, - std::make_shared(*hosts), - hosts_per_locality), + updateHostsParams(hosts, hosts_per_locality, + std::make_shared(*hosts), hosts_per_locality), {}, hosts_added, hosts_removed, absl::nullopt); // Updates not delivered yet. @@ -2255,9 +2248,8 @@ TEST_F(ClusterManagerImplTest, MergedUpdates) { hosts_removed.push_back((*hosts)[0]); cluster.prioritySet().updateHosts( 0, - HostSetImpl::updateHostsParams(hosts, hosts_per_locality, - std::make_shared(*hosts), - hosts_per_locality), + updateHostsParams(hosts, hosts_per_locality, + std::make_shared(*hosts), hosts_per_locality), {}, hosts_added, hosts_removed, absl::nullopt); EXPECT_EQ(3, factory_.stats_.counter("cluster_manager.cluster_updated").value()); @@ -2292,9 +2284,8 @@ TEST_F(ClusterManagerImplTest, MergedUpdatesOutOfWindow) { time_system_.sleep(std::chrono::seconds(60)); cluster.prioritySet().updateHosts( 0, - HostSetImpl::updateHostsParams(hosts, hosts_per_locality, - std::make_shared(*hosts), - hosts_per_locality), + updateHostsParams(hosts, hosts_per_locality, + std::make_shared(*hosts), hosts_per_locality), {}, hosts_added, hosts_removed, absl::nullopt); EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value()); @@ -2320,9 +2311,8 @@ TEST_F(ClusterManagerImplTest, MergedUpdatesInsideWindow) { time_system_.sleep(std::chrono::seconds(2)); cluster.prioritySet().updateHosts( 0, - HostSetImpl::updateHostsParams(hosts, hosts_per_locality, - std::make_shared(*hosts), - hosts_per_locality), + updateHostsParams(hosts, hosts_per_locality, + std::make_shared(*hosts), hosts_per_locality), {}, hosts_added, hosts_removed, absl::nullopt); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value()); @@ -2356,9 +2346,8 @@ TEST_F(ClusterManagerImplTest, MergedUpdatesOutOfWindowDisabled) { // and outside a merge window, merging is disabled. cluster.prioritySet().updateHosts( 0, - HostSetImpl::updateHostsParams(hosts, hosts_per_locality, - std::make_shared(*hosts), - hosts_per_locality), + updateHostsParams(hosts, hosts_per_locality, + std::make_shared(*hosts), hosts_per_locality), {}, hosts_added, hosts_removed, absl::nullopt); EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value()); @@ -2423,9 +2412,8 @@ TEST_F(ClusterManagerImplTest, MergedUpdatesDestroyedOnUpdate) { hosts_removed.push_back((*hosts)[0]); cluster.prioritySet().updateHosts( 0, - HostSetImpl::updateHostsParams(hosts, hosts_per_locality, - std::make_shared(*hosts), - hosts_per_locality), + updateHostsParams(hosts, hosts_per_locality, + std::make_shared(*hosts), hosts_per_locality), {}, hosts_added, hosts_removed, absl::nullopt); EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value()); @@ -2435,15 +2423,13 @@ TEST_F(ClusterManagerImplTest, MergedUpdatesDestroyedOnUpdate) { hosts_removed.clear(); cluster.prioritySet().updateHosts( 0, - HostSetImpl::updateHostsParams(hosts, hosts_per_locality, - std::make_shared(*hosts), - hosts_per_locality), + updateHostsParams(hosts, hosts_per_locality, + std::make_shared(*hosts), hosts_per_locality), {}, hosts_added, hosts_removed, absl::nullopt); cluster.prioritySet().updateHosts( 0, - HostSetImpl::updateHostsParams(hosts, hosts_per_locality, - std::make_shared(*hosts), - hosts_per_locality), + updateHostsParams(hosts, hosts_per_locality, + std::make_shared(*hosts), hosts_per_locality), {}, hosts_added, hosts_removed, absl::nullopt); EXPECT_EQ(1, factory_.stats_.counter("cluster_manager.cluster_updated").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value()); diff --git a/test/common/upstream/load_balancer_impl_test.cc b/test/common/upstream/load_balancer_impl_test.cc index ec85b87d307f1..4c93285502562 100644 --- a/test/common/upstream/load_balancer_impl_test.cc +++ b/test/common/upstream/load_balancer_impl_test.cc @@ -464,9 +464,8 @@ class RoundRobinLoadBalancerTest : public LoadBalancerTestBase { HostsPerLocalityConstSharedPtr hosts_per_locality) { local_priority_set_->updateHosts( 0, - HostSetImpl::updateHostsParams(hosts, hosts_per_locality, - std::make_shared(*hosts), - hosts_per_locality), + updateHostsParams(hosts, hosts_per_locality, + std::make_shared(*hosts), hosts_per_locality), {}, empty_host_vector_, empty_host_vector_, absl::nullopt); } diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc index 0aa858c200876..f72e8207b6fc7 100644 --- a/test/common/upstream/upstream_impl_test.cc +++ b/test/common/upstream/upstream_impl_test.cc @@ -1663,9 +1663,9 @@ class TestBatchUpdateCb : public PrioritySet::BatchUpdateCb { HostVector hosts_removed{}; host_update_cb.updateHosts( 0, - HostSetImpl::updateHostsParams(hosts_, hosts_per_locality_, - std::make_shared(*hosts_), - hosts_per_locality_), + updateHostsParams(hosts_, hosts_per_locality_, + std::make_shared(*hosts_), + hosts_per_locality_), {}, hosts_added, hosts_removed, absl::nullopt); } @@ -1676,9 +1676,9 @@ class TestBatchUpdateCb : public PrioritySet::BatchUpdateCb { HostVector hosts_removed{hosts_->front()}; host_update_cb.updateHosts( 1, - HostSetImpl::updateHostsParams(empty_hosts, HostsPerLocalityImpl::empty(), - std::make_shared(*empty_hosts), - HostsPerLocalityImpl::empty()), + updateHostsParams(empty_hosts, HostsPerLocalityImpl::empty(), + std::make_shared(*empty_hosts), + HostsPerLocalityImpl::empty()), {}, hosts_added, hosts_removed, absl::nullopt); } } @@ -1729,12 +1729,11 @@ TEST(PrioritySet, Extend) { HostVector hosts_added{hosts->front()}; HostVector hosts_removed{}; - priority_set.updateHosts( - 1, - HostSetImpl::updateHostsParams(hosts, hosts_per_locality, - std::make_shared(*hosts), - hosts_per_locality), - {}, hosts_added, hosts_removed, absl::nullopt); + priority_set.updateHosts(1, + updateHostsParams(hosts, hosts_per_locality, + std::make_shared(*hosts), + hosts_per_locality), + {}, hosts_added, hosts_removed, absl::nullopt); } EXPECT_EQ(1, priority_changes); EXPECT_EQ(1, membership_changes); @@ -2305,8 +2304,8 @@ TEST_F(HostSetImplLocalityTest, AllUnhealthy) { makeHostsPerLocality({{hosts_[0]}, {hosts_[1]}, {hosts_[2]}}); LocalityWeightsConstSharedPtr locality_weights{new LocalityWeights{1, 1, 1}}; auto hosts = makeHostsFromHostsPerLocality(hosts_per_locality); - host_set_.updateHosts(HostSetImpl::updateHostsParams(hosts, hosts_per_locality), locality_weights, - {}, {}, absl::nullopt); + host_set_.updateHosts(updateHostsParams(hosts, hosts_per_locality), locality_weights, {}, {}, + absl::nullopt); EXPECT_FALSE(host_set_.chooseHealthyLocality().has_value()); } @@ -2346,9 +2345,9 @@ TEST_F(HostSetImplLocalityTest, EmptyLocality) { makeHostsPerLocality({{hosts_[0], hosts_[1], hosts_[2]}, {}}); LocalityWeightsConstSharedPtr locality_weights{new LocalityWeights{1, 1}}; auto hosts = makeHostsFromHostsPerLocality(hosts_per_locality); - host_set_.updateHosts(HostSetImpl::updateHostsParams( - hosts, hosts_per_locality, - std::make_shared(*hosts), hosts_per_locality), + host_set_.updateHosts(updateHostsParams(hosts, hosts_per_locality, + std::make_shared(*hosts), + hosts_per_locality), locality_weights, {}, {}, absl::nullopt); // Verify that we are not RRing between localities. EXPECT_EQ(0, host_set_.chooseHealthyLocality().value()); @@ -2360,9 +2359,9 @@ TEST_F(HostSetImplLocalityTest, AllZeroWeights) { HostsPerLocalitySharedPtr hosts_per_locality = makeHostsPerLocality({{hosts_[0]}, {hosts_[1]}}); LocalityWeightsConstSharedPtr locality_weights{new LocalityWeights{0, 0}}; auto hosts = makeHostsFromHostsPerLocality(hosts_per_locality); - host_set_.updateHosts(HostSetImpl::updateHostsParams( - hosts, hosts_per_locality, - std::make_shared(*hosts), hosts_per_locality), + host_set_.updateHosts(updateHostsParams(hosts, hosts_per_locality, + std::make_shared(*hosts), + hosts_per_locality), locality_weights, {}, {}); EXPECT_FALSE(host_set_.chooseHealthyLocality().has_value()); } @@ -2373,9 +2372,9 @@ TEST_F(HostSetImplLocalityTest, Unweighted) { makeHostsPerLocality({{hosts_[0]}, {hosts_[1]}, {hosts_[2]}}); LocalityWeightsConstSharedPtr locality_weights{new LocalityWeights{1, 1, 1}}; auto hosts = makeHostsFromHostsPerLocality(hosts_per_locality); - host_set_.updateHosts(HostSetImpl::updateHostsParams( - hosts, hosts_per_locality, - std::make_shared(*hosts), hosts_per_locality), + host_set_.updateHosts(updateHostsParams(hosts, hosts_per_locality, + std::make_shared(*hosts), + hosts_per_locality), locality_weights, {}, {}, absl::nullopt); EXPECT_EQ(0, host_set_.chooseHealthyLocality().value()); EXPECT_EQ(1, host_set_.chooseHealthyLocality().value()); @@ -2390,9 +2389,9 @@ TEST_F(HostSetImplLocalityTest, Weighted) { HostsPerLocalitySharedPtr hosts_per_locality = makeHostsPerLocality({{hosts_[0]}, {hosts_[1]}}); LocalityWeightsConstSharedPtr locality_weights{new LocalityWeights{1, 2}}; auto hosts = makeHostsFromHostsPerLocality(hosts_per_locality); - host_set_.updateHosts(HostSetImpl::updateHostsParams( - hosts, hosts_per_locality, - std::make_shared(*hosts), hosts_per_locality), + host_set_.updateHosts(updateHostsParams(hosts, hosts_per_locality, + std::make_shared(*hosts), + hosts_per_locality), locality_weights, {}, {}, absl::nullopt); EXPECT_EQ(1, host_set_.chooseHealthyLocality().value()); EXPECT_EQ(0, host_set_.chooseHealthyLocality().value()); @@ -2408,9 +2407,9 @@ TEST_F(HostSetImplLocalityTest, MissingWeight) { makeHostsPerLocality({{hosts_[0]}, {hosts_[1]}, {hosts_[2]}}); LocalityWeightsConstSharedPtr locality_weights{new LocalityWeights{1, 0, 1}}; auto hosts = makeHostsFromHostsPerLocality(hosts_per_locality); - host_set_.updateHosts(HostSetImpl::updateHostsParams( - hosts, hosts_per_locality, - std::make_shared(*hosts), hosts_per_locality), + host_set_.updateHosts(updateHostsParams(hosts, hosts_per_locality, + std::make_shared(*hosts), + hosts_per_locality), locality_weights, {}, {}, absl::nullopt); EXPECT_EQ(0, host_set_.chooseHealthyLocality().value()); EXPECT_EQ(2, host_set_.chooseHealthyLocality().value()); @@ -2434,12 +2433,11 @@ TEST_F(HostSetImplLocalityTest, UnhealthyFailover) { makeHostsPerLocality({healthy_hosts, {hosts_[5]}}); auto hosts = makeHostsFromHostsPerLocality(hosts_per_locality); - host_set_.updateHosts( - HostSetImpl::updateHostsParams( - hosts, hosts_per_locality, - makeHostsFromHostsPerLocality(healthy_hosts_per_locality), - healthy_hosts_per_locality), - locality_weights, {}, {}, absl::nullopt); + host_set_.updateHosts(updateHostsParams(hosts, hosts_per_locality, + makeHostsFromHostsPerLocality( + healthy_hosts_per_locality), + healthy_hosts_per_locality), + locality_weights, {}, {}, absl::nullopt); }; const auto expectPicks = [this](uint32_t locality_0_picks, uint32_t locality_1_picks) { @@ -2484,9 +2482,9 @@ TEST(OverProvisioningFactorTest, LocalityPickChanges) { makeHostsPerLocality({{hosts[0]}, {hosts[2]}}); auto healthy_hosts = makeHostsFromHostsPerLocality(healthy_hosts_per_locality); - host_set.updateHosts(HostSetImpl::updateHostsParams(std::make_shared(hosts), - hosts_per_locality, healthy_hosts, - healthy_hosts_per_locality), + host_set.updateHosts(updateHostsParams(std::make_shared(hosts), + hosts_per_locality, healthy_hosts, + healthy_hosts_per_locality), locality_weights, {}, {}, absl::nullopt); uint32_t cnts[] = {0, 0}; for (uint32_t i = 0; i < 100; ++i) { diff --git a/test/common/upstream/utility.h b/test/common/upstream/utility.h index d7f03be8c6800..b5ef983aab6dc 100644 --- a/test/common/upstream/utility.h +++ b/test/common/upstream/utility.h @@ -134,6 +134,24 @@ parseHealthCheckFromV1Json(const std::string& json_string) { return health_check; } +inline PrioritySet::UpdateHostsParams +updateHostsParams(HostVectorConstSharedPtr hosts, HostsPerLocalityConstSharedPtr hosts_per_locality, + HealthyHostVectorConstSharedPtr healthy_hosts, + HostsPerLocalityConstSharedPtr healthy_hosts_per_locality) { + return HostSetImpl::updateHostsParams( + hosts, hosts_per_locality, std::move(healthy_hosts), std::move(healthy_hosts_per_locality), + std::make_shared(), HostsPerLocalityImpl::empty(), + std::make_shared(), HostsPerLocalityImpl::empty()); +} + +inline PrioritySet::UpdateHostsParams +updateHostsParams(HostVectorConstSharedPtr hosts, + HostsPerLocalityConstSharedPtr hosts_per_locality) { + return updateHostsParams(std::move(hosts), std::move(hosts_per_locality), + std::make_shared(), + HostsPerLocalityImpl::empty()); +} + } // namespace } // namespace Upstream } // namespace Envoy diff --git a/test/integration/eds_integration_test.cc b/test/integration/eds_integration_test.cc index 3a9ece0396ae9..4aebd48832124 100644 --- a/test/integration/eds_integration_test.cc +++ b/test/integration/eds_integration_test.cc @@ -137,7 +137,7 @@ TEST_P(EdsIntegrationTest, EndpointWarmingSuccessfulHc) { // Verifies that endpoints are ignored until health checked when configured to when the first // health check fails. TEST_P(EdsIntegrationTest, EndpointWarmingFailedHc) { - config_helper_.addConfigModifier([this](envoy::config::bootstrap::v2::Bootstrap& bootstrap) { + config_helper_.addConfigModifier([](envoy::config::bootstrap::v2::Bootstrap& bootstrap) { // Switch predefined cluster_0 to EDS filesystem sourcing. auto* cluster_0 = bootstrap.mutable_static_resources()->mutable_clusters(0); cluster_0->mutable_common_lb_config()->set_ignore_new_hosts_until_first_hc(true); From 510a7a9dd1153b2aafbcb65a0affa2df9eb7c7be Mon Sep 17 00:00:00 2001 From: Snow Pettersen Date: Mon, 6 May 2019 17:44:05 -0400 Subject: [PATCH 21/28] add UT for partitionHosts Signed-off-by: Snow Pettersen --- test/common/upstream/upstream_impl_test.cc | 40 ++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc index f72e8207b6fc7..c5139a4a61aaa 100644 --- a/test/common/upstream/upstream_impl_test.cc +++ b/test/common/upstream/upstream_impl_test.cc @@ -2509,6 +2509,46 @@ TEST(OverProvisioningFactorTest, LocalityPickChanges) { setUpHostSetWithOPFAndTestPicks(200, 50, 50); }; +// Verifies that partionHosts correctly splits hosts based on their health flags. +TEST(HostPartitionTest, PartitionHosts) { + std::shared_ptr info{new NiceMock()}; + HostVector hosts{ + makeTestHost(info, "tcp://127.0.0.1:80"), makeTestHost(info, "tcp://127.0.0.1:81"), + makeTestHost(info, "tcp://127.0.0.1:82"), makeTestHost(info, "tcp://127.0.0.1:83")}; + + hosts[0]->healthFlagSet(Host::HealthFlag::FAILED_ACTIVE_HC); + hosts[1]->healthFlagSet(Host::HealthFlag::DEGRADED_ACTIVE_HC); + hosts[2]->healthFlagSet(Host::HealthFlag::PENDING_ACTIVE_HC); + hosts[2]->healthFlagSet(Host::HealthFlag::FAILED_ACTIVE_HC); + + auto hosts_per_locality = makeHostsPerLocality({{hosts[0], hosts[1]}, {hosts[2], hosts[3]}}); + + auto update_hosts_params = + HostSetImpl::partitionHosts(std::make_shared(hosts), hosts_per_locality); + + EXPECT_EQ(4, update_hosts_params.hosts->size()); + EXPECT_EQ(1, update_hosts_params.healthy_hosts->get().size()); + EXPECT_EQ(hosts[3], update_hosts_params.healthy_hosts->get()[0]); + EXPECT_EQ(1, update_hosts_params.degraded_hosts->get().size()); + EXPECT_EQ(hosts[1], update_hosts_params.degraded_hosts->get()[0]); + EXPECT_EQ(1, update_hosts_params.excluded_hosts->get().size()); + EXPECT_EQ(hosts[2], update_hosts_params.excluded_hosts->get()[0]); + + EXPECT_EQ(2, update_hosts_params.hosts_per_locality->get()[0].size()); + EXPECT_EQ(2, update_hosts_params.hosts_per_locality->get()[1].size()); + + EXPECT_EQ(0, update_hosts_params.healthy_hosts_per_locality->get()[0].size()); + EXPECT_EQ(1, update_hosts_params.healthy_hosts_per_locality->get()[1].size()); + EXPECT_EQ(hosts[3], update_hosts_params.healthy_hosts_per_locality->get()[1][0]); + + EXPECT_EQ(1, update_hosts_params.degraded_hosts_per_locality->get()[0].size()); + EXPECT_EQ(0, update_hosts_params.degraded_hosts_per_locality->get()[1].size()); + EXPECT_EQ(hosts[1], update_hosts_params.degraded_hosts_per_locality->get()[0][0]); + + EXPECT_EQ(0, update_hosts_params.excluded_hosts_per_locality->get()[0].size()); + EXPECT_EQ(1, update_hosts_params.excluded_hosts_per_locality->get()[1].size()); + EXPECT_EQ(hosts[2], update_hosts_params.excluded_hosts_per_locality->get()[1][0]); +} } // namespace } // namespace Upstream } // namespace Envoy From c89bb7b48de2f0e6fd6dd1a59837409ecd9eb1f3 Mon Sep 17 00:00:00 2001 From: Snow Pettersen Date: Mon, 6 May 2019 17:46:58 -0400 Subject: [PATCH 22/28] add version notes Signed-off-by: Snow Pettersen --- docs/root/intro/version_history.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index c79613e832283..0b9299bb8a09d 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -30,6 +30,9 @@ Version history it in a future update. This is a mechanism to work around a race condition in which an EDS implementation may remove a host before it has stopped passing active HC, thus causing the host to become stranded until a future update. +* upstream: added :ref:`an option ` + that allows ignoring new hosts for the purpose of load balancing calculations until they have + been health checked for the first time. * grpc-json: added support for :ref:`auto mapping `. From 8a9153aea86403fff7d9f38ba6c5187e0bd59776 Mon Sep 17 00:00:00 2001 From: Snow Pettersen Date: Mon, 6 May 2019 19:46:41 -0400 Subject: [PATCH 23/28] fix more updateHostsParams usages Signed-off-by: Snow Pettersen --- .../common/upstream/load_balancer_benchmark.cc | 5 ++--- .../upstream/load_balancer_simulation_test.cc | 12 ++++++------ .../upstream/original_dst_cluster_test.cc | 4 ++-- test/common/upstream/subset_lb_test.cc | 18 +++++++++--------- 4 files changed, 19 insertions(+), 20 deletions(-) diff --git a/test/common/upstream/load_balancer_benchmark.cc b/test/common/upstream/load_balancer_benchmark.cc index 329e4eef53c4b..9bd79569b52fc 100644 --- a/test/common/upstream/load_balancer_benchmark.cc +++ b/test/common/upstream/load_balancer_benchmark.cc @@ -30,9 +30,8 @@ class BaseTester { HostVectorConstSharedPtr updated_hosts{new HostVector(hosts)}; priority_set_.updateHosts( 0, - HostSetImpl::updateHostsParams(updated_hosts, nullptr, - std::make_shared(*updated_hosts), - nullptr), + updateHostsParams(updated_hosts, nullptr, + std::make_shared(*updated_hosts), nullptr), {}, hosts, {}, absl::nullopt); } diff --git a/test/common/upstream/load_balancer_simulation_test.cc b/test/common/upstream/load_balancer_simulation_test.cc index 76f820a161bdd..734e57b194c10 100644 --- a/test/common/upstream/load_balancer_simulation_test.cc +++ b/test/common/upstream/load_balancer_simulation_test.cc @@ -56,9 +56,9 @@ TEST(DISABLED_LeastRequestLoadBalancerWeightTest, Weight) { HostsPerLocalitySharedPtr updated_locality_hosts{new HostsPerLocalityImpl(hosts)}; priority_set.updateHosts( 0, - HostSetImpl::updateHostsParams(updated_hosts, updated_locality_hosts, - std::make_shared(*updated_hosts), - updated_locality_hosts), + updateHostsParams(updated_hosts, updated_locality_hosts, + std::make_shared(*updated_hosts), + updated_locality_hosts), {}, hosts, {}, absl::nullopt); Stats::IsolatedStoreImpl stats_store; @@ -162,9 +162,9 @@ class DISABLED_SimulationTest : public testing::Test { auto per_zone_local_shared = makeHostsPerLocality(std::move(per_zone_local)); local_priority_set_->updateHosts( 0, - HostSetImpl::updateHostsParams( - originating_hosts, per_zone_local_shared, - std::make_shared(*originating_hosts), per_zone_local_shared), + updateHostsParams(originating_hosts, per_zone_local_shared, + std::make_shared(*originating_hosts), + per_zone_local_shared), {}, empty_vector_, empty_vector_, absl::nullopt); HostConstSharedPtr selected = lb.chooseHost(nullptr); diff --git a/test/common/upstream/original_dst_cluster_test.cc b/test/common/upstream/original_dst_cluster_test.cc index 23314f2f25fb3..96e1a76278866 100644 --- a/test/common/upstream/original_dst_cluster_test.cc +++ b/test/common/upstream/original_dst_cluster_test.cc @@ -466,8 +466,8 @@ TEST_F(OriginalDstClusterTest, MultipleClusters) { const HostsPerLocalityConstSharedPtr empty_hosts_per_locality{new HostsPerLocalityImpl()}; second.updateHosts(0, - HostSetImpl::updateHostsParams(new_hosts, empty_hosts_per_locality, - healthy_hosts, empty_hosts_per_locality), + updateHostsParams(new_hosts, empty_hosts_per_locality, healthy_hosts, + empty_hosts_per_locality), {}, added, removed, absl::nullopt); }); diff --git a/test/common/upstream/subset_lb_test.cc b/test/common/upstream/subset_lb_test.cc index a18cff8abadc9..002191906dadf 100644 --- a/test/common/upstream/subset_lb_test.cc +++ b/test/common/upstream/subset_lb_test.cc @@ -317,9 +317,9 @@ class SubsetLoadBalancerTest : public testing::TestWithParam { if (GetParam() == REMOVES_FIRST && !remove.empty()) { local_priority_set_.updateHosts( 0, - HostSetImpl::updateHostsParams(local_hosts_, local_hosts_per_locality_, - std::make_shared(*local_hosts_), - local_hosts_per_locality_), + updateHostsParams(local_hosts_, local_hosts_per_locality_, + std::make_shared(*local_hosts_), + local_hosts_per_locality_), {}, {}, remove, absl::nullopt); } @@ -334,17 +334,17 @@ class SubsetLoadBalancerTest : public testing::TestWithParam { if (!add.empty()) { local_priority_set_.updateHosts( 0, - HostSetImpl::updateHostsParams(local_hosts_, local_hosts_per_locality_, - std::make_shared(*local_hosts_), - local_hosts_per_locality_), + updateHostsParams(local_hosts_, local_hosts_per_locality_, + std::make_shared(*local_hosts_), + local_hosts_per_locality_), {}, add, {}, absl::nullopt); } } else if (!add.empty() || !remove.empty()) { local_priority_set_.updateHosts( 0, - HostSetImpl::updateHostsParams(local_hosts_, local_hosts_per_locality_, - std::make_shared(*local_hosts_), - local_hosts_per_locality_), + updateHostsParams(local_hosts_, local_hosts_per_locality_, + std::make_shared(*local_hosts_), + local_hosts_per_locality_), {}, add, remove, absl::nullopt); } } From 820cd358c9d02d7b0783c1f38736a3b6c208a705 Mon Sep 17 00:00:00 2001 From: Snow Pettersen Date: Mon, 6 May 2019 20:12:17 -0400 Subject: [PATCH 24/28] fix format failures Signed-off-by: Snow Pettersen --- docs/root/intro/version_history.rst | 2 +- test/integration/eds_integration_test.cc | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 0b9299bb8a09d..877fcd7a04354 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -30,7 +30,7 @@ Version history it in a future update. This is a mechanism to work around a race condition in which an EDS implementation may remove a host before it has stopped passing active HC, thus causing the host to become stranded until a future update. -* upstream: added :ref:`an option ` +* upstream: added :ref:`an option ` that allows ignoring new hosts for the purpose of load balancing calculations until they have been health checked for the first time. * grpc-json: added support for :ref:`auto mapping diff --git a/test/integration/eds_integration_test.cc b/test/integration/eds_integration_test.cc index 4aebd48832124..736712051600d 100644 --- a/test/integration/eds_integration_test.cc +++ b/test/integration/eds_integration_test.cc @@ -126,7 +126,7 @@ TEST_P(EdsIntegrationTest, EndpointWarmingSuccessfulHc) { EXPECT_EQ(0, test_server_->gauge("cluster.cluster_0.membership_healthy")->value()); // Wait for the first HC and verify the host is healthy and that it is no longer being excluded. - // The other endpoint should stil be excluded. + // The other endpoint should still be excluded. waitForNextUpstreamRequest(0); upstream_request_->encodeHeaders(Http::TestHeaderMapImpl{{":status", "200"}}, true); test_server_->waitForGaugeEq("cluster.cluster_0.membership_excluded", 0); @@ -153,7 +153,7 @@ TEST_P(EdsIntegrationTest, EndpointWarmingFailedHc) { EXPECT_EQ(0, test_server_->gauge("cluster.cluster_0.membership_healthy")->value()); // Wait for the first HC and verify the host is healthy and that it is no longer being excluded. - // The other endpoint should stil be excluded. + // The other endpoint should still be excluded. waitForNextUpstreamRequest(0); upstream_request_->encodeHeaders(Http::TestHeaderMapImpl{{":status", "503"}}, true); test_server_->waitForGaugeEq("cluster.cluster_0.membership_excluded", 0); From b69d22ef9b3c1b676629dd3871cdb3e7e8f473ea Mon Sep 17 00:00:00 2001 From: Snow Pettersen Date: Mon, 6 May 2019 20:13:49 -0400 Subject: [PATCH 25/28] remove redundant this Signed-off-by: Snow Pettersen --- test/integration/eds_integration_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/eds_integration_test.cc b/test/integration/eds_integration_test.cc index 736712051600d..de2367de9a19f 100644 --- a/test/integration/eds_integration_test.cc +++ b/test/integration/eds_integration_test.cc @@ -110,7 +110,7 @@ TEST_P(EdsIntegrationTest, RemoveAfterHcFail) { // Verifies that endpoints are ignored until health checked when configured to. TEST_P(EdsIntegrationTest, EndpointWarmingSuccessfulHc) { - config_helper_.addConfigModifier([this](envoy::config::bootstrap::v2::Bootstrap& bootstrap) { + config_helper_.addConfigModifier([](envoy::config::bootstrap::v2::Bootstrap& bootstrap) { // Switch predefined cluster_0 to EDS filesystem sourcing. auto* cluster_0 = bootstrap.mutable_static_resources()->mutable_clusters(0); cluster_0->mutable_common_lb_config()->set_ignore_new_hosts_until_first_hc(true); From b79ef40e3e8223c56d21f19596cb58b2997e019e Mon Sep 17 00:00:00 2001 From: Snow Pettersen Date: Tue, 7 May 2019 16:57:33 -0400 Subject: [PATCH 26/28] normalize handling of pending flag, add TODO Signed-off-by: Snow Pettersen --- include/envoy/upstream/upstream.h | 1 + .../common/upstream/health_checker_base_impl.cc | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/include/envoy/upstream/upstream.h b/include/envoy/upstream/upstream.h index 0c503187a097c..bf995d6e2c2cb 100644 --- a/include/envoy/upstream/upstream.h +++ b/include/envoy/upstream/upstream.h @@ -264,6 +264,7 @@ typedef std::shared_ptr LocalityWeightsConstSharedPtr; * Base host set interface. This contains all of the endpoints for a given LocalityLbEndpoints * priority level. */ +// TODO(snowp): Remove the const ref accessors in favor of the shared_ptr ones. class HostSet { public: virtual ~HostSet() {} diff --git a/source/common/upstream/health_checker_base_impl.cc b/source/common/upstream/health_checker_base_impl.cc index 0d24e04f8d7d3..05b29b058916e 100644 --- a/source/common/upstream/health_checker_base_impl.cc +++ b/source/common/upstream/health_checker_base_impl.cc @@ -246,7 +246,6 @@ void HealthCheckerImplBase::ActiveHealthCheckSession::handleSuccess(bool degrade // it to healthy. This makes startup faster with a small reduction in overall reliability // depending on the HC settings. if (first_check_ || ++num_healthy_ == parent_.healthy_threshold_) { - host_->healthFlagClear(Host::HealthFlag::PENDING_ACTIVE_HC); host_->healthFlagClear(Host::HealthFlag::FAILED_ACTIVE_HC); parent_.incHealthy(); changed_state = HealthTransition::Changed; @@ -258,6 +257,15 @@ void HealthCheckerImplBase::ActiveHealthCheckSession::handleSuccess(bool degrade } } + // Clear the pending flag if it is set. By removing this we're marking the host as having been + // health checked. + if (host_->healthFlagGet(Host::HealthFlag::PENDING_ACTIVE_HC)) { + host_->healthFlagClear(Host::HealthFlag::PENDING_ACTIVE_HC); + // Even though the health value of the host might have not changed, we set this to Changed to + // that the cluster can update its list of excluded hosts. + changed_state = HealthTransition::Changed; + } + if (degraded != host_->healthFlagGet(Host::HealthFlag::DEGRADED_ACTIVE_HC)) { if (degraded) { host_->healthFlagSet(Host::HealthFlag::DEGRADED_ACTIVE_HC); @@ -308,10 +316,12 @@ HealthTransition HealthCheckerImplBase::ActiveHealthCheckSession::setUnhealthy( } } + // Clear the pending flag if it is set. By removing this we're marking the host as having been + // health checked. if (host_->healthFlagGet(Host::HealthFlag::PENDING_ACTIVE_HC)) { host_->healthFlagClear(Host::HealthFlag::PENDING_ACTIVE_HC); - // Even though the health value of the host changed, we set this to Changed to that the cluster - // can update its list of excluded hosts. + // Even though the health value of the host didn't change, we set this to Changed to that the + // cluster can update its list of excluded hosts. changed_state = HealthTransition::Changed; } From 1a931fbaec2af6a436d2981a459e6f57fc5bc8bb Mon Sep 17 00:00:00 2001 From: Snow Pettersen Date: Tue, 7 May 2019 17:06:54 -0400 Subject: [PATCH 27/28] increase stats limit Signed-off-by: Snow Pettersen --- test/integration/stats_integration_test.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/integration/stats_integration_test.cc b/test/integration/stats_integration_test.cc index e82b274ccc559..6fdd74e70856b 100644 --- a/test/integration/stats_integration_test.cc +++ b/test/integration/stats_integration_test.cc @@ -210,8 +210,9 @@ TEST_P(ClusterMemoryTestRunner, MemoryLargeClusterSizeWithStats) { // 2019/04/12 6477 59576 Implementing Endpoint lease... // 2019/04/23 6659 59512 Reintroduce dispatcher stats... // 2019/04/24 6161 49415 Pack tags and tag extracted names + // 2019/05/07 6794 49957 Stats for excluded hosts in cluster - EXPECT_EQ(m_per_cluster, 49415); + EXPECT_EQ(m_per_cluster, 49957); } } // namespace From 5510ac73b2d4f23ceaaf4dcbd9c29367bf9c488f Mon Sep 17 00:00:00 2001 From: Snow Pettersen Date: Tue, 7 May 2019 20:10:36 -0400 Subject: [PATCH 28/28] move clearPendingFlag to helper method Signed-off-by: Snow Pettersen --- .../upstream/health_checker_base_impl.cc | 30 +++++++++---------- .../upstream/health_checker_base_impl.h | 4 +++ 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/source/common/upstream/health_checker_base_impl.cc b/source/common/upstream/health_checker_base_impl.cc index 05b29b058916e..69da2bad4e39f 100644 --- a/source/common/upstream/health_checker_base_impl.cc +++ b/source/common/upstream/health_checker_base_impl.cc @@ -257,14 +257,7 @@ void HealthCheckerImplBase::ActiveHealthCheckSession::handleSuccess(bool degrade } } - // Clear the pending flag if it is set. By removing this we're marking the host as having been - // health checked. - if (host_->healthFlagGet(Host::HealthFlag::PENDING_ACTIVE_HC)) { - host_->healthFlagClear(Host::HealthFlag::PENDING_ACTIVE_HC); - // Even though the health value of the host might have not changed, we set this to Changed to - // that the cluster can update its list of excluded hosts. - changed_state = HealthTransition::Changed; - } + changed_state = clearPendingFlag(changed_state); if (degraded != host_->healthFlagGet(Host::HealthFlag::DEGRADED_ACTIVE_HC)) { if (degraded) { @@ -316,14 +309,7 @@ HealthTransition HealthCheckerImplBase::ActiveHealthCheckSession::setUnhealthy( } } - // Clear the pending flag if it is set. By removing this we're marking the host as having been - // health checked. - if (host_->healthFlagGet(Host::HealthFlag::PENDING_ACTIVE_HC)) { - host_->healthFlagClear(Host::HealthFlag::PENDING_ACTIVE_HC); - // Even though the health value of the host didn't change, we set this to Changed to that the - // cluster can update its list of excluded hosts. - changed_state = HealthTransition::Changed; - } + changed_state = clearPendingFlag(changed_state); if ((first_check_ || parent_.always_log_health_check_failures_) && parent_.event_logger_) { parent_.event_logger_->logUnhealthy(parent_.healthCheckerType(), host_, type, first_check_); @@ -354,6 +340,18 @@ void HealthCheckerImplBase::ActiveHealthCheckSession::handleFailure( } } +HealthTransition +HealthCheckerImplBase::ActiveHealthCheckSession::clearPendingFlag(HealthTransition changed_state) { + if (host_->healthFlagGet(Host::HealthFlag::PENDING_ACTIVE_HC)) { + host_->healthFlagClear(Host::HealthFlag::PENDING_ACTIVE_HC); + // Even though the health value of the host might have not changed, we set this to Changed to + // that the cluster can update its list of excluded hosts. + return HealthTransition::Changed; + } + + return changed_state; +} + void HealthCheckerImplBase::ActiveHealthCheckSession::onIntervalBase() { onInterval(); timeout_timer_->enableTimer(parent_.timeout_); diff --git a/source/common/upstream/health_checker_base_impl.h b/source/common/upstream/health_checker_base_impl.h index 8c683428a350f..9d950b2d0abe0 100644 --- a/source/common/upstream/health_checker_base_impl.h +++ b/source/common/upstream/health_checker_base_impl.h @@ -63,6 +63,10 @@ class HealthCheckerImplBase : public HealthChecker, HostSharedPtr host_; private: + // Clears the pending flag if it is set. By clearing this flag we're marking the host as having + // been health checked. + // Returns the changed state to use following the flag update. + HealthTransition clearPendingFlag(HealthTransition changed_state); virtual void onInterval() PURE; void onIntervalBase(); virtual void onTimeout() PURE;