upstream: allow excluding hosts from lb calculations until initial health check

api/envoy/admin/v2alpha/clusters.proto

@@ -82,6 +82,9 @@ message HostHealthStatus {
   // health checking.
   bool pending_dynamic_removal = 5;
 
+  // The host has not yet been health checked.
+  bool pending_active_hc = 6;
+
   // Health status as reported by EDS. Note: only HEALTHY and UNHEALTHY are currently supported
   // here.
   // TODO(mrice32): pipe through remaining EDS health status possibilities.

api/envoy/api/v2/cds.proto

@@ -526,6 +526,27 @@ message Cluster {
     // because merging those updates isn't currently safe. See
     // https://github.com/envoyproxy/envoy/pull/3941.
     google.protobuf.Duration update_merge_window = 4;
+
+    // If set to true, Envoy will not consider new hosts when computing load balancing weights until
+    // they have been health checked for the first time. This will have no effect unless
+    // active health checking is also configured.
+    //
+    // Ignoring a host means that for any load balancing calculations that adjust weights based
+    // on the ratio of eligible hosts and total hosts (priority spillover, locality weighting, etc.)
+    // will exclude these hosts in the denominator.
+    //
+    // For example, with hosts in two priorities P0 and P1, where P0 looks like
+    // {healthy, unhealthy (new), unhealthy (new)}
+    // and where P1 looks like
+    // {healthy, healthy}
+    // all traffic will still hit P0, as 1 / (3 - 2) = 1.
+    //
+    // Enabling this will allow scaling up the number of hosts for a given cluster without entering
+    // panic mode or triggering priority spillover, assuming the hosts pass the first health check.
+    //
+    // If panic mode is triggered, new hosts are still eligible for traffic; they simply do not
+    // contribute to the calculation when deciding whether panic mode is enabled or not.
+    bool ignore_new_hosts_until_first_hc = 5;
   }
 
   // Common configuration for all load balancer implementations.

include/envoy/upstream/types.h

@@ -55,6 +55,8 @@ struct HealthyAvailability : PriorityAvailability {
 struct Healthy {};
 // Phantom type indicating that the type is related to degraded hosts.
 struct Degraded {};
+// Phantom type indicating that the type is related to excluded hosts.
+struct Excluded {};
 
 } // namespace Upstream
 } // namespace Envoy

include/envoy/upstream/upstream.h

@@ -54,7 +54,9 @@ class Host : virtual public HostDescription {
   m(DEGRADED_EDS_HEALTH, 0x10)                                                   \
   /* The host is pending removal from discovery but is stabilized due to */      \
   /* active HC. */                                                               \
-  m(PENDING_DYNAMIC_REMOVAL, 0x20)
+  m(PENDING_DYNAMIC_REMOVAL, 0x20)                                               \
+  /* The host is pending its initial active health check. */                     \
+  m(PENDING_ACTIVE_HC, 0x40)
   // clang-format on
 
 #define DECLARE_ENUM(name, value) name = value,
@@ -190,19 +192,23 @@ class Host : virtual public HostDescription {
    * @param new_used supplies the new value of host being in use to be stored.
    */
   virtual void used(bool new_used) PURE;
+
+  virtual bool warmed() const PURE;
 };
 
 typedef std::shared_ptr<const Host> HostConstSharedPtr;
 
 typedef std::vector<HostSharedPtr> HostVector;
 typedef Phantom<HostVector, Healthy> HealthyHostVector;
 typedef Phantom<HostVector, Degraded> DegradedHostVector;
+typedef Phantom<HostVector, Excluded> ExcludedHostVector;
 typedef std::unordered_map<std::string, Upstream::HostSharedPtr> HostMap;
 typedef std::shared_ptr<HostVector> HostVectorSharedPtr;
 typedef std::shared_ptr<const HostVector> HostVectorConstSharedPtr;
 
 typedef std::shared_ptr<const HealthyHostVector> HealthyHostVectorConstSharedPtr;
 typedef std::shared_ptr<const DegradedHostVector> DegradedHostVectorConstSharedPtr;
+typedef std::shared_ptr<const ExcludedHostVector> ExcludedHostVectorConstSharedPtr;
 
 typedef std::unique_ptr<HostVector> HostListPtr;
 typedef std::unordered_map<envoy::api::v2::core::Locality, uint32_t, LocalityHash, LocalityEqualTo>
@@ -285,6 +291,12 @@ class HostSet {
    */
   virtual const HostVector& degradedHosts() const PURE;
 
+  /*
+   * @return all excluded hosts contained in the set at the current time. Excluded hosts should be
+   * ignored when computing load balancing weights, but may overlap with hosts in hosts().
+   * */
+  virtual const HostVector& excludedHosts() const PURE;
+
   /**
    * @return hosts per locality.
    */
@@ -300,6 +312,11 @@ class HostSet {
    */
   virtual const HostsPerLocality& degradedHostsPerLocality() const PURE;
 
+  /**
+   * @return same as hostsPerLocality but only contains excluded hosts.
+   */
+  virtual const HostsPerLocality& excludedHostsPerLocality() const PURE;
+
   /**
    * @return weights for each locality in the host set.
    */
@@ -378,9 +395,11 @@ class PrioritySet {
     HostVectorConstSharedPtr hosts;
     HealthyHostVectorConstSharedPtr healthy_hosts;
     DegradedHostVectorConstSharedPtr degraded_hosts;
+    ExcludedHostVectorConstSharedPtr excluded_hosts;
     HostsPerLocalityConstSharedPtr hosts_per_locality;
     HostsPerLocalityConstSharedPtr healthy_hosts_per_locality;
     HostsPerLocalityConstSharedPtr degraded_hosts_per_locality;
+    HostsPerLocalityConstSharedPtr excluded_hosts_per_locality;
   };
 
   /**
@@ -775,6 +794,12 @@ class ClusterInfo {
    */
   virtual bool drainConnectionsOnHostRemoval() const PURE;
 
+  /**
+   * @return true if this cluster is configured to ignore hosts for the purpose of load balancing
+   * computations until they have been health checked for the first time.
+   */
+  virtual bool warmHosts() const PURE;
+
   /**
    * @return eds cluster service_name of the cluster.
    */

source/common/upstream/cluster_manager_impl.cc

@@ -687,17 +687,22 @@ void ClusterManagerImpl::postThreadLocalClusterUpdate(const Cluster& cluster, ui
       host_set->healthyHostsPerLocality().clone();
   HostsPerLocalityConstSharedPtr degraded_hosts_per_locality_copy =
       host_set->degradedHostsPerLocality().clone();
+  ExcludedHostVectorConstSharedPtr excluded_hosts_copy(
+      new ExcludedHostVector(host_set->excludedHosts()));
+  auto excluded_hosts_per_locality_copy = host_set->excludedHostsPerLocality().clone();
 
   tls_->runOnAllThreads([this, name = cluster.info()->name(), priority, hosts_copy,
                          healthy_hosts_copy, degraded_hosts_copy, hosts_per_locality_copy,
                          healthy_hosts_per_locality_copy, degraded_hosts_per_locality_copy,
+                         excluded_hosts_copy, excluded_hosts_per_locality_copy,
                          locality_weights = host_set->localityWeights(), hosts_added, hosts_removed,
                          overprovisioning_factor = host_set->overprovisioningFactor()]() {
     ThreadLocalClusterManagerImpl::updateClusterMembership(
         name, priority,
         HostSetImpl::updateHostsParams(hosts_copy, hosts_per_locality_copy, healthy_hosts_copy,
                                        healthy_hosts_per_locality_copy, degraded_hosts_copy,
-                                       degraded_hosts_per_locality_copy),
+                                       degraded_hosts_per_locality_copy, excluded_hosts_copy,
+                                       excluded_hosts_per_locality_copy),
         locality_weights, hosts_added, hosts_removed, *tls_, overprovisioning_factor);
   });
 }

source/common/upstream/health_checker_base_impl.cc

@@ -230,6 +230,7 @@ void HealthCheckerImplBase::ActiveHealthCheckSession::handleSuccess(bool degrade
     // it to healthy. This makes startup faster with a small reduction in overall reliability
     // depending on the HC settings.
     if (first_check_ || ++num_healthy_ == parent_.healthy_threshold_) {
+      host_->healthFlagClear(Host::HealthFlag::PENDING_ACTIVE_HC);
       host_->healthFlagClear(Host::HealthFlag::FAILED_ACTIVE_HC);
       parent_.incHealthy();
       changed_state = HealthTransition::Changed;
@@ -291,6 +292,11 @@ HealthTransition HealthCheckerImplBase::ActiveHealthCheckSession::setUnhealthy(
     }
   }
 
+  if (host_->healthFlagGet(Host::HealthFlag::PENDING_ACTIVE_HC)) {
+    host_->healthFlagClear(Host::HealthFlag::PENDING_ACTIVE_HC);
+    changed_state = HealthTransition::Changed;
+  }
+
   if ((first_check_ || parent_.always_log_health_check_failures_) && parent_.event_logger_) {
     parent_.event_logger_->logUnhealthy(parent_.healthCheckerType(), host_, type, first_check_);
   }

source/common/upstream/load_balancer_impl.cc

@@ -136,20 +136,20 @@ void LoadBalancerBase::recalculatePerPriorityState(uint32_t priority,
   // by the overprovisioning factor.
   HostSet& host_set = *priority_set.hostSetsPerPriority()[priority];
   per_priority_health.get()[priority] = 0;
-  if (!host_set.hosts().empty()) {
+  const auto host_count = host_set.hosts().size() - host_set.excludedHosts().size();
+
+  if (host_count > 0) {
     // Each priority level's health is ratio of healthy hosts to total number of hosts in a priority
     // multiplied by overprovisioning factor of 1.4 and capped at 100%. It means that if all
     // hosts are healthy that priority's health is 100%*1.4=140% and is capped at 100% which results
     // in 100%. If 80% of hosts are healthy, that priority's health is still 100% (80%*1.4=112% and
     // capped at 100%).
-    per_priority_health.get()[priority] =
-        std::min<uint32_t>(100, (host_set.overprovisioningFactor() *
-                                 host_set.healthyHosts().size() / host_set.hosts().size()));
+    per_priority_health.get()[priority] = std::min<uint32_t>(
+        100, (host_set.overprovisioningFactor() * host_set.healthyHosts().size() / host_count));
 
     // We perform the same computation for degraded hosts.
-    per_priority_degraded.get()[priority] =
-        std::min<uint32_t>(100, (host_set.overprovisioningFactor() *
-                                 host_set.degradedHosts().size() / host_set.hosts().size()));
+    per_priority_degraded.get()[priority] = std::min<uint32_t>(
+        100, (host_set.overprovisioningFactor() * host_set.degradedHosts().size() / host_count));
   }
 
   // Now that we've updated health for the changed priority level, we need to calculate percentage
@@ -442,13 +442,12 @@ HostConstSharedPtr LoadBalancerBase::chooseHost(LoadBalancerContext* context) {
 bool LoadBalancerBase::isGlobalPanic(const HostSet& host_set) {
   uint64_t global_panic_threshold = std::min<uint64_t>(
       100, runtime_.snapshot().getInteger(RuntimePanicThreshold, default_healthy_panic_percent_));
-  double healthy_percent = host_set.hosts().empty()
-                               ? 0
-                               : 100.0 * host_set.healthyHosts().size() / host_set.hosts().size();
+  const auto host_count = host_set.hosts().size() - host_set.excludedHosts().size();
+  double healthy_percent =
+      host_count == 0 ? 0.0 : 100.0 * host_set.healthyHosts().size() / host_count;
 
-  double degraded_percent = host_set.hosts().empty()
-                                ? 0
-                                : 100.0 * host_set.degradedHosts().size() / host_set.hosts().size();
+  double degraded_percent =
+      host_count == 0 ? 0.0 : 100.0 * host_set.degradedHosts().size() / host_count;
   // If the % of healthy hosts in the cluster is less than our panic threshold, we use all hosts.
   if ((healthy_percent + degraded_percent) < global_panic_threshold) {
     return true;
@@ -464,6 +463,8 @@ void ZoneAwareLoadBalancerBase::calculateLocalityPercentage(
     total_hosts += locality_hosts.size();
   }
 
+  // TODO(snowp): Should we ignore excluded hosts here too?
+
   size_t i = 0;
   for (const auto& locality_hosts : hosts_per_locality.get()) {
     ret[i++] = total_hosts > 0 ? 10000ULL * locality_hosts.size() / total_hosts : 0;

source/common/upstream/subset_lb.cc

@@ -551,6 +551,14 @@ void SubsetLoadBalancer::HostSubsetImpl::update(const HostVector& hosts_added,
     }
   }
 
+  auto excluded_hosts = std::make_shared<ExcludedHostVector>();
+  excluded_hosts->get().reserve(original_host_set_.excludedHosts().size());
+  for (const auto& host : original_host_set_.excludedHosts()) {
+    if (cached_predicate(*host)) {
+      excluded_hosts->get().emplace_back(host);
+    }
+  }
+
   // If we only have one locality we can avoid the first call to filter() by
   // just creating a new HostsPerLocality from the list of all hosts.
   //
@@ -568,6 +576,8 @@ void SubsetLoadBalancer::HostSubsetImpl::update(const HostVector& hosts_added,
       original_host_set_.healthyHostsPerLocality().filter({cached_predicate})[0];
   HostsPerLocalityConstSharedPtr degraded_hosts_per_locality =
       original_host_set_.degradedHostsPerLocality().filter({cached_predicate})[0];
+  auto excluded_hosts_per_locality =
+      original_host_set_.excludedHostsPerLocality().filter({cached_predicate})[0];
 
   // We can use the cached predicate here, since we trust that the hosts in hosts_added were also
   // present in the list of all hosts.
@@ -589,9 +599,10 @@ void SubsetLoadBalancer::HostSubsetImpl::update(const HostVector& hosts_added,
 
   HostSetImpl::updateHosts(HostSetImpl::updateHostsParams(
                                hosts, hosts_per_locality, healthy_hosts, healthy_hosts_per_locality,
-                               degraded_hosts, degraded_hosts_per_locality),
+                               degraded_hosts, degraded_hosts_per_locality, excluded_hosts,
+                               excluded_hosts_per_locality),
                            determineLocalityWeights(*hosts_per_locality), filtered_added,
-                           filtered_removed);
+                           filtered_removed, absl::nullopt);
 }
 
 LocalityWeightsConstSharedPtr SubsetLoadBalancer::HostSubsetImpl::determineLocalityWeights(