http: prefetch for upstreams

api/envoy/config/cluster/v3/cluster.proto

@@ -583,11 +583,10 @@ message Cluster {
     google.protobuf.Duration max_interval = 2 [(validate.rules).duration = {gt {nanos: 1000000}}];
   }
 
-  // [#not-implemented-hide:]
   message PreconnectPolicy {
     // Indicates how many streams (rounded up) can be anticipated per-upstream for each
     // incoming stream. This is useful for high-QPS or latency-sensitive services. Preconnecting
-    // will only be done if the upstream is healthy.
+    // will only be done if the upstream is healthy and the cluster has traffic.
     //
     // For example if this is 2, for an incoming HTTP/1.1 stream, 2 connections will be
     // established, one for the new incoming stream, and one for a presumed follow-up stream. For
@@ -605,8 +604,7 @@ message Cluster {
     //
     // If this value is not set, or set explicitly to one, Envoy will fetch as many connections
     // as needed to serve streams in flight. This means in steady state if a connection is torn down,
-    // a subsequent streams will pay an upstream-rtt latency penalty waiting for streams to be
-    // preconnected.
+    // a subsequent streams will pay an upstream-rtt latency penalty waiting for a new connection.
     //
     // This is limited somewhat arbitrarily to 3 because preconnecting too aggressively can
     // harm latency more than the preconnecting helps.
@@ -616,24 +614,25 @@ message Cluster {
     // Indicates how many many streams (rounded up) can be anticipated across a cluster for each
     // stream, useful for low QPS services. This is currently supported for a subset of
     // deterministic non-hash-based load-balancing algorithms (weighted round robin, random).
-    // Unlike per_upstream_preconnect_ratio this preconnects across the upstream instances in a
+    // Unlike *per_upstream_preconnect_ratio* this preconnects across the upstream instances in a
     // cluster, doing best effort predictions of what upstream would be picked next and
     // pre-establishing a connection.
     //
+    // Preconnecting will be limited to one preconnect per configured upstream in the cluster and will
+    // only be done if there are healthy upstreams and the cluster has traffic.
+    //
     // For example if preconnecting is set to 2 for a round robin HTTP/2 cluster, on the first
     // incoming stream, 2 connections will be preconnected - one to the first upstream for this
     // cluster, one to the second on the assumption there will be a follow-up stream.
     //
-    // Preconnecting will be limited to one preconnect per configured upstream in the cluster.
-    //
     // If this value is not set, or set explicitly to one, Envoy will fetch as many connections
     // as needed to serve streams in flight, so during warm up and in steady state if a connection
     // is closed (and per_upstream_preconnect_ratio is not set), there will be a latency hit for
     // connection establishment.
     //
     // If both this and preconnect_ratio are set, Envoy will make sure both predicted needs are met,
-    // basically preconnecting max(predictive-preconnect, per-upstream-preconnect), for each upstream.
-    // TODO(alyssawilk) per LB docs and LB overview docs when unhiding.
+    // basically preconnecting max(predictive-preconnect, per-upstream-preconnect), for each
+    // upstream.
     google.protobuf.DoubleValue predictive_preconnect_ratio = 2
         [(validate.rules).double = {lte: 3.0 gte: 1.0}];
   }
@@ -1028,7 +1027,6 @@ message Cluster {
   // Configuration to track optional cluster stats.
   TrackClusterStats track_cluster_stats = 49;
 
-  // [#not-implemented-hide:]
   // Preconnect configuration for this cluster.
   PreconnectPolicy preconnect_policy = 50;
 

docs/root/version_history/current.rst

@@ -19,6 +19,7 @@ Removed Config or Runtime
 
 New Features
 ------------
+* http: added support for :ref:`:ref:`preconnecting <envoy_v3_api_msg_config.cluster.v3.Cluster.PreconnectPolicy>`. Preconnecting is off by default, but recommended for clusters serving latency-sensitive traffic, especially if using HTTP/1.1.
 * tcp_proxy: add support for converting raw TCP streams into HTTP/1.1 CONNECT requests. See :ref:`upgrade documentation <tunneling-tcp-over-http>` for details.
 
 Deprecated

source/common/conn_pool/conn_pool_base.cc

@@ -35,6 +35,28 @@ void ConnPoolImplBase::destructAllConnections() {
   dispatcher_.clearDeferredDeleteList();
 }
 
+bool ConnPoolImplBase::shouldConnect(size_t pending_streams, size_t active_streams,
+                                     uint32_t connecting_capacity, float preconnect_ratio,
+                                     bool anticipate_incoming_stream) {
+  // This is set to true any time global preconnect is being calculated.
+  // ClusterManagerImpl::maybePreconnect is called directly before a stream is created, so the
+  // stream must be anticipated.
+  //
+  // Also without this, we would never pre-establish a connection as the first
+  // connection in a pool because pending/active streams could both be 0.
+  int anticipated_streams = anticipate_incoming_stream ? 1 : 0;
+
+  // The number of streams we want to be provisioned for is the number of
+  // pending, active, and anticipated streams times the preconnect ratio.
+  // The number of streams we are (theoretically) provisioned for is the
+  // connecting stream capacity plus the number of active streams.
+  //
+  // If preconnect ratio is not set, it defaults to 1, and this simplifies to the
+  // legacy value of pending_streams_.size() > connecting_stream_capacity_
+  return (pending_streams + active_streams + anticipated_streams) * preconnect_ratio >
+         connecting_capacity + active_streams;
+}
+
 bool ConnPoolImplBase::shouldCreateNewConnection(float global_preconnect_ratio) const {
   // If the host is not healthy, don't make it do extra work, especially as
   // upstream selection logic may result in bypassing this upstream entirely.
@@ -48,9 +70,8 @@ bool ConnPoolImplBase::shouldCreateNewConnection(float global_preconnect_ratio)
   // preconnect limit, preconnect.
   // We may eventually want to track preconnect_attempts to allow more preconnecting for
   // heavily weighted upstreams or sticky picks.
-  if (global_preconnect_ratio > 1.0 &&
-      ((pending_streams_.size() + 1 + num_active_streams_) * global_preconnect_ratio >
-       (connecting_stream_capacity_ + num_active_streams_))) {
+  if (shouldConnect(pending_streams_.size(), num_active_streams_, connecting_stream_capacity_,
+                    global_preconnect_ratio, true)) {
     return true;
   }
 
@@ -61,8 +82,8 @@ bool ConnPoolImplBase::shouldCreateNewConnection(float global_preconnect_ratio)
   //
   // If preconnect ratio is not set, it defaults to 1, and this simplifies to the
   // legacy value of pending_streams_.size() > connecting_stream_capacity_
-  return (pending_streams_.size() + num_active_streams_) * perUpstreamPreconnectRatio() >
-         (connecting_stream_capacity_ + num_active_streams_);
+  return shouldConnect(pending_streams_.size(), num_active_streams_, connecting_stream_capacity_,
+                       perUpstreamPreconnectRatio());
 }
 
 float ConnPoolImplBase::perUpstreamPreconnectRatio() const {
@@ -189,8 +210,7 @@ ConnectionPool::Cancellable* ConnPoolImplBase::newStream(AttachContext& context)
     ActiveClient& client = *ready_clients_.front();
     ENVOY_CONN_LOG(debug, "using existing connection", client);
     attachStreamToClient(client, context);
-    // Even if there's a ready client, we may want to preconnect a new connection
-    // to handle the next incoming stream.
+    // Even if there's a ready client, we may want to preconnect to handle the next incoming stream.
     tryCreateNewConnections();
     return nullptr;
   }
@@ -368,7 +388,7 @@ void ConnPoolImplBase::onConnectionEvent(ActiveClient& client, absl::string_view
       // NOTE: We move the existing pending streams to a temporary list. This is done so that
       //       if retry logic submits a new stream to the pool, we don't fail it inline.
       purgePendingStreams(client.real_host_description_, failure_reason, reason);
-      // See if we should preconnect another connection based on active connections.
+      // See if we should preconnect based on active connections.
       tryCreateNewConnections();
     }