http: prefetch for upstreams

api/envoy/config/cluster/v3/cluster.proto

@@ -583,11 +583,10 @@ message Cluster {
     google.protobuf.Duration max_interval = 2 [(validate.rules).duration = {gt {nanos: 1000000}}];
   }
 
-  // [#not-implemented-hide:]
   message PreconnectPolicy {
     // Indicates how many streams (rounded up) can be anticipated per-upstream for each
     // incoming stream. This is useful for high-QPS or latency-sensitive services. Preconnecting
-    // will only be done if the upstream is healthy.
+    // will only be done if the upstream is healthy and the cluster has traffic.
     //
     // For example if this is 2, for an incoming HTTP/1.1 stream, 2 connections will be
     // established, one for the new incoming stream, and one for a presumed follow-up stream. For
@@ -605,8 +604,7 @@ message Cluster {
     //
     // If this value is not set, or set explicitly to one, Envoy will fetch as many connections
     // as needed to serve streams in flight. This means in steady state if a connection is torn down,
-    // a subsequent streams will pay an upstream-rtt latency penalty waiting for streams to be
-    // preconnected.
+    // a subsequent streams will pay an upstream-rtt latency penalty waiting for a new connection.
     //
     // This is limited somewhat arbitrarily to 3 because preconnecting too aggressively can
     // harm latency more than the preconnecting helps.
@@ -616,24 +614,25 @@ message Cluster {
     // Indicates how many many streams (rounded up) can be anticipated across a cluster for each
     // stream, useful for low QPS services. This is currently supported for a subset of
     // deterministic non-hash-based load-balancing algorithms (weighted round robin, random).
-    // Unlike per_upstream_preconnect_ratio this preconnects across the upstream instances in a
+    // Unlike *per_upstream_preconnect_ratio* this preconnects across the upstream instances in a
     // cluster, doing best effort predictions of what upstream would be picked next and
     // pre-establishing a connection.
     //
+    // Preconnecting will be limited to one preconnect per configured upstream in the cluster and will
+    // only be done if there are healthy upstreams and the cluster has traffic.
+    //
     // For example if preconnecting is set to 2 for a round robin HTTP/2 cluster, on the first
     // incoming stream, 2 connections will be preconnected - one to the first upstream for this
     // cluster, one to the second on the assumption there will be a follow-up stream.
     //
-    // Preconnecting will be limited to one preconnect per configured upstream in the cluster.
-    //
     // If this value is not set, or set explicitly to one, Envoy will fetch as many connections
     // as needed to serve streams in flight, so during warm up and in steady state if a connection
     // is closed (and per_upstream_preconnect_ratio is not set), there will be a latency hit for
     // connection establishment.
     //
     // If both this and preconnect_ratio are set, Envoy will make sure both predicted needs are met,
-    // basically preconnecting max(predictive-preconnect, per-upstream-preconnect), for each upstream.
-    // TODO(alyssawilk) per LB docs and LB overview docs when unhiding.
+    // basically preconnecting max(predictive-preconnect, per-upstream-preconnect), for each
+    // upstream.
     google.protobuf.DoubleValue predictive_preconnect_ratio = 2
         [(validate.rules).double = {lte: 3.0 gte: 1.0}];
   }
@@ -1028,7 +1027,6 @@ message Cluster {
   // Configuration to track optional cluster stats.
   TrackClusterStats track_cluster_stats = 49;
 
-  // [#not-implemented-hide:]
   // Preconnect configuration for this cluster.
   PreconnectPolicy preconnect_policy = 50;
 

docs/root/version_history/current.rst

@@ -73,6 +73,7 @@ New Features
 * health_check: added option to use :ref:`no_traffic_healthy_interval <envoy_v3_api_field_config.core.v3.HealthCheck.no_traffic_healthy_interval>` which allows a different no traffic interval when the host is healthy.
 * http: added HCM :ref:`timeout config field <envoy_v3_api_field_extensions.filters.network.http_connection_manager.v3.HttpConnectionManager.request_headers_timeout>` to control how long a downstream has to finish sending headers before the stream is cancelled.
 * http: added frame flood and abuse checks to the upstream HTTP/2 codec. This check is off by default and can be enabled by setting the `envoy.reloadable_features.upstream_http2_flood_checks` runtime key to true.
+* http: added support for :ref:`:ref:`preconnecting <envoy_v3_api_msg_config.cluster.v3.Cluster.PreconnectPolicy>`. Preconnecting is off by default, but recommended for clusters serving latency-sensitive traffic, especially if using HTTP/1.1.
 * http: clusters now support selecting HTTP/1 or HTTP/2 based on ALPN, configurable via :ref:`alpn_config <envoy_v3_api_field_extensions.upstreams.http.v3.HttpProtocolOptions.auto_config>` in the :ref:`http_protocol_options <envoy_v3_api_msg_extensions.upstreams.http.v3.HttpProtocolOptions>` message.
 * jwt_authn: added support for :ref:`per-route config <envoy_v3_api_msg_extensions.filters.http.jwt_authn.v3.PerRouteConfig>`.
 * kill_request: added new :ref:`HTTP kill request filter <config_http_filters_kill_request>`.

source/common/conn_pool/conn_pool_base.cc

@@ -189,8 +189,7 @@ ConnectionPool::Cancellable* ConnPoolImplBase::newStream(AttachContext& context)
     ActiveClient& client = *ready_clients_.front();
     ENVOY_CONN_LOG(debug, "using existing connection", client);
     attachStreamToClient(client, context);
-    // Even if there's a ready client, we may want to preconnect a new connection
-    // to handle the next incoming stream.
+    // Even if there's a ready client, we may want to preconnect to handle the next incoming stream.
     tryCreateNewConnections();
     return nullptr;
   }
@@ -368,7 +367,7 @@ void ConnPoolImplBase::onConnectionEvent(ActiveClient& client, absl::string_view
       // NOTE: We move the existing pending streams to a temporary list. This is done so that
       //       if retry logic submits a new stream to the pool, we don't fail it inline.
       purgePendingStreams(client.real_host_description_, failure_reason, reason);
-      // See if we should preconnect another connection based on active connections.
+      // See if we should preconnect based on active connections.
       tryCreateNewConnections();
     }
 

source/common/conn_pool/conn_pool_base.h

@@ -158,8 +158,7 @@ class ConnPoolImplBase : protected Logger::Loggable<Logger::Id::pool> {
   void checkForDrained();
   void scheduleOnUpstreamReady();
   ConnectionPool::Cancellable* newStream(AttachContext& context);
-  // Called if this pool is likely to be picked soon, to determine if it's worth
-  // preconnecting a connection.
+  // Called if this pool is likely to be picked soon, to determine if it's worth preconnecting.
   bool maybePreconnect(float global_preconnect_ratio);
 
   virtual ConnectionPool::Cancellable* newPendingStream(AttachContext& context) PURE;
@@ -185,6 +184,7 @@ class ConnPoolImplBase : protected Logger::Loggable<Logger::Id::pool> {
   bool hasPendingStreams() const { return !pending_streams_.empty(); }
 
 protected:
+  // Creates up to 3 connections, based on the preconnect ratio.
   virtual void onConnected(Envoy::ConnectionPool::ActiveClient&) {}
 
   // Creates up to 3 connections, based on the preconnect ratio.