routing/cluster changes: move cluster selection to Envoy's routing table

library/common/config/config.cc

@@ -46,6 +46,7 @@ const std::string config_header = R"(
 - &enable_interface_binding false
 - &h2_connection_keepalive_idle_interval 100000s
 - &h2_connection_keepalive_timeout 10s
+- &h2_hostnames ["api.foo.bar"]
 - &metadata {}
 - &stats_domain 127.0.0.1
 - &stats_flush_interval 60s
@@ -75,7 +76,7 @@ const std::string config_header = R"(
       address:
         socket_address: { address: *statsd_host, port_value: *statsd_port }
 
-!ignore protocol_defs: &http1_protocol_options_defs
+!ignore http1_protocol_options_defs: &http1_protocol_options
     envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
       "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
       explicit_http_config:
@@ -89,6 +90,18 @@ const std::string config_header = R"(
         auto_sni: true
         auto_san_validation: true
 
+!ignore http2_protocol_options_defs: &http2_protocol_options
+    envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
+      "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
+      explicit_http_config:
+        http2_protocol_options:
+          connection_keepalive:
+            connection_idle_interval: *h2_connection_keepalive_idle_interval
+            timeout: *h2_connection_keepalive_timeout
+      upstream_http_protocol_options:
+        auto_sni: true
+        auto_san_validation: true
+
 !ignore admin_interface_defs: &admin_interface
     address:
       socket_address:
@@ -114,7 +127,7 @@ R"(
 )";
 
 const char* config_template = R"(
-!ignore base_protocol_options_defs: &base_protocol_options
+!ignore alpn_protocol_options_defs: &alpn_protocol_options
   envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
     "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
     auto_config:
@@ -214,18 +227,50 @@ const char* config_template = R"(
           route_config:
             name: api_router
             virtual_hosts:
-            - name: api
+            - name: h2
+              include_attempt_count_in_response: true
+              virtual_clusters: *virtual_clusters
+              domains: *h2_hostnames
+              routes:
+#{custom_routes}
+              - match: { prefix: "/" }
+                request_headers_to_remove:
+                - x-forwarded-proto
+                route:
+                  cluster: base_h2
+                  timeout: 0s
+                  retry_policy:
+                    per_try_idle_timeout: *per_try_idle_timeout
+                    retry_back_off:
+                      base_interval: 0.25s
+                      max_interval: 60s
+            - name: catchall
               include_attempt_count_in_response: true
               virtual_clusters: *virtual_clusters
               domains: ["*"]
               routes:
 #{custom_routes}
+              - match:
+                  prefix: "/"
+                  headers:
+                  - name: ":scheme"
+                    string_match:
+                      exact: http
+                request_headers_to_remove:
+                - x-forwarded-proto
+                route:
+                  cluster: base_clear
+                  timeout: 0s
+                  retry_policy:
+                    per_try_idle_timeout: *per_try_idle_timeout
+                    retry_back_off:
+                      base_interval: 0.25s
+                      max_interval: 60s
               - match: { prefix: "/" }
                 request_headers_to_remove:
                 - x-forwarded-proto
-                - x-envoy-mobile-cluster
                 route:
-                  cluster_header: x-envoy-mobile-cluster
+                  cluster: base
                   timeout: 0s
                   retry_policy:
                     per_try_idle_timeout: *per_try_idle_timeout
@@ -321,15 +366,15 @@ R"(
     // Therefore, the ejection time is short and the interval for unejection is tight, but not too
     // tight to cause unnecessary churn.
 R"(
-    typed_extension_protocol_options: *http1_protocol_options_defs
+    typed_extension_protocol_options: *alpn_protocol_options
   - name: base_clear
     connect_timeout: *connect_timeout
     lb_policy: CLUSTER_PROVIDED
     cluster_type: *base_cluster_type
     transport_socket: { name: envoy.transport_sockets.raw_buffer }
     upstream_connection_options: *upstream_opts
     circuit_breakers: *circuit_breakers_settings
-    typed_extension_protocol_options: *http1_protocol_options_defs
+    typed_extension_protocol_options: *http1_protocol_options
   - name: base_h2
     http2_protocol_options: {}
     connect_timeout: *connect_timeout
@@ -349,15 +394,7 @@ R"(
             trust_chain_verification: *trust_chain_verification
     upstream_connection_options: *upstream_opts
     circuit_breakers: *circuit_breakers_settings
-    typed_extension_protocol_options: *base_protocol_options
-  - name: base_alpn
-    connect_timeout: *connect_timeout
-    lb_policy: CLUSTER_PROVIDED
-    cluster_type: *base_cluster_type
-    transport_socket: *base_tls_socket
-    upstream_connection_options: *upstream_opts
-    circuit_breakers: *circuit_breakers_settings
-    typed_extension_protocol_options: *base_protocol_options
+    typed_extension_protocol_options: *http2_protocol_options
 stats_flush_interval: *stats_flush_interval
 stats_sinks: *stats_sinks
 stats_config:

library/common/http/client.cc

@@ -414,7 +414,6 @@ void Client::sendHeaders(envoy_stream_t stream, envoy_headers headers, bool end_
   if (direct_stream) {
     ScopeTrackerScopeState scope(direct_stream.get(), scopeTracker());
     RequestHeaderMapPtr internal_headers = Utility::toRequestHeaders(headers);
-    setDestinationCluster(*internal_headers);
     // Set the x-forwarded-proto header to https because Envoy Mobile only has clusters with TLS
     // enabled. This is done here because the ApiListener's synthetic connection would make the
     // Http::ConnectionManager set the scheme to http otherwise. In the future we might want to
@@ -585,53 +584,8 @@ void Client::removeStream(envoy_stream_t stream_handle) {
 }
 
 namespace {
-
-const LowerCaseString ClusterHeader{"x-envoy-mobile-cluster"};
 const LowerCaseString H2UpstreamHeader{"x-envoy-mobile-upstream-protocol"};
-
-// Alternate clusters included here are a stopgap to make it less likely for a given connection
-// class to suffer "catastrophic" failure of all outbound requests due to a network blip, by
-// distributing requests across a minimum of two potential connections per connection class.
-// Long-term we will be working to generally provide more responsive connection handling within
-// Envoy itself.
-
-const char* BaseCluster = "base";
-const char* H2Cluster = "base_h2";
-const char* ClearTextCluster = "base_clear";
-const char* AlpnCluster = "base_alpn";
-
 } // namespace
 
-void Client::setDestinationCluster(Http::RequestHeaderMap& headers) {
-  // Determine upstream cluster:
-  // - Use TLS by default.
-  // - Use http/2 or ALPN if requested explicitly via x-envoy-mobile-upstream-protocol.
-  // - Force http/1.1 if request scheme is http (cleartext).
-  const char* cluster{};
-  auto h2_header = headers.get(H2UpstreamHeader);
-  if (headers.getSchemeValue() == Headers::get().SchemeValues.Http) {
-    cluster = ClearTextCluster;
-  } else if (!h2_header.empty()) {
-    ASSERT(h2_header.size() == 1);
-    const auto value = h2_header[0]->value().getStringView();
-    if (value == "http2") {
-      cluster = H2Cluster;
-    } else if (value == "alpn") {
-      cluster = AlpnCluster;
-    } else {
-      RELEASE_ASSERT(value == "http1", fmt::format("using unsupported protocol version {}", value));
-      cluster = BaseCluster;
-    }
-  } else {
-    cluster = BaseCluster;
-  }
-
-  if (!h2_header.empty()) {
-    headers.remove(H2UpstreamHeader);
-  }
-
-  headers.addCopy(ClusterHeader, std::string{cluster});
-}
-
 } // namespace Http
 } // namespace Envoy

library/common/http/client.h

@@ -317,7 +317,6 @@ class Client : public Logger::Loggable<Logger::Id::http> {
   };
   DirectStreamSharedPtr getStream(envoy_stream_t stream_handle, GetStreamFilters filters);
   void removeStream(envoy_stream_t stream_handle);
-  void setDestinationCluster(RequestHeaderMap& headers);
 
   ApiListener& api_listener_;
   Event::ProvisionalDispatcher& dispatcher_;

library/java/io/envoyproxy/envoymobile/engine/EnvoyConfiguration.java

@@ -38,6 +38,7 @@ public enum TrustChainVerification {
   public final Boolean enableInterfaceBinding;
   public final Integer h2ConnectionKeepaliveIdleIntervalMilliseconds;
   public final Integer h2ConnectionKeepaliveTimeoutSeconds;
+  public final List<String> h2Hostnames;
   public final List<EnvoyHTTPFilterFactory> httpPlatformFilterFactories;
   public final Integer statsFlushSeconds;
   public final Integer streamIdleTimeoutSeconds;
@@ -70,6 +71,7 @@ public enum TrustChainVerification {
    * @param h2ConnectionKeepaliveIdleIntervalMilliseconds rate in milliseconds seconds to send h2
    *     pings on stream creation.
    * @param h2ConnectionKeepaliveTimeoutSeconds rate in seconds to timeout h2 pings.
+   * @param h2Hostnames                  hostnames to force h2 connections for.
    * @param statsFlushSeconds            interval at which to flush Envoy stats.
    * @param streamIdleTimeoutSeconds     idle timeout for HTTP streams.
    * @param perTryIdleTimeoutSeconds     per try idle timeout for HTTP streams.
@@ -88,6 +90,7 @@ public EnvoyConfiguration(
       List<String> dnsFallbackNameservers, Boolean dnsFilterUnroutableFamilies,
       boolean enableHappyEyeballs, boolean enableInterfaceBinding,
       int h2ConnectionKeepaliveIdleIntervalMilliseconds, int h2ConnectionKeepaliveTimeoutSeconds,
+      List<String> h2Hostnames,
       int statsFlushSeconds, int streamIdleTimeoutSeconds, int perTryIdleTimeoutSeconds,
       String appVersion, String appId, TrustChainVerification trustChainVerification,
       String virtualClusters, List<EnvoyNativeFilterConfig> nativeFilterChain,
@@ -109,6 +112,7 @@ public EnvoyConfiguration(
     this.h2ConnectionKeepaliveIdleIntervalMilliseconds =
         h2ConnectionKeepaliveIdleIntervalMilliseconds;
     this.h2ConnectionKeepaliveTimeoutSeconds = h2ConnectionKeepaliveTimeoutSeconds;
+    this.h2Hostnames = h2Hostnames;
     this.statsFlushSeconds = statsFlushSeconds;
     this.streamIdleTimeoutSeconds = streamIdleTimeoutSeconds;
     this.perTryIdleTimeoutSeconds = perTryIdleTimeoutSeconds;
@@ -161,6 +165,15 @@ String resolveTemplate(final String templateYAML, final String platformFilterTem
       dnsFallbackNameserversAsString = sj.toString();
     }
 
+    String h2HostnamesAsString = "[]";
+    if (!h2Hostnames.isEmpty()) {
+      StringJoiner sj = new StringJoiner(",", "[", "]");
+      for (String hostname : h2Hostnames) {
+        sj.add(String.format("\"%s\"", hostname));
+      }
+      h2HostnamesAsString = sj.toString();
+    }
+
     String dnsResolverConfig = String.format(
         "{\"@type\":\"type.googleapis.com/envoy.extensions.network.dns_resolver.cares.v3.CaresDnsResolverConfig\",\"resolvers\":%s,\"use_resolvers_as_fallback\": %s, \"filter_unroutable_families\": %s}",
         dnsFallbackNameserversAsString, !dnsFallbackNameservers.isEmpty() ? "true" : "false",
@@ -185,6 +198,7 @@ String resolveTemplate(final String templateYAML, final String platformFilterTem
                               h2ConnectionKeepaliveIdleIntervalMilliseconds / 1000.0))
         .append(String.format("- &h2_connection_keepalive_timeout %ss\n",
                               h2ConnectionKeepaliveTimeoutSeconds))
+        .append(String.format("- &h2_hostnames %s\n", h2HostnamesAsString))
         .append(String.format("- &stream_idle_timeout %ss\n", streamIdleTimeoutSeconds))
         .append(String.format("- &per_try_idle_timeout %ss\n", perTryIdleTimeoutSeconds))
         .append(String.format("- &metadata { device_os: %s, app_version: %s, app_id: %s }\n",

library/kotlin/io/envoyproxy/envoymobile/EngineBuilder.kt

@@ -42,6 +42,7 @@ open class EngineBuilder(
   private var enableInterfaceBinding = false
   private var h2ConnectionKeepaliveIdleIntervalMilliseconds = 100000000
   private var h2ConnectionKeepaliveTimeoutSeconds = 10
+  private var h2Hostnames = listOf<String>()
   private var statsFlushSeconds = 60
   private var streamIdleTimeoutSeconds = 15
   private var perTryIdleTimeoutSeconds = 15
@@ -231,6 +232,18 @@ open class EngineBuilder(
     return this
   }
 
+  /**
+   * Add a list of hostnames to force h2 connections for.
+   *
+   * @param h2Hostnames addresses to use.
+   *
+   * @return this builder.
+   */
+  fun addDNSFallbackNameservers(h2Hostnames: List<String>): EngineBuilder {
+    this.h2Hostnames = h2Hostnames
+    return this
+  }
+
   /**
    * Add an interval at which to flush Envoy stats.
    *
@@ -442,6 +455,7 @@ open class EngineBuilder(
             enableInterfaceBinding,
             h2ConnectionKeepaliveIdleIntervalMilliseconds,
             h2ConnectionKeepaliveTimeoutSeconds,
+            h2Hostnames,
             statsFlushSeconds,
             streamIdleTimeoutSeconds,
             perTryIdleTimeoutSeconds,
@@ -476,6 +490,7 @@ open class EngineBuilder(
             enableInterfaceBinding,
             h2ConnectionKeepaliveIdleIntervalMilliseconds,
             h2ConnectionKeepaliveTimeoutSeconds,
+            h2Hostnames,
             statsFlushSeconds,
             streamIdleTimeoutSeconds,
             perTryIdleTimeoutSeconds,

library/objective-c/EnvoyConfiguration.m

@@ -17,6 +17,7 @@ - (instancetype)initWithAdminInterfaceEnabled:(BOOL)adminInterfaceEnabled
     h2ConnectionKeepaliveIdleIntervalMilliseconds:
         (UInt32)h2ConnectionKeepaliveIdleIntervalMilliseconds
               h2ConnectionKeepaliveTimeoutSeconds:(UInt32)h2ConnectionKeepaliveTimeoutSeconds
+                                      h2Hostnames:(NSArray<NSString *> *)h2Hostnames
                                 statsFlushSeconds:(UInt32)statsFlushSeconds
                          streamIdleTimeoutSeconds:(UInt32)streamIdleTimeoutSeconds
                          perTryIdleTimeoutSeconds:(UInt32)perTryIdleTimeoutSeconds
@@ -50,6 +51,7 @@ - (instancetype)initWithAdminInterfaceEnabled:(BOOL)adminInterfaceEnabled
   self.h2ConnectionKeepaliveIdleIntervalMilliseconds =
       h2ConnectionKeepaliveIdleIntervalMilliseconds;
   self.h2ConnectionKeepaliveTimeoutSeconds = h2ConnectionKeepaliveTimeoutSeconds;
+  self.h2Hostnames = h2Hostnames;
   self.statsFlushSeconds = statsFlushSeconds;
   self.streamIdleTimeoutSeconds = streamIdleTimeoutSeconds;
   self.perTryIdleTimeoutSeconds = perTryIdleTimeoutSeconds;
@@ -100,6 +102,18 @@ - (nullable NSString *)resolveTemplate:(NSString *)templateYAML {
         appendString:[[NSString alloc] initWithUTF8String:route_cache_reset_filter_insert]];
   }
 
+  BOOL hasH2Hostnames = self.h2Hostnames.count > 0;
+  if (hasH2Hostnames) {
+    templateYAML = [templateYAML stringByReplacingOccurrencesOfString:@"#{fake_remote_responses}"
+                                                           withString:self.directResponses];
+    [customClusters appendString:[[NSString alloc] initWithUTF8String:fake_remote_cluster_insert]];
+    [customListeners
+        appendString:[[NSString alloc] initWithUTF8String:fake_remote_listener_insert]];
+    [customRoutes appendString:self.directResponseMatchers];
+    [customFilters
+        appendString:[[NSString alloc] initWithUTF8String:route_cache_reset_filter_insert]];
+  }
+
   templateYAML = [templateYAML stringByReplacingOccurrencesOfString:@"#{custom_clusters}"
                                                          withString:customClusters];
   templateYAML = [templateYAML stringByReplacingOccurrencesOfString:@"#{custom_listeners}"
@@ -138,6 +152,7 @@ - (nullable NSString *)resolveTemplate:(NSString *)templateYAML {
                             (double)self.h2ConnectionKeepaliveIdleIntervalMilliseconds / 1000.0];
   [definitions appendFormat:@"- &h2_connection_keepalive_timeout %lus\n",
                             (unsigned long)self.h2ConnectionKeepaliveTimeoutSeconds];
+  [definitions appendFormat:@"- &h2_hostnames %@\n", hasH2Hostnames ? @"[]" : @"[]"];
   [definitions
       appendFormat:@"- &stream_idle_timeout %lus\n", (unsigned long)self.streamIdleTimeoutSeconds];
   [definitions

library/objective-c/EnvoyEngine.h

@@ -341,6 +341,7 @@ extern const int kEnvoyFilterResumeStatusResumeIteration;
 @property (nonatomic, assign) BOOL enableInterfaceBinding;
 @property (nonatomic, assign) UInt32 h2ConnectionKeepaliveIdleIntervalMilliseconds;
 @property (nonatomic, assign) UInt32 h2ConnectionKeepaliveTimeoutSeconds;
+@property (nonatomic, strong) NSArray<NSString *> *h2Hostnames;
 @property (nonatomic, assign) UInt32 statsFlushSeconds;
 @property (nonatomic, assign) UInt32 streamIdleTimeoutSeconds;
 @property (nonatomic, assign) UInt32 perTryIdleTimeoutSeconds;
@@ -369,6 +370,7 @@ extern const int kEnvoyFilterResumeStatusResumeIteration;
     h2ConnectionKeepaliveIdleIntervalMilliseconds:
         (UInt32)h2ConnectionKeepaliveIdleIntervalMilliseconds
               h2ConnectionKeepaliveTimeoutSeconds:(UInt32)h2ConnectionKeepaliveTimeoutSeconds
+                                      h2Hostnames:(NSArray<NSString *> *)h2Hostnames
                                 statsFlushSeconds:(UInt32)statsFlushSeconds
                          streamIdleTimeoutSeconds:(UInt32)streamIdleTimeoutSeconds
                          perTryIdleTimeoutSeconds:(UInt32)perTryIdleTimeoutSeconds

library/swift/EngineBuilder.swift

@@ -25,6 +25,7 @@ open class EngineBuilder: NSObject {
   private var enableInterfaceBinding: Bool = false
   private var h2ConnectionKeepaliveIdleIntervalMilliseconds: UInt32 = 100000000
   private var h2ConnectionKeepaliveTimeoutSeconds: UInt32 = 10
+  private var h2Hostnames: [String] = []
   private var statsFlushSeconds: UInt32 = 60
   private var streamIdleTimeoutSeconds: UInt32 = 15
   private var perTryIdleTimeoutSeconds: UInt32 = 15
@@ -185,6 +186,18 @@ open class EngineBuilder: NSObject {
     return self
   }
 
+  /// Add a list of hostnames to force h2 connections for.
+  ///
+  /// - parameter h2Hostnames: Hostnames to use.
+  ///
+  /// - returns: This builder.
+  @discardableResult
+  public func addH2Hostnames(
+    _ h2Hostnames: [String]) -> Self {
+    self.h2Hostnames = h2Hostnames
+    return self
+  }
+
   /// Add an interval at which to flush Envoy stats.
   ///
   /// - parameter statsFlushSeconds: Interval at which to flush Envoy stats.
@@ -381,6 +394,7 @@ open class EngineBuilder: NSObject {
       h2ConnectionKeepaliveIdleIntervalMilliseconds:
         self.h2ConnectionKeepaliveIdleIntervalMilliseconds,
       h2ConnectionKeepaliveTimeoutSeconds: self.h2ConnectionKeepaliveTimeoutSeconds,
+      h2Hostnames: self.h2Hostnames,
       statsFlushSeconds: self.statsFlushSeconds,
       streamIdleTimeoutSeconds: self.streamIdleTimeoutSeconds,
       perTryIdleTimeoutSeconds: self.perTryIdleTimeoutSeconds,