router: scoped rds (2d): integrate SRDS API into HttpConnectionManager

source/common/http/BUILD

@@ -120,6 +120,7 @@ envoy_cc_library(
     hdrs = ["conn_manager_config.h"],
     deps = [
         ":date_provider_lib",
+        "//include/envoy/config:config_provider_interface",
         "//include/envoy/http:filter_interface",
         "//include/envoy/router:rds_interface",
         "//source/common/network:utility_lib",
@@ -159,6 +160,7 @@ envoy_cc_library(
         "//include/envoy/network:drain_decision_interface",
         "//include/envoy/network:filter_interface",
         "//include/envoy/router:rds_interface",
+        "//include/envoy/router:scopes_interface",
         "//include/envoy/runtime:runtime_interface",
         "//include/envoy/server:overload_manager_interface",
         "//include/envoy/ssl:connection_interface",

source/common/http/conn_manager_config.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "envoy/config/config_provider.h"
 #include "envoy/http/filter.h"
 #include "envoy/router/rds.h"
 #include "envoy/stats/scope.h"
@@ -247,10 +248,18 @@ class ConnectionManagerConfig {
   virtual std::chrono::milliseconds delayedCloseTimeout() const PURE;
 
   /**
-   * @return Router::RouteConfigProvider& the configuration provider used to acquire a route
-   *         config for each request flow.
+   * @return Router::RouteConfigProvider* the configuration provider used to acquire a route
+   *         config for each request flow. Pointer ownership is _not_ transferred to the caller of
+   *         this function. This will return nullptr when scoped routing is enabled.
    */
-  virtual Router::RouteConfigProvider& routeConfigProvider() PURE;
+  virtual Router::RouteConfigProvider* routeConfigProvider() PURE;
+
+  /**
+   * @return Config::ConfigProvider* the configuration provider used to acquire scoped routing
+   * configuration for each request flow. Pointer ownership is _not_ transferred to the caller of
+   * this function. This will return nullptr when scoped routing is not enabled.
+   */
+  virtual Config::ConfigProvider* scopedRouteConfigProvider() PURE;
 
   /**
    * @return const std::string& the server name to write into responses.

source/common/http/conn_manager_impl.cc

@@ -399,7 +399,7 @@ void ConnectionManagerImpl::chargeTracingStats(const Tracing::Reason& tracing_re
 
 ConnectionManagerImpl::ActiveStream::ActiveStream(ConnectionManagerImpl& connection_manager)
     : connection_manager_(connection_manager),
-      snapped_route_config_(connection_manager.config_.routeConfigProvider().config()),
+      snapped_route_config_(connection_manager.config_.routeConfigProvider()->config()),
       stream_id_(connection_manager.random_generator_.random()),
       request_response_timespan_(new Stats::Timespan(
           connection_manager_.stats_.named_.downstream_rq_time_, connection_manager_.timeSource())),

source/common/http/conn_manager_impl.h

@@ -18,6 +18,7 @@
 #include "envoy/network/drain_decision.h"
 #include "envoy/network/filter.h"
 #include "envoy/router/rds.h"
+#include "envoy/router/scopes.h"
 #include "envoy/runtime/runtime.h"
 #include "envoy/server/overload_manager.h"
 #include "envoy/ssl/connection.h"
@@ -491,6 +492,7 @@ class ConnectionManagerImpl : Logger::Loggable<Logger::Id::http>,
 
     ConnectionManagerImpl& connection_manager_;
     Router::ConfigConstSharedPtr snapped_route_config_;
+    Router::ScopedConfigConstSharedPtr snapped_scoped_route_config_;
     Tracing::SpanPtr active_span_;
     const uint64_t stream_id_;
     StreamEncoder* response_encoder_{};

source/common/protobuf/utility.h

@@ -139,6 +139,25 @@ class RepeatedPtrUtil {
     }
     return HashUtil::xxHash64(text);
   }
+
+  /**
+   * Converts a proto repeated field into a generic vector of const Protobuf::Message unique_ptr's.
+   *
+   * @param repeated_field the proto repeated field to convert.
+   * @return ProtobufType::ConstMessagePtrVector the vector of const Message pointers.
+   */
+  template <typename ProtoType>
+  static ProtobufTypes::ConstMessagePtrVector
+  convertToConstMessagePtrVector(const Protobuf::RepeatedPtrField<ProtoType>& repeated_field) {
+    ProtobufTypes::ConstMessagePtrVector ret_vector;
+    std::transform(repeated_field.begin(), repeated_field.end(), std::back_inserter(ret_vector),
+                   [](const ProtoType& proto_message) -> std::unique_ptr<const Protobuf::Message> {
+                     Protobuf::Message* clone = proto_message.New();
+                     clone->MergeFrom(proto_message);
+                     return std::unique_ptr<const Protobuf::Message>(clone);
+                   });
+    return ret_vector;
+  }
 };
 
 class ProtoValidationException : public EnvoyException {

source/common/router/scoped_rds.cc

@@ -19,6 +19,46 @@ using Envoy::Config::ConfigProviderPtr;
 namespace Envoy {
 namespace Router {
 
+namespace ScopedRoutesConfigProviderUtil {
+
+ConfigProviderPtr
+create(const envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager&
+           config,
+       Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix,
+       ConfigProviderManager& scoped_routes_config_provider_manager) {
+  ASSERT(config.route_specifier_case() == envoy::config::filter::network::http_connection_manager::
+                                              v2::HttpConnectionManager::kScopedRoutes);
+
+  switch (config.scoped_routes().config_specifier_case()) {
+  case envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::
+      kScopedRouteConfigurationsList: {
+    const envoy::config::filter::network::http_connection_manager::v2::
+        ScopedRouteConfigurationsList& scoped_route_list =
+            config.scoped_routes().scoped_route_configurations_list();
+    return scoped_routes_config_provider_manager.createStaticConfigProvider(
+        RepeatedPtrUtil::convertToConstMessagePtrVector(
+            scoped_route_list.scoped_route_configurations()),
+        factory_context,
+        ScopedRoutesConfigProviderManagerOptArg(config.scoped_routes().name(),
+                                                config.scoped_routes().rds_config_source(),
+                                                config.scoped_routes().scope_key_builder()));
+  }
+
+  case envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::kScopedRds:
+    return scoped_routes_config_provider_manager.createXdsConfigProvider(
+        config.scoped_routes().scoped_rds(), factory_context, stat_prefix,
+        ScopedRoutesConfigProviderManagerOptArg(config.scoped_routes().name(),
+                                                config.scoped_routes().rds_config_source(),
+                                                config.scoped_routes().scope_key_builder()));
+
+  default:
+    // Proto validation enforces that is not reached.
+    NOT_REACHED_GCOVR_EXCL_LINE;
+  }
+}
+
+} // namespace ScopedRoutesConfigProviderUtil
+
 InlineScopedRoutesConfigProvider::InlineScopedRoutesConfigProvider(
     ProtobufTypes::ConstMessagePtrVector&& config_protos, std::string name,
     Server::Configuration::FactoryContext& factory_context,

source/common/router/scoped_rds.h

@@ -13,6 +13,19 @@
 namespace Envoy {
 namespace Router {
 
+// Scoped routing configuration utilities.
+namespace ScopedRoutesConfigProviderUtil {
+
+// If enabled in the HttpConnectionManager config, returns a ConfigProvider for scoped routing
+// configuration.
+Envoy::Config::ConfigProviderPtr
+create(const envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager&
+           config,
+       Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix,
+       Envoy::Config::ConfigProviderManager& scoped_routes_config_provider_manager);
+
+} // namespace ScopedRoutesConfigProviderUtil
+
 class ScopedRoutesConfigProviderManager;
 
 // A ConfigProvider for inline scoped routing configuration.

source/extensions/filters/network/http_connection_manager/BUILD

@@ -17,6 +17,7 @@ envoy_cc_library(
     srcs = ["config.cc"],
     hdrs = ["config.h"],
     deps = [
+        "//include/envoy/config:config_provider_manager_interface",
         "//include/envoy/filesystem:filesystem_interface",
         "//include/envoy/http:filter_interface",
         "//include/envoy/registry",
@@ -36,6 +37,7 @@ envoy_cc_library(
         "//source/common/json:json_loader_lib",
         "//source/common/protobuf:utility_lib",
         "//source/common/router:rds_lib",
+        "//source/common/router:scoped_rds_lib",
         "//source/extensions/filters/network:well_known_names",
         "//source/extensions/filters/network/common:factory_base_lib",
     ],

source/extensions/filters/network/http_connection_manager/config.cc

@@ -23,6 +23,7 @@
 #include "common/json/config_schemas.h"
 #include "common/protobuf/utility.h"
 #include "common/router/rds_impl.h"
+#include "common/router/scoped_rds.h"
 
 namespace Envoy {
 namespace Extensions {
@@ -69,6 +70,7 @@ std::unique_ptr<Http::InternalAddressConfig> createInternalAddressConfig(
 // Singleton registration via macro defined in envoy/singleton/manager.h
 SINGLETON_MANAGER_REGISTRATION(date_provider);
 SINGLETON_MANAGER_REGISTRATION(route_config_provider_manager);
+SINGLETON_MANAGER_REGISTRATION(scoped_routes_config_provider_manager);
 
 Network::FilterFactoryCb
 HttpConnectionManagerFilterConfigFactory::createFilterFactoryFromProtoTyped(
@@ -88,14 +90,21 @@ HttpConnectionManagerFilterConfigFactory::createFilterFactoryFromProtoTyped(
             return std::make_shared<Router::RouteConfigProviderManagerImpl>(context.admin());
           });
 
+  std::shared_ptr<Config::ConfigProviderManager> scoped_routes_config_provider_manager =
+      context.singletonManager().getTyped<Config::ConfigProviderManager>(
+          SINGLETON_MANAGER_REGISTERED_NAME(scoped_routes_config_provider_manager), [&context] {
+            return std::make_shared<Router::ScopedRoutesConfigProviderManager>(context.admin());
+          });
+
   std::shared_ptr<HttpConnectionManagerConfig> filter_config(new HttpConnectionManagerConfig(
-      proto_config, context, *date_provider, *route_config_provider_manager));
+      proto_config, context, *date_provider, *route_config_provider_manager,
+      *scoped_routes_config_provider_manager));
 
   // This lambda captures the shared_ptrs created above, thus preserving the
   // reference count. Moreover, keep in mind the capture list determines
   // destruction order.
-  return [route_config_provider_manager, filter_config, &context,
-          date_provider](Network::FilterManager& filter_manager) -> void {
+  return [route_config_provider_manager, scoped_routes_config_provider_manager, filter_config,
+          &context, date_provider](Network::FilterManager& filter_manager) -> void {
     filter_manager.addReadFilter(Network::ReadFilterSharedPtr{new Http::ConnectionManagerImpl(
         *filter_config, context.drainDecision(), context.random(), context.httpContext(),
         context.runtime(), context.localInfo(), context.clusterManager(),
@@ -125,7 +134,8 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig(
     const envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager&
         config,
     Server::Configuration::FactoryContext& context, Http::DateProvider& date_provider,
-    Router::RouteConfigProviderManager& route_config_provider_manager)
+    Router::RouteConfigProviderManager& route_config_provider_manager,
+    Config::ConfigProviderManager& scoped_routes_config_provider_manager)
     : context_(context), stats_prefix_(fmt::format("http.{}.", config.stat_prefix())),
       stats_(Http::ConnectionManagerImpl::generateStats(stats_prefix_, context_.scope())),
       tracing_stats_(
@@ -135,6 +145,7 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig(
       xff_num_trusted_hops_(config.xff_num_trusted_hops()),
       skip_xff_append_(config.skip_xff_append()), via_(config.via()),
       route_config_provider_manager_(route_config_provider_manager),
+      scoped_routes_config_provider_manager_(scoped_routes_config_provider_manager),
       http2_settings_(Http::Utility::parseHttp2Settings(config.http2_protocol_options())),
       http1_settings_(Http::Utility::parseHttp1Settings(config.http_protocol_options())),
       max_request_headers_kb_(PROTOBUF_GET_WRAPPED_OR_DEFAULT(
@@ -162,9 +173,25 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig(
                                                       0
 #endif
                                                       ))) {
-
-  route_config_provider_ = Router::RouteConfigProviderUtil::create(config, context_, stats_prefix_,
-                                                                   route_config_provider_manager_);
+  // If scoped RDS is enabled, avoid creating a route config provider. Route config providers will
+  // be managed by the scoped routing logic instead.
+  switch (config.route_specifier_case()) {
+  case envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager::kRds:
+  case envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager::
+      kRouteConfig:
+    route_config_provider_ = Router::RouteConfigProviderUtil::create(
+        config, context_, stats_prefix_, route_config_provider_manager_);
+    break;
+  case envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager::
+      kScopedRoutes:
+    ENVOY_LOG(warn, "Scoped routing has been enabled but it is not yet fully implemented! HTTP "
+                    "request routing DOES NOT work (yet) with this configuration.");
+    scoped_routes_config_provider_ = Router::ScopedRoutesConfigProviderUtil::create(
+        config, context_, stats_prefix_, scoped_routes_config_provider_manager_);
+    break;
+  default:
+    NOT_REACHED_GCOVR_EXCL_LINE;
+  }
 
   switch (config.forward_client_cert_details()) {
   case envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager::SANITIZE:

source/extensions/filters/network/http_connection_manager/config.h

@@ -7,6 +7,7 @@
 #include <map>
 #include <string>
 
+#include "envoy/config/config_provider_manager.h"
 #include "envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.pb.validate.h"
 #include "envoy/http/filter.h"
 #include "envoy/router/route_config_provider_manager.h"
@@ -78,7 +79,8 @@ class HttpConnectionManagerConfig : Logger::Loggable<Logger::Id::config>,
       const envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager&
           config,
       Server::Configuration::FactoryContext& context, Http::DateProvider& date_provider,
-      Router::RouteConfigProviderManager& route_config_provider_manager);
+      Router::RouteConfigProviderManager& route_config_provider_manager,
+      Config::ConfigProviderManager& scoped_routes_config_provider_manager);
 
   // Http::FilterChainFactory
   void createFilterChain(Http::FilterChainFactoryCallbacks& callbacks) override;
@@ -104,7 +106,12 @@ class HttpConnectionManagerConfig : Logger::Loggable<Logger::Id::config>,
   absl::optional<std::chrono::milliseconds> idleTimeout() const override { return idle_timeout_; }
   std::chrono::milliseconds streamIdleTimeout() const override { return stream_idle_timeout_; }
   std::chrono::milliseconds requestTimeout() const override { return request_timeout_; }
-  Router::RouteConfigProvider& routeConfigProvider() override { return *route_config_provider_; }
+  Router::RouteConfigProvider* routeConfigProvider() override {
+    return route_config_provider_.get();
+  }
+  Config::ConfigProvider* scopedRouteConfigProvider() override {
+    return scoped_routes_config_provider_.get();
+  }
   const std::string& serverName() override { return server_name_; }
   Http::ConnectionManagerStats& stats() override { return stats_; }
   Http::ConnectionManagerTracingStats& tracingStats() override { return tracing_stats_; }
@@ -151,6 +158,7 @@ class HttpConnectionManagerConfig : Logger::Loggable<Logger::Id::config>,
   Http::ForwardClientCertType forward_client_cert_;
   std::vector<Http::ClientCertDetailsType> set_current_client_cert_details_;
   Router::RouteConfigProviderManager& route_config_provider_manager_;
+  Config::ConfigProviderManager& scoped_routes_config_provider_manager_;
   CodecType codec_type_;
   const Http::Http2Settings http2_settings_;
   const Http::Http1Settings http1_settings_;
@@ -162,6 +170,7 @@ class HttpConnectionManagerConfig : Logger::Loggable<Logger::Id::config>,
   std::chrono::milliseconds stream_idle_timeout_;
   std::chrono::milliseconds request_timeout_;
   Router::RouteConfigProviderPtr route_config_provider_;
+  Config::ConfigProviderPtr scoped_routes_config_provider_;
   std::chrono::milliseconds drain_timeout_;
   bool generate_request_id_;
   Http::DateProvider& date_provider_;

source/server/http/BUILD

@@ -53,6 +53,7 @@ envoy_cc_library(
         "//source/common/network:utility_lib",
         "//source/common/profiler:profiler_lib",
         "//source/common/router:config_lib",
+        "//source/common/router:scoped_config_lib",
         "//source/common/stats:histogram_lib",
         "//source/common/stats:isolated_store_lib",
         "//source/common/stats:stats_lib",

source/server/http/admin.cc

@@ -1157,6 +1157,7 @@ AdminImpl::AdminImpl(const std::string& profile_path, Server::Instance& server)
       tracing_stats_(
           Http::ConnectionManagerImpl::generateTracingStats("http.admin.", no_op_store_)),
       route_config_provider_(server.timeSource()),
+      scoped_route_config_provider_(server.timeSource()),
       // TODO(jsedgwick) add /runtime_reset endpoint that removes all admin-set values
       handlers_{
           {"/", "Admin home page", MAKE_ADMIN_HANDLER(handlerAdminHome), false, false},

source/server/http/admin.h

@@ -27,6 +27,7 @@
 #include "common/http/default_server_string.h"
 #include "common/http/utility.h"
 #include "common/network/raw_buffer_socket.h"
+#include "common/router/scoped_config_impl.h"
 #include "common/stats/isolated_store_impl.h"
 
 #include "server/http/config_tracker_impl.h"
@@ -103,7 +104,10 @@ class AdminImpl : public Admin,
   std::chrono::milliseconds streamIdleTimeout() const override { return {}; }
   std::chrono::milliseconds requestTimeout() const override { return {}; }
   std::chrono::milliseconds delayedCloseTimeout() const override { return {}; }
-  Router::RouteConfigProvider& routeConfigProvider() override { return route_config_provider_; }
+  Router::RouteConfigProvider* routeConfigProvider() override { return &route_config_provider_; }
+  Config::ConfigProvider* scopedRouteConfigProvider() override {
+    return &scoped_route_config_provider_;
+  }
   const std::string& serverName() override { return Http::DefaultServerString::get(); }
   Http::ConnectionManagerStats& stats() override { return stats_; }
   Http::ConnectionManagerTracingStats& tracingStats() override { return tracing_stats_; }
@@ -160,6 +164,28 @@ class AdminImpl : public Admin,
     TimeSource& time_source_;
   };
 
+  /**
+   * Implementation of ScopedRouteConfigProvider that returns a null scoped route config.
+   */
+  struct NullScopedRouteConfigProvider : public Config::ConfigProvider {
+    NullScopedRouteConfigProvider(TimeSource& time_source)
+        : config_(std::make_shared<const Router::NullScopedConfigImpl>()),
+          time_source_(time_source) {}
+
+    ~NullScopedRouteConfigProvider() override = default;
+
+    // Config::ConfigProvider
+    SystemTime lastUpdated() const override { return time_source_.systemTime(); }
+    const Protobuf::Message* getConfigProto() const override { return nullptr; }
+    std::string getConfigVersion() const override { return ""; }
+    ConfigConstSharedPtr getConfig() const override { return config_; }
+    ApiType apiType() const override { return ApiType::Full; }
+    ConfigProtoVector getConfigProtos() const override { return {}; }
+
+    Router::ScopedConfigConstSharedPtr config_;
+    TimeSource& time_source_;
+  };
+
   friend class AdminStatsTest;
 
   /**
@@ -310,6 +336,7 @@ class AdminImpl : public Admin,
   Stats::IsolatedStoreImpl no_op_store_;
   Http::ConnectionManagerTracingStats tracing_stats_;
   NullRouteConfigProvider route_config_provider_;
+  NullScopedRouteConfigProvider scoped_route_config_provider_;
   std::list<UrlHandler> handlers_;
   const uint32_t max_request_headers_kb_{Http::DEFAULT_MAX_REQUEST_HEADERS_KB};
   absl::optional<std::chrono::milliseconds> idle_timeout_;