diff --git a/api/envoy/admin/v2alpha/BUILD b/api/envoy/admin/v2alpha/BUILD index 3806af07dd9fe..6b9e6bb44475b 100644 --- a/api/envoy/admin/v2alpha/BUILD +++ b/api/envoy/admin/v2alpha/BUILD @@ -10,6 +10,7 @@ api_proto_library_internal( "//envoy/api/v2:cds", "//envoy/api/v2:lds", "//envoy/api/v2:rds", + "//envoy/api/v2:srds", "//envoy/config/bootstrap/v2:bootstrap", ], ) diff --git a/api/envoy/admin/v2alpha/config_dump.proto b/api/envoy/admin/v2alpha/config_dump.proto index cfc1cdd54d8f5..ba5b36df36c7c 100644 --- a/api/envoy/admin/v2alpha/config_dump.proto +++ b/api/envoy/admin/v2alpha/config_dump.proto @@ -9,6 +9,7 @@ option java_package = "io.envoyproxy.envoy.admin.v2alpha"; import "envoy/api/v2/cds.proto"; import "envoy/api/v2/lds.proto"; import "envoy/api/v2/rds.proto"; +import "envoy/api/v2/srds.proto"; import "envoy/config/bootstrap/v2/bootstrap.proto"; import "google/protobuf/any.proto"; @@ -178,3 +179,43 @@ message RoutesConfigDump { // The dynamically loaded route configs. repeated DynamicRouteConfig dynamic_route_configs = 3 [(gogoproto.nullable) = false]; } + +// Envoy's scoped RDS implementation fills this message with all currently loaded route +// configuration scopes (defined via ScopedRouteConfigurationsSet protos). This message lists both +// the scopes defined inline with the higher order object (i.e., the HttpConnectionManager) and the +// dynamically obtained scopes via the SRDS API. +message ScopedRoutesConfigDump { + message InlineScopedRouteConfigs { + // The name assigned to the scoped route configurations. + string name = 1; + + // The scoped route configurations. + repeated envoy.api.v2.ScopedRouteConfiguration scoped_route_configs = 2; + + // The timestamp when the scoped route config set was last updated. + google.protobuf.Timestamp last_updated = 3; + } + + message DynamicScopedRouteConfigs { + // The name assigned to the scoped route configurations. + string name = 1; + + // This is the per-resource version information. This version is currently taken from the + // :ref:`version_info ` field at the time that + // the scoped routes configuration was loaded. + string version_info = 2; + + // The scoped route configurations. + repeated envoy.api.v2.ScopedRouteConfiguration scoped_route_configs = 3; + + // The timestamp when the scoped route config set was last updated. + google.protobuf.Timestamp last_updated = 4; + } + + // The statically loaded scoped route configs. + repeated InlineScopedRouteConfigs inline_scoped_route_configs = 1 [(gogoproto.nullable) = false]; + + // The dynamically loaded scoped route configs. + repeated DynamicScopedRouteConfigs dynamic_scoped_route_configs = 2 + [(gogoproto.nullable) = false]; +} diff --git a/api/envoy/api/v2/BUILD b/api/envoy/api/v2/BUILD index 66efb5d30ec5e..3cc2d6b2c2ecc 100644 --- a/api/envoy/api/v2/BUILD +++ b/api/envoy/api/v2/BUILD @@ -144,3 +144,24 @@ api_go_grpc_library( "//envoy/api/v2/route:route_go_proto", ], ) + +api_proto_library_internal( + name = "srds", + srcs = ["srds.proto"], + has_services = 1, + visibility = [":friends"], + deps = [ + ":discovery", + "//envoy/api/v2/core:base", + "//envoy/api/v2/route", + ], +) + +api_go_grpc_library( + name = "srds", + proto = ":srds", + deps = [ + ":discovery_go_proto", + "//envoy/api/v2/core:base_go_proto", + ], +) diff --git a/api/envoy/api/v2/srds.proto b/api/envoy/api/v2/srds.proto new file mode 100644 index 0000000000000..4574ff98e2547 --- /dev/null +++ b/api/envoy/api/v2/srds.proto @@ -0,0 +1,135 @@ +syntax = "proto3"; + +package envoy.api.v2; + +option java_outer_classname = "SrdsProto"; +option java_package = "io.envoyproxy.envoy.api.v2"; +option java_multiple_files = true; +option java_generic_services = true; + +import "envoy/api/v2/discovery.proto"; + +import "google/api/annotations.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +option (gogoproto.equal_all) = true; + +// [#protodoc-title: HTTP scoped routing configuration] +// * Routing :ref:`architecture overview ` +// +// .. attention:: +// +// The Scoped RDS API is not yet fully implemented and *should not* be enabled in +// :ref:`envoy_api_msg_config.filter.network.http_connection_manager.v2.HttpConnectionManager`. +// +// TODO(AndresGuedez): Update :ref:`arch_overview_http_routing` with scoped routing overview and +// configuration details. + +// The Scoped Routes Discovery Service (SRDS) API distributes +// :ref:`ScopedRouteConfiguration` resources. Each +// ScopedRouteConfiguration resource represents a "routing scope" containing a mapping that allows +// the HTTP connection manager to dynamically assign a routing table (specified via +// a :ref:`RouteConfiguration` message) to each HTTP request. +service ScopedRoutesDiscoveryService { + rpc StreamScopedRoutes(stream DiscoveryRequest) returns (stream DiscoveryResponse) { + } + + rpc DeltaScopedRoutes(stream DeltaDiscoveryRequest) returns (stream DeltaDiscoveryResponse) { + } + + rpc FetchScopedRoutes(DiscoveryRequest) returns (DiscoveryResponse) { + option (google.api.http) = { + post: "/v2/discovery:scoped-routes" + body: "*" + }; + } +} + +// Specifies a routing scope, which associates a :ref:`envoy_api_msg_RouteConfiguration` (identified +// by its resource name) to a :ref:`Key`. +// +// The HTTP connection manager builds up a table consisting of these Key to RouteConfiguration +// mappings, and looks up the RouteConfiguration to use per request according to the algorithm +// specified in the +// :ref:`scope_key_builder` +// assigned to the HttpConnectionManager. +// +// For example, with the following configurations (in YAML): +// +// HttpConnectionManager config: +// +// .. code:: +// +// ... +// scope_key_builder: +// fragments: +// - header_element: +// name: X-Route-Selector +// element_separator: , +// element: +// separator: = +// key: vip +// +// ScopedRouteConfiguration resources (specified statically via +// HttpConnectionManager.ScopedRoutes.scoped_route_configurations_list or obtained dynamically via +// SRDS): +// +// .. code:: +// +// (1) +// name: route-scope1 +// route_configuration_name: route-config1 +// key: +// fragments: +// - string_key: 172.10.10.20 +// +// (2) +// name: route-scope2 +// route_configuration_name: route-config2 +// key: +// fragments: +// - string_key: 172.20.20.30 +// +// A request from a client such as: +// +// .. code:: +// +// GET / HTTP/1.1 +// Host: foo.com +// X-Route-Selector: vip=172.10.10.20 +// +// Would result in the routing table defined by the `route-config1` RouteConfiguration being +// assigned to the HTTP request/stream. +// +// [#comment:next free field: 4] +message ScopedRouteConfiguration { + // The name assigned to the routing scope. + string name = 1 [(validate.rules).string.min_bytes = 1]; + + // Specifies a key which is matched against the output of the + // :ref:`scope_key_builder` + // specified in the HttpConnectionManager. The matching is done per HTTP request and is dependent + // on the order of the fragments contained in the Key. + message Key { + message Fragment { + oneof type { + option (validate.required) = true; + + // A string to match against. + string string_key = 1; + } + } + + // The ordered set of fragments to match against. + repeated Fragment fragments = 1 [(validate.rules).repeated .min_items = 1]; + } + + // The resource name to use for a :ref:`envoy_api_msg_DiscoveryRequest` to an RDS server to + // fetch the :ref:`envoy_api_msg_RouteConfiguration` associated with this scope. + string route_configuration_name = 2 [(validate.rules).string.min_bytes = 1]; + + // The key to match against. + Key key = 3 [(validate.rules).message.required = true]; +} diff --git a/api/envoy/config/filter/network/http_connection_manager/v2/BUILD b/api/envoy/config/filter/network/http_connection_manager/v2/BUILD index 687780a833dee..95d3811f426af 100644 --- a/api/envoy/config/filter/network/http_connection_manager/v2/BUILD +++ b/api/envoy/config/filter/network/http_connection_manager/v2/BUILD @@ -7,6 +7,7 @@ api_proto_library_internal( srcs = ["http_connection_manager.proto"], deps = [ "//envoy/api/v2:rds", + "//envoy/api/v2:srds", "//envoy/api/v2/core:base", "//envoy/api/v2/core:config_source", "//envoy/api/v2/core:protocol", @@ -20,6 +21,7 @@ api_go_proto_library( proto = ":http_connection_manager", deps = [ "//envoy/api/v2:rds_go_grpc", + "//envoy/api/v2:srds_go_grpc", "//envoy/api/v2/core:base_go_proto", "//envoy/api/v2/core:config_source_go_proto", "//envoy/api/v2/core:protocol_go_proto", diff --git a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto index 18a479d3d7f97..b72125f43f7a3 100644 --- a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto +++ b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto @@ -10,6 +10,7 @@ option go_package = "v2"; import "envoy/api/v2/core/config_source.proto"; import "envoy/api/v2/core/protocol.proto"; import "envoy/api/v2/rds.proto"; +import "envoy/api/v2/srds.proto"; import "envoy/config/filter/accesslog/v2/accesslog.proto"; import "envoy/type/percent.proto"; @@ -24,7 +25,7 @@ import "gogoproto/gogo.proto"; // [#protodoc-title: HTTP connection manager] // HTTP connection manager :ref:`configuration overview `. -// [#comment:next free field: 31] +// [#comment:next free field: 32] message HttpConnectionManager { enum CodecType { option (gogoproto.goproto_enum_prefix) = false; @@ -61,6 +62,11 @@ message HttpConnectionManager { // The route table for the connection manager is static and is specified in this property. envoy.api.v2.RouteConfiguration route_config = 4; + + // A route table will be dynamically assigned to each request based on request attributes + // (e.g., the value of a header). The "routing scopes" (i.e., route tables) and "scope keys" are + // specified in this message. + ScopedRoutes scoped_routes = 31; } // A list of individual HTTP filters that make up the filter chain for @@ -419,6 +425,115 @@ message Rds { string route_config_name = 2 [(validate.rules).string.min_bytes = 1]; } +// This message is used to work around the limitations with 'oneof' and repeated fields. +message ScopedRouteConfigurationsList { + repeated envoy.api.v2.ScopedRouteConfiguration scoped_route_configurations = 1 + [(validate.rules).repeated .min_items = 1]; +} + +message ScopedRoutes { + // The name assigned to the scoped routing configuration. + string name = 1 [(validate.rules).string.min_bytes = 1]; + + // Specifies the mechanism for constructing "scope keys" based on HTTP request attributes. These + // keys are matched against a set of :ref:`Key` + // objects assembled from :ref:`ScopedRouteConfiguration` + // messages distributed via SRDS (the Scoped Route Discovery Service) or assigned statically via + // :ref:`scoped_route_configurations_list`. + // + // Upon receiving a request's headers, the Router will build a key using the algorithm specified + // by this message. This key will be used to look up the routing table (i.e., the + // :ref:`RouteConfiguration`) to use for the request. + message ScopeKeyBuilder { + // Specifies the mechanism for constructing key fragments which are composed into scope keys. + message FragmentBuilder { + // Specifies how the value of a header should be extracted. + // The following example maps the structure of a header to the fields in this message. + // + // .. code:: + // + // X-Header: a=b;c=d + // | || | + // | || \----> + // | || + // | |\----> + // | | + // | \----> + // | + // \----> + // + // Each 'a=b' key-value pair constitutes an 'element' of the header field. + message HeaderValueExtractor { + // The name of the header field to extract the value from. + string name = 1 [(validate.rules).string.min_bytes = 1]; + + // The element separator (e.g., ';' separates 'a;b;c;d'). + string element_separator = 2; + + // Specifies a header field's key value pair to match on. + message KvElement { + // The separator between key and value (e.g., '=' separates 'k=v;...'). + string separator = 1; + + // The key to match on. + string key = 2; + } + + oneof extract_type { + // Specifies the index of the element to extract. + int32 index = 3; + + // Specifies the key value pair to extract the value from. + KvElement element = 4; + } + } + + oneof type { + option (validate.required) = true; + + // Specifies how a header field's value should be extracted. + HeaderValueExtractor header_value_extractor = 1; + } + } + + // The final scope key consists of the ordered union of these fragments. + repeated FragmentBuilder fragments = 1 [(validate.rules).repeated .min_items = 1]; + } + + // The algorithm to use for constructing a scope key for each request. + ScopeKeyBuilder scope_key_builder = 2 [(validate.rules).message.required = true]; + + // Configuration source specifier for RDS. + // This config source is used to subscribe to RouteConfiguration resources specified in + // ScopedRouteConfiguration messages. + envoy.api.v2.core.ConfigSource rds_config_source = 3 + [(validate.rules).message.required = true, (gogoproto.nullable) = false]; + + oneof config_specifier { + option (validate.required) = true; + + // The set of routing scopes corresponding to the HCM. A scope is assigned to a request by + // matching a key constructed from the request's attributes according to the algorithm specified + // by the + // :ref:`ScopeKeyBuilder` + // in this message. + ScopedRouteConfigurationsList scoped_route_configurations_list = 4; + + // The set of routing scopes associated with the HCM will be dynamically loaded via the SRDS + // API. A scope is assigned to a request by matching a key constructed from the request's + // attributes according to the algorithm specified by the + // :ref:`ScopeKeyBuilder` + // in this message. + ScopedRds scoped_rds = 5; + } +} + +message ScopedRds { + // Configuration source specifier for scoped RDS. + envoy.api.v2.core.ConfigSource scoped_rds_config_source = 1 + [(validate.rules).message.required = true, (gogoproto.nullable) = false]; +} + message HttpFilter { // The name of the filter to instantiate. The name must match a supported // filter. The built-in filters are: diff --git a/docs/build.sh b/docs/build.sh index 6d6a88c2bb7a5..0788ac4fbed36 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -76,6 +76,7 @@ PROTO_RST=" /envoy/api/v2/cluster/circuit_breaker/envoy/api/v2/cluster/circuit_breaker.proto.rst /envoy/api/v2/rds/envoy/api/v2/rds.proto.rst /envoy/api/v2/route/route/envoy/api/v2/route/route.proto.rst + /envoy/api/v2/srds/envoy/api/v2/srds.proto.rst /envoy/api/v2/lds/envoy/api/v2/lds.proto.rst /envoy/api/v2/listener/listener/envoy/api/v2/listener/listener.proto.rst /envoy/api/v2/ratelimit/ratelimit/envoy/api/v2/ratelimit/ratelimit.proto.rst diff --git a/docs/root/api-v2/http_routes/http_routes.rst b/docs/root/api-v2/http_routes/http_routes.rst index 45a2dbca1d930..241f94601a5c8 100644 --- a/docs/root/api-v2/http_routes/http_routes.rst +++ b/docs/root/api-v2/http_routes/http_routes.rst @@ -6,4 +6,5 @@ HTTP route management :maxdepth: 2 ../api/v2/rds.proto + ../api/v2/srds.proto ../api/v2/route/route.proto diff --git a/include/envoy/config/BUILD b/include/envoy/config/BUILD index 8c16805bf754f..777cf7a32fb25 100644 --- a/include/envoy/config/BUILD +++ b/include/envoy/config/BUILD @@ -14,6 +14,7 @@ envoy_cc_library( external_deps = ["abseil_optional"], deps = [ "//include/envoy/common:time_interface", + "//source/common/common:assert_lib", "//source/common/protobuf", ], ) diff --git a/include/envoy/config/config_provider.h b/include/envoy/config/config_provider.h index a42f512dea4e9..0b22f74618dec 100644 --- a/include/envoy/config/config_provider.h +++ b/include/envoy/config/config_provider.h @@ -4,6 +4,7 @@ #include "envoy/common/time.h" +#include "common/common/assert.h" #include "common/protobuf/protobuf.h" #include "absl/types/optional.h" @@ -41,6 +42,23 @@ class ConfigProvider { }; using ConfigConstSharedPtr = std::shared_ptr; + /** + * The type of API represented by a ConfigProvider. + */ + enum class ApiType { + /** + * A "Full" API delivers a complete configuration as part of each resource (top level + * config proto); i.e., each resource contains the whole representation of the config intent. An + * example of this type of API is RDS. + */ + Full, + /** + * A "Delta" API delivers a subset of the config intent as part of each resource (top level + * config proto). Examples of this type of API are CDS, LDS and SRDS. + */ + Delta + }; + /** * Stores the config proto as well as the associated version. */ @@ -51,10 +69,26 @@ class ConfigProvider { std::string version_; }; + using ConfigProtoVector = std::vector; + /** + * Stores the config protos associated with a "Delta" API. + */ + struct ConfigProtoInfoVector { + const ConfigProtoVector config_protos_; + + // Only populated by dynamic config providers. + std::string version_; + }; + virtual ~ConfigProvider() = default; /** - * Returns a ConfigProtoInfo associated with the provider. + * The type of API. + */ + virtual ApiType apiType() const PURE; + + /** + * Returns a ConfigProtoInfo associated with a ApiType::Full provider. * @return absl::optional> an optional ConfigProtoInfo; the value is set when a * config is available. */ @@ -69,6 +103,19 @@ class ConfigProvider { return ConfigProtoInfo

{*config_proto, getConfigVersion()}; } + /** + * Returns a ConfigProtoInfoVector associated with a ApiType::Delta provider. + * @return absl::optional an optional ConfigProtoInfoVector; the value is + * set when a config is available. + */ + absl::optional configProtoInfoVector() const { + const ConfigProtoVector config_protos = getConfigProtos(); + if (config_protos.empty()) { + return absl::nullopt; + } + return ConfigProtoInfoVector{std::move(config_protos), getConfigVersion()}; + } + /** * Returns the Config corresponding to the provider. * @return std::shared_ptr a shared pointer to the Config. @@ -92,13 +139,20 @@ class ConfigProvider { * @return Protobuf::Message* the config proto corresponding to the Config instantiated by the * provider. */ - virtual const Protobuf::Message* getConfigProto() const PURE; + virtual const Protobuf::Message* getConfigProto() const { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } + + /** + * Returns the config protos associated with the provider. + * @return const ConfigProtoVector the config protos corresponding to the Config instantiated by + * the provider. + */ + virtual const ConfigProtoVector getConfigProtos() const { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } /** * Returns the config version associated with the provider. * @return std::string the config version. */ - virtual std::string getConfigVersion() const PURE; + virtual std::string getConfigVersion() const { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } /** * Returns the config implementation associated with the provider. diff --git a/include/envoy/config/config_provider_manager.h b/include/envoy/config/config_provider_manager.h index 3c3eddba9c2f0..48aaaea5516dd 100644 --- a/include/envoy/config/config_provider_manager.h +++ b/include/envoy/config/config_provider_manager.h @@ -25,6 +25,17 @@ namespace Config { */ class ConfigProviderManager { public: + class OptionalArg { + public: + virtual ~OptionalArg() = default; + }; + + class NullOptionalArg : public OptionalArg { + public: + NullOptionalArg() = default; + ~NullOptionalArg() override = default; + }; + virtual ~ConfigProviderManager() = default; /** @@ -34,6 +45,7 @@ class ConfigProviderManager { * @param config_source_proto supplies the proto containing the xDS API configuration. * @param factory_context is the context to use for the provider. * @param stat_prefix supplies the prefix to use for statistics. + * @param optarg supplies an optional argument with data specific to the concrete class. * @return ConfigProviderPtr a newly allocated dynamic config provider which shares underlying * data structures with other dynamic providers configured with the same * API source. @@ -41,17 +53,32 @@ class ConfigProviderManager { virtual ConfigProviderPtr createXdsConfigProvider(const Protobuf::Message& config_source_proto, Server::Configuration::FactoryContext& factory_context, - const std::string& stat_prefix) PURE; + const std::string& stat_prefix, const OptionalArg& optarg) PURE; /** * Returns a ConfigProvider associated with a statically specified configuration. * @param config_proto supplies the configuration proto. * @param factory_context is the context to use for the provider. + * @param optarg supplies an optional argument with data specific to the concrete class. * @return ConfigProviderPtr a newly allocated static config provider. */ virtual ConfigProviderPtr createStaticConfigProvider(const Protobuf::Message& config_proto, - Server::Configuration::FactoryContext& factory_context) PURE; + Server::Configuration::FactoryContext& factory_context, + const OptionalArg& optarg) PURE; + + /** + * Returns a ConfigProvider associated with a statically specified configuration. This is intended + * to be used when a set of configuration protos is required to build the full configuration. + * @param config_protos supplies a vector of configuration protos. + * @param factory_context is the context to use for the provider. + * @param optarg supplies an optional argument with data specific to the concrete class. + * @return ConfigProviderPtr a newly allocated static config provider. + */ + virtual ConfigProviderPtr + createStaticConfigProvider(std::vector>&& config_protos, + Server::Configuration::FactoryContext& factory_context, + const OptionalArg& optarg) PURE; }; } // namespace Config diff --git a/include/envoy/router/BUILD b/include/envoy/router/BUILD index 1952414e09d19..4bbbb3f298500 100644 --- a/include/envoy/router/BUILD +++ b/include/envoy/router/BUILD @@ -50,6 +50,16 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "scopes_interface", + hdrs = ["scopes.h"], + deps = [ + ":router_interface", + "//include/envoy/config:config_provider_interface", + "//include/envoy/http:header_map_interface", + ], +) + envoy_cc_library( name = "router_ratelimit_interface", hdrs = ["router_ratelimit.h"], diff --git a/include/envoy/router/scopes.h b/include/envoy/router/scopes.h new file mode 100644 index 0000000000000..d3207036f5f1c --- /dev/null +++ b/include/envoy/router/scopes.h @@ -0,0 +1,30 @@ +#pragma once + +#include + +#include "envoy/config/config_provider.h" +#include "envoy/router/router.h" + +namespace Envoy { +namespace Router { + +/** + * The scoped routing configuration. + */ +class ScopedConfig : public Envoy::Config::ConfigProvider::Config { +public: + ~ScopedConfig() override = default; + + /** + * Based on the incoming HTTP request headers, returns the configuration to use for selecting a + * target route. + * @param headers the request headers to match the scoped routing configuration against. + * @return ConfigConstSharedPtr the router's Config matching the request headers. + */ + virtual ConfigConstSharedPtr getRouterConfig(const Http::HeaderMap& headers) const PURE; +}; + +using ScopedConfigConstSharedPtr = std::shared_ptr; + +} // namespace Router +} // namespace Envoy diff --git a/source/common/config/config_provider_impl.cc b/source/common/config/config_provider_impl.cc index da3d65043a96d..4ee6e2289f346 100644 --- a/source/common/config/config_provider_impl.cc +++ b/source/common/config/config_provider_impl.cc @@ -3,29 +3,51 @@ namespace Envoy { namespace Config { -ImmutableConfigProviderImplBase::ImmutableConfigProviderImplBase( +ImmutableConfigProviderBase::ImmutableConfigProviderBase( Server::Configuration::FactoryContext& factory_context, - ConfigProviderManagerImplBase& config_provider_manager, ConfigProviderInstanceType type) + ConfigProviderManagerImplBase& config_provider_manager, + ConfigProviderInstanceType instance_type, ApiType api_type) : last_updated_(factory_context.timeSource().systemTime()), - config_provider_manager_(config_provider_manager), type_(type) { + config_provider_manager_(config_provider_manager), instance_type_(instance_type), + api_type_(api_type) { config_provider_manager_.bindImmutableConfigProvider(this); } -ImmutableConfigProviderImplBase::~ImmutableConfigProviderImplBase() { +ImmutableConfigProviderBase::~ImmutableConfigProviderBase() { config_provider_manager_.unbindImmutableConfigProvider(this); } -ConfigSubscriptionInstanceBase::~ConfigSubscriptionInstanceBase() { +ConfigSubscriptionCommonBase::~ConfigSubscriptionCommonBase() { init_target_.ready(); config_provider_manager_.unbindSubscription(manager_identifier_); } -bool ConfigSubscriptionInstanceBase::checkAndApplyConfig(const Protobuf::Message& config_proto, - const std::string& config_name, - const std::string& version_info) { +void ConfigSubscriptionCommonBase::bindConfigProvider(MutableConfigProviderCommonBase* provider) { + // All config providers bound to a ConfigSubscriptionCommonBase must be of the same concrete + // type; this is assumed by ConfigSubscriptionInstance::checkAndApplyConfigUpdate() and is + // verified by the assertion below. NOTE: an inlined statement ASSERT() triggers a potentially + // evaluated expression warning from clang due to `typeid(**mutable_config_providers_.begin())`. + // To avoid this, we use a lambda to separate the first mutable provider dereference from the + // typeid() statement. + ASSERT([&]() { + if (!mutable_config_providers_.empty()) { + const auto& first_provider = **mutable_config_providers_.begin(); + return typeid(*provider) == typeid(first_provider); + } + return true; + }()); + mutable_config_providers_.insert(provider); +} + +bool ConfigSubscriptionInstance::checkAndApplyConfigUpdate(const Protobuf::Message& config_proto, + const std::string& config_name, + const std::string& version_info) { const uint64_t new_hash = MessageUtil::hash(config_proto); - if (config_info_ && config_info_.value().last_config_hash_ == new_hash) { - return false; + if (config_info_) { + ASSERT(config_info_.value().last_config_hash_.has_value()); + if (config_info_.value().last_config_hash_.value() == new_hash) { + return false; + } } config_info_ = {new_hash, version_info}; @@ -39,31 +61,25 @@ bool ConfigSubscriptionInstanceBase::checkAndApplyConfig(const Protobuf::Message // bindConfigProvider()). // This makes it safe to call any of the provider's onConfigProtoUpdate() to get a new config // impl, which can then be passed to all providers. + auto* typed_provider = static_cast(provider); if (new_config == nullptr) { - if ((new_config = provider->onConfigProtoUpdate(config_proto)) == nullptr) { + if ((new_config = typed_provider->onConfigProtoUpdate(config_proto)) == nullptr) { return false; } } - provider->onConfigUpdate(new_config); + typed_provider->onConfigUpdate(new_config); } return true; } -void ConfigSubscriptionInstanceBase::bindConfigProvider(MutableConfigProviderImplBase* provider) { - // All config providers bound to a ConfigSubscriptionInstanceBase must be of the same concrete - // type; this is assumed by checkAndApplyConfig() and is verified by the assertion below. - // NOTE: an inlined statement ASSERT() triggers a potentially evaluated expression warning from - // clang due to `typeid(**mutable_config_providers_.begin())`. To avoid this, we use a lambda to - // separate the first mutable provider dereference from the typeid() statement. - ASSERT([&]() { - if (!mutable_config_providers_.empty()) { - const auto& first_provider = **mutable_config_providers_.begin(); - return typeid(*provider) == typeid(first_provider); - } - return true; - }()); - mutable_config_providers_.insert(provider); +void DeltaConfigSubscriptionInstance::applyConfigUpdate( + const std::function& updateFn) { + for (auto* provider : mutable_config_providers_) { + auto* typed_provider = static_cast(provider); + ConfigProvider::ConfigConstSharedPtr config = typed_provider->getConfig(); + typed_provider->onConfigUpdate([config, updateFn]() { updateFn(config); }); + } } ConfigProviderManagerImplBase::ConfigProviderManagerImplBase(Server::Admin& admin, @@ -87,14 +103,14 @@ ConfigProviderManagerImplBase::immutableConfigProviders(ConfigProviderInstanceTy } void ConfigProviderManagerImplBase::bindImmutableConfigProvider( - ImmutableConfigProviderImplBase* provider) { - ASSERT(provider->type() == ConfigProviderInstanceType::Static || - provider->type() == ConfigProviderInstanceType::Inline); + ImmutableConfigProviderBase* provider) { + ASSERT(provider->instanceType() == ConfigProviderInstanceType::Static || + provider->instanceType() == ConfigProviderInstanceType::Inline); ConfigProviderMap::iterator it; - if ((it = immutable_config_providers_map_.find(provider->type())) == + if ((it = immutable_config_providers_map_.find(provider->instanceType())) == immutable_config_providers_map_.end()) { immutable_config_providers_map_.insert(std::make_pair( - provider->type(), + provider->instanceType(), std::make_unique(std::initializer_list({provider})))); } else { it->second->insert(provider); @@ -102,10 +118,10 @@ void ConfigProviderManagerImplBase::bindImmutableConfigProvider( } void ConfigProviderManagerImplBase::unbindImmutableConfigProvider( - ImmutableConfigProviderImplBase* provider) { - ASSERT(provider->type() == ConfigProviderInstanceType::Static || - provider->type() == ConfigProviderInstanceType::Inline); - auto it = immutable_config_providers_map_.find(provider->type()); + ImmutableConfigProviderBase* provider) { + ASSERT(provider->instanceType() == ConfigProviderInstanceType::Static || + provider->instanceType() == ConfigProviderInstanceType::Inline); + auto it = immutable_config_providers_map_.find(provider->instanceType()); ASSERT(it != immutable_config_providers_map_.end()); it->second->erase(provider); } diff --git a/source/common/config/config_provider_impl.h b/source/common/config/config_provider_impl.h index 50e916d1d0348..0ef64d0c2b8e1 100644 --- a/source/common/config/config_provider_impl.h +++ b/source/common/config/config_provider_impl.h @@ -19,14 +19,15 @@ namespace Envoy { namespace Config { -// This file provides a set of base classes, (ImmutableConfigProviderImplBase, -// MutableConfigProviderImplBase, ConfigProviderManagerImplBase, ConfigSubscriptionInstanceBase), -// conforming to the ConfigProvider/ConfigProviderManager interfaces, which in tandem provide a -// framework for implementing statically defined (i.e., immutable) and dynamic (mutable via -// subscriptions) configuration for Envoy. +// This file provides a set of base classes, (ImmutableConfigProviderBase, +// MutableConfigProviderCommonBase, MutableConfigProviderBase, DeltaMutableConfigProviderBase, +// ConfigProviderManagerImplBase, ConfigSubscriptionCommonBase, ConfigSubscriptionInstance, +// DeltaConfigSubscriptionInstance), conforming to the ConfigProvider/ConfigProviderManager +// interfaces, which in tandem provide a framework for implementing statically defined (i.e., +// immutable) and dynamic (mutable via subscriptions) configuration for Envoy. // // The mutability property applies to the ConfigProvider itself and _not_ the underlying config -// proto, which is always immutable. MutableConfigProviderImplBase objects receive config proto +// proto, which is always immutable. MutableConfigProviderCommonBase objects receive config proto // updates via xDS subscriptions, resulting in new ConfigProvider::Config objects being instantiated // with the corresponding change in behavior corresponding to updated config. ConfigProvider::Config // objects must be latched/associated with the appropriate objects in the connection and request @@ -49,27 +50,36 @@ namespace Config { // 1) Create a class derived from ConfigProviderManagerImplBase and implement the required // interface. // When implementing createXdsConfigProvider(), it is expected that getSubscription() will -// be called to fetch either an existing ConfigSubscriptionInstanceBase if the config source -// configuration matches, or a newly instantiated subscription otherwise. +// be called to fetch either an existing ConfigSubscriptionCommonBase if the config +// source configuration matches, or a newly instantiated subscription otherwise. // // For immutable providers: -// 1) Create a class derived from ImmutableConfigProviderImplBase and implement the required +// 1) Create a class derived from ImmutableConfigProviderBase and implement the required // interface. // // For mutable (xDS) providers: -// 1) Create a class derived from MutableConfigProviderImplBase and implement the required -// interface. -// 2) Create a class derived from ConfigSubscriptionInstanceBase; this is the entity -// responsible for owning and managing the Envoy::Config::Subscription that provides -// the underlying config subscription. +// 1) According to the API type, create a class derived from MutableConfigProviderBase or +// DeltaMutableConfigProviderBase and implement the required interface. +// 2) According to the API type, create a class derived from ConfigSubscriptionInstance or +// DeltaConfigSubscriptionInstance; this is the entity responsible for owning and managing the +// Envoy::Config::Subscription that provides the underlying config subscription. +// a) For a ConfigProvider::ApiType::Full subscription instance (i.e., a +// ConfigSubscriptionInstance child): // - When subscription callbacks (onConfigUpdate, onConfigUpdateFailed) are issued by the -// underlying subscription, the corresponding ConfigSubscriptionInstanceBase functions must be -// called as well. -// - On a successful config update, checkAndApplyConfig() should be called to instantiate the -// new config implementation and propagate it to the shared config providers and all -// worker threads. -// - On a successful return from checkAndApplyConfig(), the config proto must be latched into -// this class and returned via the getConfigProto() override. +// underlying subscription, the corresponding ConfigSubscriptionInstance functions +// must be called as well. +// - On a successful config update, checkAndApplyConfigUpdate() should be called to instantiate +// the new config implementation and propagate it to the shared config providers and all worker +// threads. +// - On a successful return from checkAndApplyConfigUpdate(), the config proto must be latched +// into this class and returned via the getConfigProto() override. +// b) For a ConfigProvider::ApiType::Delta subscription instance (i.e., a +// DeltaConfigSubscriptionInstance child): +// - When subscription callbacks (onConfigUpdate, onConfigUpdateFailed) are issued by the +// underlying subscription, the corresponding ConfigSubscriptionInstance functions must be called +// as well. +// - On a successful config update, applyConfigUpdate() should be called to propagate the config +// updates to all bound config providers and worker threads. class ConfigProviderManagerImplBase; @@ -90,38 +100,40 @@ enum class ConfigProviderInstanceType { * ConfigProvider implementation for immutable configuration. * * TODO(AndresGuedez): support sharing of config protos and config impls, as is - * done with the MutableConfigProviderImplBase. + * done with the MutableConfigProviderCommonBase. * * This class can not be instantiated directly; instead, it provides the foundation for * immutable config provider implementations which derive from it. */ -class ImmutableConfigProviderImplBase : public ConfigProvider { +class ImmutableConfigProviderBase : public ConfigProvider { public: - ~ImmutableConfigProviderImplBase() override; + ~ImmutableConfigProviderBase() override; // Envoy::Config::ConfigProvider SystemTime lastUpdated() const override { return last_updated_; } + ApiType apiType() const override { return api_type_; } - ConfigProviderInstanceType type() const { return type_; } + ConfigProviderInstanceType instanceType() const { return instance_type_; } protected: - ImmutableConfigProviderImplBase(Server::Configuration::FactoryContext& factory_context, - ConfigProviderManagerImplBase& config_provider_manager, - ConfigProviderInstanceType type); + ImmutableConfigProviderBase(Server::Configuration::FactoryContext& factory_context, + ConfigProviderManagerImplBase& config_provider_manager, + ConfigProviderInstanceType instance_type, ApiType api_type); private: SystemTime last_updated_; ConfigProviderManagerImplBase& config_provider_manager_; - ConfigProviderInstanceType type_; + ConfigProviderInstanceType instance_type_; + ApiType api_type_; }; -class MutableConfigProviderImplBase; +class MutableConfigProviderCommonBase; /** - * Provides generic functionality required by all xDS ConfigProvider subscriptions, including - * shared lifetime management via shared_ptr. + * Provides common DS API subscription functionality required by the ConfigProvider::ApiType + * specific base classes (see ConfigSubscriptionInstance and DeltaConfigSubscriptionInstance). * - * To do so, this class keeps track of a set of MutableConfigProviderImplBase instances associated + * To do so, this class keeps track of a set of MutableConfigProviderCommonBase instances associated * with an underlying subscription; providers are bound/unbound as needed as they are created and * destroyed. * @@ -134,14 +146,14 @@ class MutableConfigProviderImplBase; * This class can not be instantiated directly; instead, it provides the foundation for * config subscription implementations which derive from it. */ -class ConfigSubscriptionInstanceBase : protected Logger::Loggable { +class ConfigSubscriptionCommonBase : protected Logger::Loggable { public: struct LastConfigInfo { - uint64_t last_config_hash_; + absl::optional last_config_hash_; std::string last_config_version_; }; - virtual ~ConfigSubscriptionInstanceBase(); + virtual ~ConfigSubscriptionCommonBase(); /** * Starts the subscription corresponding to a config source. @@ -167,35 +179,27 @@ class ConfigSubscriptionInstanceBase : protected Logger::Loggable&& config_info) { + config_info_ = std::move(config_info); + } + + const std::string name_; + std::unordered_set mutable_config_providers_; + absl::optional config_info_; + private: - void bindConfigProvider(MutableConfigProviderImplBase* provider); + void bindConfigProvider(MutableConfigProviderCommonBase* provider); - void unbindConfigProvider(MutableConfigProviderImplBase* provider) { + void unbindConfigProvider(MutableConfigProviderCommonBase* provider) { mutable_config_providers_.erase(provider); } - const std::string name_; Init::TargetImpl init_target_; - std::unordered_set mutable_config_providers_; const uint64_t manager_identifier_; ConfigProviderManagerImplBase& config_provider_manager_; TimeSource& time_source_; SystemTime last_updated_; - absl::optional config_info_; - // ConfigSubscriptionInstanceBase, MutableConfigProviderImplBase and ConfigProviderManagerImplBase - // are tightly coupled with the current shared ownership model; use friend classes to explicitly - // denote the binding between them. + // ConfigSubscriptionCommonBase, MutableConfigProviderCommonBase and + // ConfigProviderManagerImplBase are tightly coupled with the current shared ownership model; use + // friend classes to explicitly denote the binding between them. // // TODO(AndresGuedez): Investigate whether a shared ownership model avoiding the s and // instead centralizing lifetime management in the ConfigProviderManagerImplBase with explicit // reference counting would be more maintainable. - friend class MutableConfigProviderImplBase; + friend class MutableConfigProviderCommonBase; + friend class MutableConfigProviderBase; + friend class DeltaMutableConfigProviderBase; friend class ConfigProviderManagerImplBase; }; -using ConfigSubscriptionInstanceBaseSharedPtr = std::shared_ptr; +using ConfigSubscriptionCommonBaseSharedPtr = std::shared_ptr; + +/** + * Provides common subscription functionality required by ConfigProvider::ApiType::Full DS APIs. + */ +class ConfigSubscriptionInstance : public ConfigSubscriptionCommonBase { +protected: + ConfigSubscriptionInstance(const std::string& name, const uint64_t manager_identifier, + ConfigProviderManagerImplBase& config_provider_manager, + TimeSource& time_source, const SystemTime& last_updated, + const LocalInfo::LocalInfo& local_info) + : ConfigSubscriptionCommonBase(name, manager_identifier, config_provider_manager, time_source, + last_updated, local_info) {} + + ~ConfigSubscriptionInstance() override = default; + + /** + * Determines whether a configuration proto is a new update, and if so, propagates it to all + * config providers associated with this subscription. + * @param config_proto supplies the newly received config proto. + * @param config_name supplies the name associated with the config. + * @param version_info supplies the version associated with the config. + * @return bool false when the config proto has no delta from the previous config, true otherwise. + */ + bool checkAndApplyConfigUpdate(const Protobuf::Message& config_proto, + const std::string& config_name, const std::string& version_info); +}; + +/** + * Provides common subscription functionality required by ConfigProvider::ApiType::Delta DS APIs. + */ +class DeltaConfigSubscriptionInstance : public ConfigSubscriptionCommonBase { +protected: + DeltaConfigSubscriptionInstance(const std::string& name, const uint64_t manager_identifier, + ConfigProviderManagerImplBase& config_provider_manager, + TimeSource& time_source, const SystemTime& last_updated, + const LocalInfo::LocalInfo& local_info) + : ConfigSubscriptionCommonBase(name, manager_identifier, config_provider_manager, time_source, + last_updated, local_info) {} + + ~DeltaConfigSubscriptionInstance() override = default; + + /** + * Propagates a config update to all config providers and worker threads associated with the + * subscription. + * + * @param updateFn the callback to run on each provider and worker thread. + */ + void applyConfigUpdate( + const std::function& updateFn); +}; /** - * Provides generic functionality required by all dynamic config providers, including distribution - * of config updates to all workers. + * Provides generic functionality required by the ConfigProvider::ApiType specific dynamic config + * providers (see MutableConfigProviderBase and DeltaMutableConfigProviderBase). * * This class can not be instantiated directly; instead, it provides the foundation for * dynamic config provider implementations which derive from it. */ -class MutableConfigProviderImplBase : public ConfigProvider { +class MutableConfigProviderCommonBase : public ConfigProvider { public: - ~MutableConfigProviderImplBase() override { subscription_->unbindConfigProvider(this); } + ~MutableConfigProviderCommonBase() override { subscription_->unbindConfigProvider(this); } // Envoy::Config::ConfigProvider SystemTime lastUpdated() const override { return subscription_->lastUpdated(); } + ApiType apiType() const override { return api_type_; } +protected: + MutableConfigProviderCommonBase(ConfigSubscriptionCommonBaseSharedPtr&& subscription, + Server::Configuration::FactoryContext& factory_context, + ApiType api_type) + : tls_(factory_context.threadLocal().allocateSlot()), subscription_(subscription), + api_type_(api_type) {} + + ThreadLocal::SlotPtr tls_; + ConfigSubscriptionCommonBaseSharedPtr subscription_; + +private: + ApiType api_type_; +}; + +/** + * Provides common mutable (dynamic) config provider functionality required by + * ConfigProvider::ApiType::Full DS APIs. + */ +class MutableConfigProviderBase : public MutableConfigProviderCommonBase { +public: // Envoy::Config::ConfigProvider + // NOTE: This is being promoted to public for internal uses to avoid an unnecessary dynamic_cast + // in the public API (ConfigProvider::config()). ConfigConstSharedPtr getConfig() const override { return tls_->getTyped().config_; } @@ -280,16 +364,20 @@ class MutableConfigProviderImplBase : public ConfigProvider { * @param config supplies the newly instantiated config. */ void onConfigUpdate(const ConfigConstSharedPtr& config) { + if (getConfig() == config) { + return; + } tls_->runOnAllThreads( [this, config]() -> void { tls_->getTyped().config_ = config; }); } protected: - MutableConfigProviderImplBase(ConfigSubscriptionInstanceBaseSharedPtr&& subscription, - Server::Configuration::FactoryContext& factory_context) - : subscription_(subscription), tls_(factory_context.threadLocal().allocateSlot()) {} + MutableConfigProviderBase(ConfigSubscriptionCommonBaseSharedPtr&& subscription, + Server::Configuration::FactoryContext& factory_context, + ApiType api_type) + : MutableConfigProviderCommonBase(std::move(subscription), factory_context, api_type) {} - const ConfigSubscriptionInstanceBaseSharedPtr& subscription() const { return subscription_; } + ~MutableConfigProviderBase() override = default; private: struct ThreadLocalConfig : public ThreadLocal::ThreadLocalObject { @@ -298,9 +386,42 @@ class MutableConfigProviderImplBase : public ConfigProvider { ConfigProvider::ConfigConstSharedPtr config_; }; +}; - ConfigSubscriptionInstanceBaseSharedPtr subscription_; - ThreadLocal::SlotPtr tls_; +/** + * Provides common mutable (dynamic) config provider functionality required by + * ConfigProvider::ApiType::Delta DS APIs. + */ +class DeltaMutableConfigProviderBase : public MutableConfigProviderCommonBase { +public: + // Envoy::Config::ConfigProvider + // This promotes getConfig() to public so that internal uses can avoid an unnecessary dynamic_cast + // in the public API (ConfigProvider::config()). + ConfigConstSharedPtr getConfig() const override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } + + /** + * Propagates a delta config update to all workers. + * @param updateCb the callback to run on each worker. + */ + void onConfigUpdate(Envoy::Event::PostCb updateCb) { tls_->runOnAllThreads(std::move(updateCb)); } + +protected: + DeltaMutableConfigProviderBase(ConfigSubscriptionCommonBaseSharedPtr&& subscription, + Server::Configuration::FactoryContext& factory_context, + ApiType api_type) + : MutableConfigProviderCommonBase(std::move(subscription), factory_context, api_type) {} + + ~DeltaMutableConfigProviderBase() override = default; + + /** + * Must be called by the derived class' constructor. + * @param initializeCb supplies the initialization callback to be issued for each worker + * thread. + */ + void initialize(ThreadLocal::Slot::InitializeCb initializeCb) { + subscription_->bindConfigProvider(this); + tls_->set(std::move(initializeCb)); + } }; /** @@ -308,9 +429,9 @@ class MutableConfigProviderImplBase : public ConfigProvider { * lifetime of subscriptions and dynamic config providers, along with determining which * subscriptions should be associated with newly instantiated providers. * - * The implementation of this class is not thread safe. Note that ImmutableConfigProviderImplBase - * and ConfigSubscriptionInstanceBase call the corresponding {bind,unbind}* functions exposed by - * this class. + * The implementation of this class is not thread safe. Note that ImmutableConfigProviderBase + * and ConfigSubscriptionCommonBase call the corresponding {bind,unbind}* functions exposed + * by this class. * * All config processing is done on the main thread, so instantiation of *ConfigProvider* objects * via createStaticConfigProvider() and createXdsConfigProvider() is naturally thread safe. Care @@ -337,14 +458,14 @@ class ConfigProviderManagerImplBase : public ConfigProviderManager, public Singl using ConfigProviderMap = std::unordered_map, EnumClassHash>; using ConfigSubscriptionMap = - std::unordered_map>; + std::unordered_map>; ConfigProviderManagerImplBase(Server::Admin& admin, const std::string& config_name); const ConfigSubscriptionMap& configSubscriptions() const { return config_subscriptions_; } /** - * Returns the set of bound ImmutableConfigProviderImplBase-derived providers of a given type. + * Returns the set of bound ImmutableConfigProviderBase-derived providers of a given type. * @param type supplies the type of config providers to return. * @return const ConfigProviderSet* the set of config providers corresponding to the type. */ @@ -362,12 +483,12 @@ class ConfigProviderManagerImplBase : public ConfigProviderManager, public Singl template std::shared_ptr getSubscription(const Protobuf::Message& config_source_proto, Init::Manager& init_manager, - const std::function& subscription_factory_fn) { - static_assert(std::is_base_of::value, - "T must be a subclass of ConfigSubscriptionInstanceBase"); + static_assert(std::is_base_of::value, + "T must be a subclass of ConfigSubscriptionCommonBase"); - ConfigSubscriptionInstanceBaseSharedPtr subscription; + ConfigSubscriptionCommonBaseSharedPtr subscription; const uint64_t manager_identifier = MessageUtil::hash(config_source_proto); auto it = config_subscriptions_.find(manager_identifier); @@ -381,7 +502,7 @@ class ConfigProviderManagerImplBase : public ConfigProviderManager, public Singl bindSubscription(manager_identifier, subscription); } else { // Because the ConfigProviderManagerImplBase's weak_ptrs only get cleaned up - // in the ConfigSubscriptionInstanceBase destructor, and the single threaded nature + // in the ConfigSubscriptionCommonBase destructor, and the single threaded nature // of this code, locking the weak_ptr will not fail. subscription = it->second.lock(); } @@ -392,7 +513,7 @@ class ConfigProviderManagerImplBase : public ConfigProviderManager, public Singl private: void bindSubscription(const uint64_t manager_identifier, - ConfigSubscriptionInstanceBaseSharedPtr& subscription) { + ConfigSubscriptionCommonBaseSharedPtr& subscription) { config_subscriptions_.insert({manager_identifier, subscription}); } @@ -400,8 +521,8 @@ class ConfigProviderManagerImplBase : public ConfigProviderManager, public Singl config_subscriptions_.erase(manager_identifier); } - void bindImmutableConfigProvider(ImmutableConfigProviderImplBase* provider); - void unbindImmutableConfigProvider(ImmutableConfigProviderImplBase* provider); + void bindImmutableConfigProvider(ImmutableConfigProviderBase* provider); + void unbindImmutableConfigProvider(ImmutableConfigProviderBase* provider); // TODO(jsedgwick) These two members are prime candidates for the owned-entry list/map // as in ConfigTracker. I.e. the ProviderImpls would have an EntryOwner for these lists @@ -411,10 +532,10 @@ class ConfigProviderManagerImplBase : public ConfigProviderManager, public Singl Server::ConfigTracker::EntryOwnerPtr config_tracker_entry_; - // See comment for friend classes in the ConfigSubscriptionInstanceBase for more details on the - // use of friends. - friend class ConfigSubscriptionInstanceBase; - friend class ImmutableConfigProviderImplBase; + // See comment for friend classes in the ConfigSubscriptionCommonBase for more details on + // the use of friends. + friend class ConfigSubscriptionCommonBase; + friend class ImmutableConfigProviderBase; }; } // namespace Config diff --git a/source/common/config/resources.h b/source/common/config/resources.h index 69ed2d91a46dc..b3768a7d9206d 100644 --- a/source/common/config/resources.h +++ b/source/common/config/resources.h @@ -17,6 +17,8 @@ class TypeUrlValues { const std::string ClusterLoadAssignment{"type.googleapis.com/envoy.api.v2.ClusterLoadAssignment"}; const std::string Secret{"type.googleapis.com/envoy.api.v2.auth.Secret"}; const std::string RouteConfiguration{"type.googleapis.com/envoy.api.v2.RouteConfiguration"}; + const std::string ScopedRouteConfiguration{ + "type.googleapis.com/envoy.api.v2.ScopedRouteConfiguration"}; }; typedef ConstSingleton TypeUrl; diff --git a/source/common/config/utility.h b/source/common/config/utility.h index 5dacc67bfa7a5..ff14809cbff59 100644 --- a/source/common/config/utility.h +++ b/source/common/config/utility.h @@ -1,5 +1,7 @@ #pragma once +#include + #include "envoy/api/api.h" #include "envoy/api/v2/core/base.pb.h" #include "envoy/config/bootstrap/v2/bootstrap.pb.h" @@ -329,6 +331,41 @@ class Utility { static void translateOpaqueConfig(const ProtobufWkt::Any& typed_config, const ProtobufWkt::Struct& config, Protobuf::Message& out_proto); + +#if 0 + template + static bool diffResourcesAndUpdateConfig( + const std::string& api_name, const ResourceVector& resources, const std::string& version_info, + std::function addOrUpdateFn, + std::function removeFn, + std::function validateFn) { + std::unordered_set resource_names; + for (const auto& resource : resources) { + if (!resource_names.insert(resource.name()).second) { + throw EnvoyException(fmt::format("duplicate resource {} found", resource.name())); + } + } + for (const auto& resource : resources) { + validateFn(resource); + } + // We need to keep track of which clusters we might need to remove. + ClusterManager::ClusterInfoMap clusters_to_remove = cm_.clusters(); + for (auto& cluster : resources) { + const std::string cluster_name = cluster.name(); + clusters_to_remove.erase(cluster_name); + if (cm_.addOrUpdateCluster(cluster, version_info)) { + ENVOY_LOG(debug, "cds: add/update cluster '{}'", cluster_name); + } + } + + for (auto cluster : clusters_to_remove) { + const std::string cluster_name = cluster.first; + if (cm_.removeCluster(cluster_name)) { + ENVOY_LOG(debug, "cds: remove cluster '{}'", cluster_name); + } + } + } +#endif }; } // namespace Config diff --git a/source/common/http/BUILD b/source/common/http/BUILD index b947d3b5293d4..4d207738e6d67 100644 --- a/source/common/http/BUILD +++ b/source/common/http/BUILD @@ -119,6 +119,7 @@ envoy_cc_library( hdrs = ["conn_manager_config.h"], deps = [ ":date_provider_lib", + "//include/envoy/config:config_provider_interface", "//include/envoy/http:filter_interface", "//include/envoy/router:rds_interface", "//source/common/network:utility_lib", @@ -158,6 +159,7 @@ envoy_cc_library( "//include/envoy/network:drain_decision_interface", "//include/envoy/network:filter_interface", "//include/envoy/router:rds_interface", + "//include/envoy/router:scopes_interface", "//include/envoy/runtime:runtime_interface", "//include/envoy/server:overload_manager_interface", "//include/envoy/ssl:connection_interface", diff --git a/source/common/http/conn_manager_config.h b/source/common/http/conn_manager_config.h index 2fb4ee9e9c575..64c97d045e7f2 100644 --- a/source/common/http/conn_manager_config.h +++ b/source/common/http/conn_manager_config.h @@ -1,5 +1,6 @@ #pragma once +#include "envoy/config/config_provider.h" #include "envoy/http/filter.h" #include "envoy/router/rds.h" #include "envoy/stats/scope.h" @@ -247,10 +248,17 @@ class ConnectionManagerConfig { virtual std::chrono::milliseconds delayedCloseTimeout() const PURE; /** - * @return Router::RouteConfigProvider& the configuration provider used to acquire a route + * @return Router::RouteConfigProvider* the configuration provider used to acquire a route * config for each request flow. */ - virtual Router::RouteConfigProvider& routeConfigProvider() PURE; + virtual Router::RouteConfigProvider* routeConfigProvider() PURE; + + /** + * @return Config::ConfigProvider* the configuration provider used to acquire scoped routing + * configuration for each request flow. Pointer ownership is _not_ transferred to the caller of + * this function. + */ + virtual Config::ConfigProvider* scopedRouteConfigProvider() PURE; /** * @return const std::string& the server name to write into responses. diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index 329758d835e1f..5b73766289f5a 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -399,11 +399,17 @@ void ConnectionManagerImpl::chargeTracingStats(const Tracing::Reason& tracing_re ConnectionManagerImpl::ActiveStream::ActiveStream(ConnectionManagerImpl& connection_manager) : connection_manager_(connection_manager), - snapped_route_config_(connection_manager.config_.routeConfigProvider().config()), stream_id_(connection_manager.random_generator_.random()), request_response_timespan_(new Stats::Timespan( connection_manager_.stats_.named_.downstream_rq_time_, connection_manager_.timeSource())), stream_info_(connection_manager_.codec_->protocol(), connection_manager_.timeSource()) { + if (connection_manager.config_.routeConfigProvider() != nullptr) { + snapped_route_config_ = connection_manager.config_.routeConfigProvider()->config(); + } + if (connection_manager.config_.scopedRouteConfigProvider() != nullptr) { + snapped_scoped_route_config_ = + connection_manager_.config_.scopedRouteConfigProvider()->config(); + } connection_manager_.stats_.named_.downstream_rq_total_.inc(); connection_manager_.stats_.named_.downstream_rq_active_.inc(); if (connection_manager_.codec_->protocol() == Protocol::Http2) { diff --git a/source/common/http/conn_manager_impl.h b/source/common/http/conn_manager_impl.h index a968d1109f7a4..ebe18aeab379e 100644 --- a/source/common/http/conn_manager_impl.h +++ b/source/common/http/conn_manager_impl.h @@ -18,6 +18,7 @@ #include "envoy/network/drain_decision.h" #include "envoy/network/filter.h" #include "envoy/router/rds.h" +#include "envoy/router/scopes.h" #include "envoy/runtime/runtime.h" #include "envoy/server/overload_manager.h" #include "envoy/ssl/connection.h" @@ -478,6 +479,7 @@ class ConnectionManagerImpl : Logger::Loggable, ConnectionManagerImpl& connection_manager_; Router::ConfigConstSharedPtr snapped_route_config_; + Router::ScopedConfigConstSharedPtr snapped_scoped_route_config_; Tracing::SpanPtr active_span_; const uint64_t stream_id_; StreamEncoder* response_encoder_{}; diff --git a/source/common/router/BUILD b/source/common/router/BUILD index 498babb17239e..bfb3351d232f7 100644 --- a/source/common/router/BUILD +++ b/source/common/router/BUILD @@ -93,6 +93,46 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "scoped_config_manager_lib", + srcs = ["scoped_config_manager.cc"], + hdrs = ["scoped_config_manager.h"], + deps = [ + "@envoy_api//envoy/api/v2:srds_cc", + ], +) + +envoy_cc_library( + name = "scoped_config_lib", + srcs = ["scoped_config_impl.cc"], + hdrs = ["scoped_config_impl.h"], + deps = [ + ":config_lib", + ":scoped_config_manager_lib", + "//include/envoy/router:scopes_interface", + "//include/envoy/thread_local:thread_local_interface", + "@envoy_api//envoy/api/v2:srds_cc", + "@envoy_api//envoy/config/filter/network/http_connection_manager/v2:http_connection_manager_cc", + ], +) + +envoy_cc_library( + name = "scoped_rds_lib", + srcs = ["scoped_rds.cc"], + hdrs = ["scoped_rds.h"], + deps = [ + ":scoped_config_lib", + "//include/envoy/config:subscription_interface", + "//include/envoy/stats:stats_interface", + "//source/common/common:assert_lib", + "//source/common/common:minimal_logger_lib", + "//source/common/config:config_provider_lib", + "//source/common/config:subscription_factory_lib", + "@envoy_api//envoy/admin/v2alpha:config_dump_cc", + "@envoy_api//envoy/api/v2:srds_cc", + ], +) + envoy_cc_library( name = "retry_state_lib", srcs = ["retry_state_impl.cc"], diff --git a/source/common/router/scoped_config_impl.cc b/source/common/router/scoped_config_impl.cc new file mode 100644 index 0000000000000..04b153176b041 --- /dev/null +++ b/source/common/router/scoped_config_impl.cc @@ -0,0 +1,16 @@ +#include "common/router/scoped_config_impl.h" + +namespace Envoy { +namespace Router { + +void ThreadLocalScopedConfigImpl::addOrUpdateRoutingScope(const ScopedRouteInfoConstSharedPtr&) {} + +void ThreadLocalScopedConfigImpl::removeRoutingScope(const std::string&) {} + +Router::ConfigConstSharedPtr +ThreadLocalScopedConfigImpl::getRouterConfig(const Http::HeaderMap&) const { + return std::make_shared(); +} + +} // namespace Router +} // namespace Envoy diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h new file mode 100644 index 0000000000000..8a812bc71ada9 --- /dev/null +++ b/source/common/router/scoped_config_impl.h @@ -0,0 +1,55 @@ +#pragma once + +#include "envoy/api/v2/srds.pb.h" +#include "envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.pb.h" +#include "envoy/router/router.h" +#include "envoy/router/scopes.h" +#include "envoy/thread_local/thread_local.h" + +#include "common/router/config_impl.h" +#include "common/router/scoped_config_manager.h" + +namespace Envoy { +namespace Router { + +/** + * TODO(AndresGuedez): implement scoped routing logic. + * + * Each Envoy worker is assigned an instance of this type. When config updates are received, + * addOrUpdateRoutingScope() and removeRoutingScope() are called to update the set of scoped routes. + * + * ConnectionManagerImpl::refreshCachedRoute() will call getRouterConfig() to obtain the + * Router::ConfigConstSharedPtr to use for route selection. + */ +class ThreadLocalScopedConfigImpl : public ScopedConfig, public ThreadLocal::ThreadLocalObject { +public: + ThreadLocalScopedConfigImpl( + envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder + scope_key_builder) + : scope_key_builder_(std::move(scope_key_builder)) {} + + ~ThreadLocalScopedConfigImpl() override = default; + + void addOrUpdateRoutingScope(const ScopedRouteInfoConstSharedPtr& scoped_route_info); + void removeRoutingScope(const std::string& scope_name); + + // Envoy::Router::ScopedConfig + Router::ConfigConstSharedPtr getRouterConfig(const Http::HeaderMap& headers) const override; + +private: + const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder + scope_key_builder_; +}; + +/** + * A NULL implementation of the scoped routing configuration. + */ +class NullScopedConfigImpl : public ScopedConfig { +public: + Router::ConfigConstSharedPtr getRouterConfig(const Http::HeaderMap&) const override { + return std::make_shared(); + } +}; + +} // namespace Router +} // namespace Envoy diff --git a/source/common/router/scoped_config_manager.cc b/source/common/router/scoped_config_manager.cc new file mode 100644 index 0000000000000..ee92a58dc4778 --- /dev/null +++ b/source/common/router/scoped_config_manager.cc @@ -0,0 +1,25 @@ +#include "common/router/scoped_config_manager.h" + +#include "envoy/common/exception.h" + +#include "common/common/fmt.h" + +namespace Envoy { +namespace Router { + +ScopedRouteInfoConstSharedPtr ScopedConfigManager::addOrUpdateRoutingScope( + const envoy::api::v2::ScopedRouteConfiguration& config_proto, const std::string&) { + auto scoped_route_info = std::make_shared(config_proto); + scoped_route_map_[config_proto.name()] = scoped_route_info; + return scoped_route_info; +} + +bool ScopedConfigManager::removeRoutingScope(const std::string& name) { + if (scoped_route_map_.erase(name) == 0) { + throw EnvoyException(fmt::format("could not find {} in scoped route map for removal", name)); + } + return true; +} + +} // namespace Router +} // namespace Envoy diff --git a/source/common/router/scoped_config_manager.h b/source/common/router/scoped_config_manager.h new file mode 100644 index 0000000000000..aa63ebf9f17ae --- /dev/null +++ b/source/common/router/scoped_config_manager.h @@ -0,0 +1,49 @@ +#pragma once + +#include +#include + +#include "envoy/api/v2/srds.pb.h" + +namespace Envoy { +namespace Router { + +// The internal representation of the configuration distributed via the ScopedRouteConfiguration +// proto. +class ScopedRouteInfo { +public: + ScopedRouteInfo(envoy::api::v2::ScopedRouteConfiguration config_proto) + : config_proto_(std::move(config_proto)) {} + + // TODO(AndresGuedez): Add the necessary APIs required for the scoped routing logic. + + const envoy::api::v2::ScopedRouteConfiguration config_proto_; +}; +using ScopedRouteInfoConstSharedPtr = std::shared_ptr; + +// A manager for routing configuration scopes. +// An instance of the manager is owned by each ScopedRdsConfigSubscription. When config updates are +// received (on the main thread), the manager is called to track changes to the set of scoped route +// configurations and build ScopedRouteInfo's as needed. +class ScopedConfigManager { +public: + // Ordered map for consistent config dumping. + using ScopedRouteMap = std::map; + + // Adds/updates a routing scope specified via the Scoped RDS API. This scope will be added to the + // set of scopes matched against the scope keys built for each HTTP request. + ScopedRouteInfoConstSharedPtr + addOrUpdateRoutingScope(const envoy::api::v2::ScopedRouteConfiguration& scoped_route_config, + const std::string& version_info); + + // Removes a routing scope from the set of scopes matched against each HTTP request. + bool removeRoutingScope(const std::string& scope_name); + + const ScopedRouteMap& scopedRouteMap() const { return scoped_route_map_; } + +private: + ScopedRouteMap scoped_route_map_; +}; + +} // namespace Router +} // namespace Envoy diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc new file mode 100644 index 0000000000000..befc15b8112e7 --- /dev/null +++ b/source/common/router/scoped_rds.cc @@ -0,0 +1,271 @@ +#include "common/router/scoped_rds.h" + +#include + +#include "envoy/admin/v2alpha/config_dump.pb.h" +#include "envoy/api/v2/srds.pb.validate.h" + +#include "common/common/assert.h" +#include "common/config/subscription_factory.h" + +// Types are deeply nested under Envoy::Config::ConfigProvider; use 'using-directives' across all +// ConfigProvider related types for consistency. +using Envoy::Config::ConfigProvider; +using Envoy::Config::ConfigProviderInstanceType; +using Envoy::Config::ConfigProviderManager; +using Envoy::Config::ConfigProviderPtr; + +namespace Envoy { +namespace Router { + +ConfigProviderPtr ScopedRoutesConfigProviderUtil::maybeCreate( + const envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& + config, + Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, + ConfigProviderManager& scoped_routes_config_provider_manager) { + if (config.route_specifier_case() != envoy::config::filter::network::http_connection_manager::v2:: + HttpConnectionManager::kScopedRoutes) { + return nullptr; + } + + switch (config.scoped_routes().config_specifier_case()) { + case envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: + kScopedRouteConfigurationsList: { + const envoy::config::filter::network::http_connection_manager::v2:: + ScopedRouteConfigurationsList& scoped_route_list = + config.scoped_routes().scoped_route_configurations_list(); + std::vector> config_protos( + scoped_route_list.scoped_route_configurations().size()); + for (const auto& it : scoped_route_list.scoped_route_configurations()) { + Protobuf::Message* clone = it.New(); + clone->CopyFrom(it); + config_protos.push_back(std::unique_ptr(clone)); + } + + return scoped_routes_config_provider_manager.createStaticConfigProvider( + std::move(config_protos), factory_context, + ScopedRoutesConfigProviderManagerOptArg(config.scoped_routes().name(), + config.scoped_routes().rds_config_source(), + config.scoped_routes().scope_key_builder())); + } + + case envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::kScopedRds: + return scoped_routes_config_provider_manager.createXdsConfigProvider( + config.scoped_routes().scoped_rds(), factory_context, stat_prefix, + ScopedRoutesConfigProviderManagerOptArg(config.scoped_routes().name(), + config.scoped_routes().rds_config_source(), + config.scoped_routes().scope_key_builder())); + + case envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: + CONFIG_SPECIFIER_NOT_SET: + return nullptr; + + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } +} + +InlineScopedRoutesConfigProvider::InlineScopedRoutesConfigProvider( + std::vector>&& config_protos, std::string name, + Server::Configuration::FactoryContext& factory_context, + ScopedRoutesConfigProviderManager& config_provider_manager, + envoy::api::v2::core::ConfigSource rds_config_source, + envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder + scope_key_builder) + : Envoy::Config::ImmutableConfigProviderBase(factory_context, config_provider_manager, + ConfigProviderInstanceType::Inline, + ConfigProvider::ApiType::Delta), + name_(std::move(name)), + config_(std::make_shared(std::move(scope_key_builder))), + config_protos_(std::make_move_iterator(config_protos.begin()), + std::make_move_iterator(config_protos.end())), + rds_config_source_(std::move(rds_config_source)) {} + +ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( + const envoy::config::filter::network::http_connection_manager::v2::ScopedRds& scoped_rds, + const uint64_t manager_identifier, const std::string& name, + Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, + ScopedRoutesConfigProviderManager& config_provider_manager) + : DeltaConfigSubscriptionInstance( + "SRDS", manager_identifier, config_provider_manager, factory_context.timeSource(), + factory_context.timeSource().systemTime(), factory_context.localInfo()), + name_(name), + scope_(factory_context.scope().createScope(stat_prefix + "scoped_rds." + name + ".")), + stats_({ALL_SCOPED_RDS_STATS(POOL_COUNTER(*scope_))}) { + subscription_ = Envoy::Config::SubscriptionFactory::subscriptionFromConfigSource( + scoped_rds.scoped_rds_config_source(), factory_context.localInfo(), + factory_context.dispatcher(), factory_context.clusterManager(), factory_context.random(), + *scope_, "envoy.api.v2.ScopedRoutesDiscoveryService.FetchScopedRoutes", + "envoy.api.v2.ScopedRoutesDiscoveryService.StreamScopedRoutes", + Grpc::Common::typeUrl( + envoy::api::v2::ScopedRouteConfiguration().GetDescriptor()->full_name()), + factory_context.api()); +} + +void ScopedRdsConfigSubscription::onConfigUpdate( + const Protobuf::RepeatedPtrField& resources, + const std::string& version_info) { + if (resources.empty()) { + ENVOY_LOG(debug, "Empty resources in scoped RDS onConfigUpdate()"); + stats_.update_empty_.inc(); + ConfigSubscriptionCommonBase::onConfigUpdateFailed(); + return; + } + std::vector scoped_routes; + for (const auto& resource_any : resources) { + scoped_routes.emplace_back( + MessageUtil::anyConvert(resource_any)); + } + + std::unordered_set resource_names; + for (const auto& scoped_route : scoped_routes) { + if (!resource_names.insert(scoped_route.name()).second) { + throw EnvoyException( + fmt::format("duplicate scoped route configuration {} found", scoped_route.name())); + } + } + for (const auto& scoped_route : scoped_routes) { + MessageUtil::validate(scoped_route); + } + + // TODO(AndresGuedez): refactor such that it can be shared with other delta APIs (e.g., CDS). + std::vector exception_msgs; + // We need to keep track of which scoped routes we might need to remove. + ScopedConfigManager::ScopedRouteMap scoped_routes_to_remove = + scoped_config_manager_.scopedRouteMap(); + for (auto& scoped_route : scoped_routes) { + const std::string& scoped_route_name = scoped_route.name(); + try { + scoped_routes_to_remove.erase(scoped_route_name); + ScopedRouteInfoConstSharedPtr scoped_route_info = + scoped_config_manager_.addOrUpdateRoutingScope(scoped_route, version_info); + if (scoped_route_info == nullptr) { + throw EnvoyException( + fmt::format("failed to create/update global routing scope {}", scoped_route_name)); + } + ENVOY_LOG(debug, "srds: add/update scoped_route '{}'", scoped_route_name); + applyConfigUpdate([scoped_route_info](const ConfigProvider::ConfigConstSharedPtr& config) { + auto* thread_local_scoped_config = const_cast( + static_cast(config.get())); + thread_local_scoped_config->addOrUpdateRoutingScope(scoped_route_info); + }); + } catch (const EnvoyException& ex) { + exception_msgs.push_back(fmt::format("{}: {}", scoped_route_name, ex.what())); + } + } + + for (const auto& scoped_route : scoped_routes_to_remove) { + const std::string scoped_route_name = scoped_route.first; + ENVOY_LOG(debug, "srds: remove scoped route '{}'", scoped_route_name); + applyConfigUpdate([scoped_route_name](const ConfigProvider::ConfigConstSharedPtr& config) { + auto* thread_local_scoped_config = const_cast( + static_cast(config.get())); + thread_local_scoped_config->removeRoutingScope(scoped_route_name); + }); + } + + ConfigSubscriptionCommonBase::onConfigUpdate(); + setLastConfigInfo(absl::optional({absl::nullopt, version_info})); + if (!exception_msgs.empty()) { + throw EnvoyException(fmt::format("Error adding/updating scoped route(s) {}", + StringUtil::join(exception_msgs, ", "))); + } + stats_.config_reload_.inc(); +} + +ScopedRdsConfigProvider::ScopedRdsConfigProvider( + ScopedRdsConfigSubscriptionSharedPtr&& subscription, + Server::Configuration::FactoryContext& factory_context, + envoy::api::v2::core::ConfigSource rds_config_source, + const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: + ScopeKeyBuilder& scope_key_builder) + : DeltaMutableConfigProviderBase(std::move(subscription), factory_context, + ConfigProvider::ApiType::Delta), + subscription_(static_cast( + MutableConfigProviderCommonBase::subscription_.get())), + rds_config_source_(std::move(rds_config_source)) { + initialize([scope_key_builder](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { + return std::make_shared(scope_key_builder); + }); +} + +ProtobufTypes::MessagePtr ScopedRoutesConfigProviderManager::dumpConfigs() const { + auto config_dump = std::make_unique(); + for (const auto& element : configSubscriptions()) { + auto subscription = element.second.lock(); + ASSERT(subscription); + + if (subscription->configInfo()) { + auto* dynamic_config = config_dump->mutable_dynamic_scoped_route_configs()->Add(); + dynamic_config->set_version_info(subscription->configInfo().value().last_config_version_); + const ScopedRdsConfigSubscription* typed_subscription = + static_cast(subscription.get()); + dynamic_config->set_name(typed_subscription->name()); + const ScopedConfigManager::ScopedRouteMap& scoped_route_map = + typed_subscription->scopedRouteMap(); + for (const auto& it : scoped_route_map) { + dynamic_config->mutable_scoped_route_configs()->Add()->MergeFrom(it.second->config_proto_); + } + TimestampUtil::systemClockToTimestamp(subscription->lastUpdated(), + *dynamic_config->mutable_last_updated()); + } + } + + for (const auto& provider : immutableConfigProviders(ConfigProviderInstanceType::Inline)) { + ASSERT(provider->configProtoInfoVector().has_value()); + auto* inline_config = config_dump->mutable_inline_scoped_route_configs()->Add(); + inline_config->set_name(static_cast(provider)->name()); + const absl::optional protos_info = + provider->configProtoInfoVector(); + const ConfigProvider::ConfigProtoVector& scoped_route_configurations = + protos_info.value().config_protos_; + for (const auto& it : scoped_route_configurations) { + inline_config->mutable_scoped_route_configs()->Add()->MergeFrom(*it); + } + TimestampUtil::systemClockToTimestamp(provider->lastUpdated(), + *inline_config->mutable_last_updated()); + } + + return config_dump; +} + +ConfigProviderPtr ScopedRoutesConfigProviderManager::createXdsConfigProvider( + const Protobuf::Message& config_source_proto, + Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, + const ConfigProviderManager::OptionalArg& optarg) { + ScopedRdsConfigSubscriptionSharedPtr subscription = + ConfigProviderManagerImplBase::getSubscription( + config_source_proto, factory_context.initManager(), + [&config_source_proto, &factory_context, &stat_prefix, + &optarg](const uint64_t manager_identifier, + ConfigProviderManagerImplBase& config_provider_manager) + -> Envoy::Config::ConfigSubscriptionCommonBaseSharedPtr { + const auto& scoped_rds_config_source = dynamic_cast< + const envoy::config::filter::network::http_connection_manager::v2::ScopedRds&>( + config_source_proto); + return std::make_shared( + scoped_rds_config_source, manager_identifier, + static_cast(optarg) + .scoped_routes_name_, + factory_context, stat_prefix, + static_cast(config_provider_manager)); + }); + + const auto& typed_optarg = static_cast(optarg); + return std::make_unique(std::move(subscription), factory_context, + typed_optarg.rds_config_source_, + typed_optarg.scope_key_builder_); +} + +ConfigProviderPtr ScopedRoutesConfigProviderManager::createStaticConfigProvider( + std::vector>&& config_protos, + Server::Configuration::FactoryContext& factory_context, + const ConfigProviderManager::OptionalArg& optarg) { + const auto& typed_optarg = static_cast(optarg); + return absl::make_unique( + std::move(config_protos), typed_optarg.scoped_routes_name_, factory_context, *this, + typed_optarg.rds_config_source_, typed_optarg.scope_key_builder_); +} + +} // namespace Router +} // namespace Envoy diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h new file mode 100644 index 0000000000000..bbd0c6175bebf --- /dev/null +++ b/source/common/router/scoped_rds.h @@ -0,0 +1,199 @@ +#pragma once + +#include + +#include "envoy/api/v2/srds.pb.h" +#include "envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.pb.h" +#include "envoy/config/subscription.h" +#include "envoy/stats/scope.h" + +#include "common/common/logger.h" +#include "common/config/config_provider_impl.h" +#include "common/router/scoped_config_impl.h" + +namespace Envoy { +namespace Router { + +// Scoped routing configuration utilities. +class ScopedRoutesConfigProviderUtil { +public: + // If enabled in the HttpConnectionManager config, returns a ConfigProvider for scoped routing + // configuration. + static Envoy::Config::ConfigProviderPtr maybeCreate( + const envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& + config, + Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, + Envoy::Config::ConfigProviderManager& scoped_routes_config_provider_manager); +}; + +class ScopedRoutesConfigProviderManager; + +// A ConfigProvider for inline scoped routing configuration. +class InlineScopedRoutesConfigProvider : public Envoy::Config::ImmutableConfigProviderBase { +public: + InlineScopedRoutesConfigProvider( + std::vector>&& config_protos, std::string name, + Server::Configuration::FactoryContext& factory_context, + ScopedRoutesConfigProviderManager& config_provider_manager, + envoy::api::v2::core::ConfigSource rds_config_source, + envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder + scope_key_builder); + + ~InlineScopedRoutesConfigProvider() override = default; + + const std::string& name() const { return name_; } + + // Envoy::Config::ConfigProvider + const Envoy::Config::ConfigProvider::ConfigProtoVector getConfigProtos() const override { + Envoy::Config::ConfigProvider::ConfigProtoVector out_protos; + std::for_each(config_protos_.begin(), config_protos_.end(), + [&out_protos](const std::unique_ptr& message) { + out_protos.push_back(message.get()); + }); + return out_protos; + } + + std::string getConfigVersion() const override { return ""; } + ConfigConstSharedPtr getConfig() const override { return config_; } + +private: + const std::string name_; + ConfigConstSharedPtr config_; + const std::vector> config_protos_; + const envoy::api::v2::core::ConfigSource rds_config_source_; +}; + +/** + * All SRDS stats. @see stats_macros.h + */ +// clang-format off +#define ALL_SCOPED_RDS_STATS(COUNTER) \ + COUNTER(config_reload) \ + COUNTER(update_empty) + +// clang-format on + +struct ScopedRdsStats { + ALL_SCOPED_RDS_STATS(GENERATE_COUNTER_STRUCT) +}; + +// A scoped RDS subscription to be used with the dynamic scoped RDS ConfigProvider. +class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptionInstance, + Envoy::Config::SubscriptionCallbacks { +public: + using ScopedRouteConfigurationMap = + std::map; + + ScopedRdsConfigSubscription( + const envoy::config::filter::network::http_connection_manager::v2::ScopedRds& scoped_rds, + const uint64_t manager_identifier, const std::string& name, + Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, + ScopedRoutesConfigProviderManager& config_provider_manager); + + ~ScopedRdsConfigSubscription() override = default; + + const std::string& name() const { return name_; } + + // Envoy::Config::ConfigSubscriptionCommonBase + void start() override { subscription_->start({}, *this); } + + // Envoy::Config::SubscriptionCallbacks + void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, + const std::string& version_info) override; + void onConfigUpdate(const Protobuf::RepeatedPtrField&, + const Protobuf::RepeatedPtrField&, const std::string&) override { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; + } + void onConfigUpdateFailed(const EnvoyException*) override { + ConfigSubscriptionCommonBase::onConfigUpdateFailed(); + } + std::string resourceName(const ProtobufWkt::Any& resource) override { + return MessageUtil::anyConvert(resource).name(); + } + const ScopedConfigManager::ScopedRouteMap& scopedRouteMap() const { + return scoped_config_manager_.scopedRouteMap(); + } + +private: + const std::string name_; + std::unique_ptr subscription_; + Stats::ScopePtr scope_; + ScopedRdsStats stats_; + ScopedConfigManager scoped_config_manager_; +}; + +using ScopedRdsConfigSubscriptionSharedPtr = std::shared_ptr; + +// A ConfigProvider for scoped RDS that dynamically fetches scoped routing configuration via a +// subscription. +class ScopedRdsConfigProvider : public Envoy::Config::DeltaMutableConfigProviderBase { +public: + ScopedRdsConfigProvider(ScopedRdsConfigSubscriptionSharedPtr&& subscription, + Server::Configuration::FactoryContext& factory_context, + envoy::api::v2::core::ConfigSource rds_config_source, + const envoy::config::filter::network::http_connection_manager::v2:: + ScopedRoutes::ScopeKeyBuilder& scope_key_builder); + + ScopedRdsConfigSubscription& subscription() { return *subscription_; } + + // Envoy::Config::ConfigProvider + ConfigConstSharedPtr getConfig() const override { + return std::dynamic_pointer_cast(tls_->get()); + } + +private: + ScopedRdsConfigSubscription* subscription_; + const envoy::api::v2::core::ConfigSource rds_config_source_; +}; + +// A ConfigProviderManager for scoped routing configuration that creates static/inline and dynamic +// (xds) config providers. +class ScopedRoutesConfigProviderManager : public Envoy::Config::ConfigProviderManagerImplBase { +public: + ScopedRoutesConfigProviderManager(Server::Admin& admin) + : Envoy::Config::ConfigProviderManagerImplBase(admin, "route_scopes") {} + + ~ScopedRoutesConfigProviderManager() override = default; + + // Envoy::Config::ConfigProviderManagerImplBase + ProtobufTypes::MessagePtr dumpConfigs() const override; + + // Envoy::Config::ConfigProviderManager + Envoy::Config::ConfigProviderPtr + createXdsConfigProvider(const Protobuf::Message& config_source_proto, + Server::Configuration::FactoryContext& factory_context, + const std::string& stat_prefix, + const Envoy::Config::ConfigProviderManager::OptionalArg& optarg) override; + Envoy::Config::ConfigProviderPtr + createStaticConfigProvider(const Protobuf::Message&, Server::Configuration::FactoryContext&, + const Envoy::Config::ConfigProviderManager::OptionalArg&) override { + ASSERT(false, + "SRDS supports delta updates and requires the use of the createStaticConfigProvider() " + "overload that accepts a config proto set as an argument."); + NOT_REACHED_GCOVR_EXCL_LINE; + } + Envoy::Config::ConfigProviderPtr createStaticConfigProvider( + std::vector>&& config_protos, + Server::Configuration::FactoryContext& factory_context, + const Envoy::Config::ConfigProviderManager::OptionalArg& optarg) override; +}; + +// The optional argument passed to the ConfigProviderManager::create*() functions. +class ScopedRoutesConfigProviderManagerOptArg + : public Envoy::Config::ConfigProviderManager::OptionalArg { +public: + ScopedRoutesConfigProviderManagerOptArg( + std::string scoped_routes_name, const envoy::api::v2::core::ConfigSource& rds_config_source, + const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: + ScopeKeyBuilder& scope_key_builder) + : scoped_routes_name_(std::move(scoped_routes_name)), rds_config_source_(rds_config_source), + scope_key_builder_(scope_key_builder) {} + + const std::string scoped_routes_name_; + const envoy::api::v2::core::ConfigSource& rds_config_source_; + const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder& + scope_key_builder_; +}; + +} // namespace Router +} // namespace Envoy diff --git a/source/extensions/filters/network/http_connection_manager/BUILD b/source/extensions/filters/network/http_connection_manager/BUILD index 29f6420e60845..1f5474a651e2f 100644 --- a/source/extensions/filters/network/http_connection_manager/BUILD +++ b/source/extensions/filters/network/http_connection_manager/BUILD @@ -17,6 +17,7 @@ envoy_cc_library( srcs = ["config.cc"], hdrs = ["config.h"], deps = [ + "//include/envoy/config:config_provider_manager_interface", "//include/envoy/filesystem:filesystem_interface", "//include/envoy/http:filter_interface", "//include/envoy/registry", @@ -36,6 +37,7 @@ envoy_cc_library( "//source/common/json:json_loader_lib", "//source/common/protobuf:utility_lib", "//source/common/router:rds_lib", + "//source/common/router:scoped_rds_lib", "//source/extensions/filters/network:well_known_names", "//source/extensions/filters/network/common:factory_base_lib", ], diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index 0014ccc55ef60..e4e790ae89a92 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -23,6 +23,7 @@ #include "common/json/config_schemas.h" #include "common/protobuf/utility.h" #include "common/router/rds_impl.h" +#include "common/router/scoped_rds.h" namespace Envoy { namespace Extensions { @@ -69,6 +70,7 @@ std::unique_ptr createInternalAddressConfig( // Singleton registration via macro defined in envoy/singleton/manager.h SINGLETON_MANAGER_REGISTRATION(date_provider); SINGLETON_MANAGER_REGISTRATION(route_config_provider_manager); +SINGLETON_MANAGER_REGISTRATION(scoped_routes_config_provider_manager); Network::FilterFactoryCb HttpConnectionManagerFilterConfigFactory::createFilterFactoryFromProtoTyped( @@ -88,14 +90,21 @@ HttpConnectionManagerFilterConfigFactory::createFilterFactoryFromProtoTyped( return std::make_shared(context.admin()); }); + std::shared_ptr scoped_routes_config_provider_manager = + context.singletonManager().getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(scoped_routes_config_provider_manager), [&context] { + return std::make_shared(context.admin()); + }); + std::shared_ptr filter_config(new HttpConnectionManagerConfig( - proto_config, context, *date_provider, *route_config_provider_manager)); + proto_config, context, *date_provider, *route_config_provider_manager, + *scoped_routes_config_provider_manager)); // This lambda captures the shared_ptrs created above, thus preserving the // reference count. Moreover, keep in mind the capture list determines // destruction order. - return [route_config_provider_manager, filter_config, &context, - date_provider](Network::FilterManager& filter_manager) -> void { + return [route_config_provider_manager, scoped_routes_config_provider_manager, filter_config, + &context, date_provider](Network::FilterManager& filter_manager) -> void { filter_manager.addReadFilter(Network::ReadFilterSharedPtr{new Http::ConnectionManagerImpl( *filter_config, context.drainDecision(), context.random(), context.httpContext(), context.runtime(), context.localInfo(), context.clusterManager(), @@ -126,7 +135,8 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( const envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& config, Server::Configuration::FactoryContext& context, Http::DateProvider& date_provider, - Router::RouteConfigProviderManager& route_config_provider_manager) + Router::RouteConfigProviderManager& route_config_provider_manager, + Config::ConfigProviderManager& scoped_routes_config_provider_manager) : context_(context), stats_prefix_(fmt::format("http.{}.", config.stat_prefix())), stats_(Http::ConnectionManagerImpl::generateStats(stats_prefix_, context_.scope())), tracing_stats_( @@ -136,6 +146,7 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( xff_num_trusted_hops_(config.xff_num_trusted_hops()), skip_xff_append_(config.skip_xff_append()), via_(config.via()), route_config_provider_manager_(route_config_provider_manager), + scoped_routes_config_provider_manager_(scoped_routes_config_provider_manager), http2_settings_(Http::Utility::parseHttp2Settings(config.http2_protocol_options())), http1_settings_(Http::Utility::parseHttp1Settings(config.http_protocol_options())), max_request_headers_kb_(PROTOBUF_GET_WRAPPED_OR_DEFAULT( @@ -163,9 +174,16 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( 0 #endif ))) { + // If scoped RDS is enabled, avoid creating a route config provider. Route config providers will + // be managed by the scoped routing logic instead. + if (config.route_specifier_case() != envoy::config::filter::network::http_connection_manager::v2:: + HttpConnectionManager::kScopedRoutes) { + route_config_provider_ = Router::RouteConfigProviderUtil::create( + config, context_, stats_prefix_, route_config_provider_manager_); + } - route_config_provider_ = Router::RouteConfigProviderUtil::create(config, context_, stats_prefix_, - route_config_provider_manager_); + scoped_routes_config_provider_ = Router::ScopedRoutesConfigProviderUtil::maybeCreate( + config, context_, stats_prefix_, scoped_routes_config_provider_manager_); switch (config.forward_client_cert_details()) { case envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager::SANITIZE: diff --git a/source/extensions/filters/network/http_connection_manager/config.h b/source/extensions/filters/network/http_connection_manager/config.h index c1e7332b653c1..0fca2f39d9b7f 100644 --- a/source/extensions/filters/network/http_connection_manager/config.h +++ b/source/extensions/filters/network/http_connection_manager/config.h @@ -7,6 +7,7 @@ #include #include +#include "envoy/config/config_provider_manager.h" #include "envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.pb.validate.h" #include "envoy/http/filter.h" #include "envoy/router/route_config_provider_manager.h" @@ -78,7 +79,8 @@ class HttpConnectionManagerConfig : Logger::Loggable, const envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& config, Server::Configuration::FactoryContext& context, Http::DateProvider& date_provider, - Router::RouteConfigProviderManager& route_config_provider_manager); + Router::RouteConfigProviderManager& route_config_provider_manager, + Config::ConfigProviderManager& scoped_routes_config_provider_manager); // Http::FilterChainFactory void createFilterChain(Http::FilterChainFactoryCallbacks& callbacks) override; @@ -104,7 +106,12 @@ class HttpConnectionManagerConfig : Logger::Loggable, absl::optional idleTimeout() const override { return idle_timeout_; } std::chrono::milliseconds streamIdleTimeout() const override { return stream_idle_timeout_; } std::chrono::milliseconds requestTimeout() const override { return request_timeout_; } - Router::RouteConfigProvider& routeConfigProvider() override { return *route_config_provider_; } + Router::RouteConfigProvider* routeConfigProvider() override { + return route_config_provider_.get(); + } + Config::ConfigProvider* scopedRouteConfigProvider() override { + return scoped_routes_config_provider_.get(); + } const std::string& serverName() override { return server_name_; } Http::ConnectionManagerStats& stats() override { return stats_; } Http::ConnectionManagerTracingStats& tracingStats() override { return tracing_stats_; } @@ -151,6 +158,7 @@ class HttpConnectionManagerConfig : Logger::Loggable, Http::ForwardClientCertType forward_client_cert_; std::vector set_current_client_cert_details_; Router::RouteConfigProviderManager& route_config_provider_manager_; + Config::ConfigProviderManager& scoped_routes_config_provider_manager_; CodecType codec_type_; const Http::Http2Settings http2_settings_; const Http::Http1Settings http1_settings_; @@ -162,6 +170,7 @@ class HttpConnectionManagerConfig : Logger::Loggable, std::chrono::milliseconds stream_idle_timeout_; std::chrono::milliseconds request_timeout_; Router::RouteConfigProviderPtr route_config_provider_; + Config::ConfigProviderPtr scoped_routes_config_provider_; std::chrono::milliseconds drain_timeout_; bool generate_request_id_; Http::DateProvider& date_provider_; diff --git a/source/server/http/BUILD b/source/server/http/BUILD index b6761f2865857..65b8bc56688cd 100644 --- a/source/server/http/BUILD +++ b/source/server/http/BUILD @@ -53,6 +53,7 @@ envoy_cc_library( "//source/common/network:utility_lib", "//source/common/profiler:profiler_lib", "//source/common/router:config_lib", + "//source/common/router:scoped_config_lib", "//source/common/stats:histogram_lib", "//source/common/stats:isolated_store_lib", "//source/common/stats:stats_lib", diff --git a/source/server/http/admin.cc b/source/server/http/admin.cc index 8fb1fde7d74e5..c9342afe3b1ca 100644 --- a/source/server/http/admin.cc +++ b/source/server/http/admin.cc @@ -1111,6 +1111,7 @@ AdminImpl::AdminImpl(const std::string& profile_path, Server::Instance& server) tracing_stats_( Http::ConnectionManagerImpl::generateTracingStats("http.admin.", no_op_store_)), route_config_provider_(server.timeSource()), + scoped_route_config_provider_(server.timeSource()), // TODO(jsedgwick) add /runtime_reset endpoint that removes all admin-set values handlers_{ {"/", "Admin home page", MAKE_ADMIN_HANDLER(handlerAdminHome), false, false}, diff --git a/source/server/http/admin.h b/source/server/http/admin.h index 7bffdeb2b0367..8b84d905c5fb2 100644 --- a/source/server/http/admin.h +++ b/source/server/http/admin.h @@ -27,6 +27,7 @@ #include "common/http/default_server_string.h" #include "common/http/utility.h" #include "common/network/raw_buffer_socket.h" +#include "common/router/scoped_config_impl.h" #include "common/stats/isolated_store_impl.h" #include "server/http/config_tracker_impl.h" @@ -103,7 +104,10 @@ class AdminImpl : public Admin, std::chrono::milliseconds streamIdleTimeout() const override { return {}; } std::chrono::milliseconds requestTimeout() const override { return {}; } std::chrono::milliseconds delayedCloseTimeout() const override { return {}; } - Router::RouteConfigProvider& routeConfigProvider() override { return route_config_provider_; } + Router::RouteConfigProvider* routeConfigProvider() override { return &route_config_provider_; } + Config::ConfigProvider* scopedRouteConfigProvider() override { + return &scoped_route_config_provider_; + } const std::string& serverName() override { return Http::DefaultServerString::get(); } Http::ConnectionManagerStats& stats() override { return stats_; } Http::ConnectionManagerTracingStats& tracingStats() override { return tracing_stats_; } @@ -159,6 +163,28 @@ class AdminImpl : public Admin, TimeSource& time_source_; }; + /** + * Implementation of ScopedRouteConfigProvider that returns a null scoped route config. + */ + struct NullScopedRouteConfigProvider : public Config::ConfigProvider { + NullScopedRouteConfigProvider(TimeSource& time_source) + : config_(std::make_shared()), + time_source_(time_source) {} + + ~NullScopedRouteConfigProvider() override = default; + + // Config::ConfigProvider + SystemTime lastUpdated() const override { return time_source_.systemTime(); } + const Protobuf::Message* getConfigProto() const override { return nullptr; } + std::string getConfigVersion() const override { return ""; } + ConfigConstSharedPtr getConfig() const override { return config_; } + ApiType apiType() const override { return ApiType::Full; } + const std::vector getConfigProtos() const override { return {}; } + + Router::ScopedConfigConstSharedPtr config_; + TimeSource& time_source_; + }; + friend class AdminStatsTest; /** @@ -306,6 +332,7 @@ class AdminImpl : public Admin, Stats::IsolatedStoreImpl no_op_store_; Http::ConnectionManagerTracingStats tracing_stats_; NullRouteConfigProvider route_config_provider_; + NullScopedRouteConfigProvider scoped_route_config_provider_; std::list handlers_; const uint32_t max_request_headers_kb_{Http::DEFAULT_MAX_REQUEST_HEADERS_KB}; absl::optional idle_timeout_; diff --git a/test/common/config/config_provider_impl_test.cc b/test/common/config/config_provider_impl_test.cc index 306c09c015a79..e7d0646230a56 100644 --- a/test/common/config/config_provider_impl_test.cc +++ b/test/common/config/config_provider_impl_test.cc @@ -16,7 +16,7 @@ namespace { class DummyConfigProviderManager; -class StaticDummyConfigProvider : public ImmutableConfigProviderImplBase { +class StaticDummyConfigProvider : public ImmutableConfigProviderBase { public: StaticDummyConfigProvider(const test::common::config::DummyConfig& config_proto, Server::Configuration::FactoryContext& factory_context, @@ -38,7 +38,7 @@ class StaticDummyConfigProvider : public ImmutableConfigProviderImplBase { test::common::config::DummyConfig config_proto_; }; -class DummyConfigSubscription : public ConfigSubscriptionInstanceBase, +class DummyConfigSubscription : public ConfigSubscriptionInstance, Envoy::Config::SubscriptionCallbacks { public: DummyConfigSubscription(const uint64_t manager_identifier, @@ -47,7 +47,7 @@ class DummyConfigSubscription : public ConfigSubscriptionInstanceBase, ~DummyConfigSubscription() override = default; - // Envoy::Config::ConfigSubscriptionInstanceBase + // Envoy::Config::ConfigSubscriptionCommonBase void start() override {} // Envoy::Config::SubscriptionCallbacks @@ -55,11 +55,11 @@ class DummyConfigSubscription : public ConfigSubscriptionInstanceBase, void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, const std::string& version_info) override { auto config = MessageUtil::anyConvert(resources[0]); - if (checkAndApplyConfig(config, "dummy_config", version_info)) { + if (checkAndApplyConfigUpdate(config, "dummy_config", version_info)) { config_proto_ = config; } - ConfigSubscriptionInstanceBase::onConfigUpdate(); + ConfigSubscriptionCommonBase::onConfigUpdate(); } void onConfigUpdate(const Protobuf::RepeatedPtrField&, const Protobuf::RepeatedPtrField&, const std::string&) override { @@ -87,14 +87,14 @@ class DummyConfig : public ConfigProvider::Config { DummyConfig(const test::common::config::DummyConfig&) {} }; -class DummyDynamicConfigProvider : public MutableConfigProviderImplBase { +class DummyDynamicConfigProvider : public MutableConfigProviderBase { public: DummyDynamicConfigProvider(DummyConfigSubscriptionSharedPtr&& subscription, - ConfigConstSharedPtr initial_config, + const ConfigConstSharedPtr& initial_config, Server::Configuration::FactoryContext& factory_context) - : MutableConfigProviderImplBase(std::move(subscription), factory_context), + : MutableConfigProviderBase(std::move(subscription), factory_context, ApiType::Full), subscription_(static_cast( - MutableConfigProviderImplBase::subscription().get())) { + MutableConfigProviderCommonBase::subscription_.get())) { initialize(initial_config); } @@ -102,7 +102,7 @@ class DummyDynamicConfigProvider : public MutableConfigProviderImplBase { DummyConfigSubscription& subscription() { return *subscription_; } - // Envoy::Config::MutableConfigProviderImplBase + // Envoy::Config::MutableConfigProviderBase ConfigProvider::ConfigConstSharedPtr onConfigProtoUpdate(const Protobuf::Message& config) override { return std::make_shared( @@ -116,8 +116,6 @@ class DummyDynamicConfigProvider : public MutableConfigProviderImplBase { } return &subscription_->config_proto().value(); } - - // Envoy::Config::ConfigProvider std::string getConfigVersion() const override { return ""; } private: @@ -162,22 +160,24 @@ class DummyConfigProviderManager : public ConfigProviderManagerImplBase { } // Envoy::Config::ConfigProviderManager - ConfigProviderPtr createXdsConfigProvider(const Protobuf::Message& config_source_proto, - Server::Configuration::FactoryContext& factory_context, - const std::string&) override { + ConfigProviderPtr + createXdsConfigProvider(const Protobuf::Message& config_source_proto, + Server::Configuration::FactoryContext& factory_context, + const std::string&, + const Envoy::Config::ConfigProviderManager::OptionalArg&) override { DummyConfigSubscriptionSharedPtr subscription = getSubscription( config_source_proto, factory_context.initManager(), [&factory_context](const uint64_t manager_identifier, ConfigProviderManagerImplBase& config_provider_manager) - -> ConfigSubscriptionInstanceBaseSharedPtr { + -> ConfigSubscriptionCommonBaseSharedPtr { return std::make_shared( manager_identifier, factory_context, static_cast(config_provider_manager)); }); ConfigProvider::ConfigConstSharedPtr initial_config; - const MutableConfigProviderImplBase* provider = - subscription->getAnyBoundMutableConfigProvider(); + const auto* provider = static_cast( + subscription->getAnyBoundMutableConfigProvider()); if (provider) { initial_config = provider->getConfig(); } @@ -188,25 +188,32 @@ class DummyConfigProviderManager : public ConfigProviderManagerImplBase { // Envoy::Config::ConfigProviderManager ConfigProviderPtr createStaticConfigProvider(const Protobuf::Message& config_proto, - Server::Configuration::FactoryContext& factory_context) override { + Server::Configuration::FactoryContext& factory_context, + const Envoy::Config::ConfigProviderManager::OptionalArg&) override { return std::make_unique( dynamic_cast(config_proto), factory_context, *this); } + ConfigProviderPtr + createStaticConfigProvider(std::vector>&&, + Server::Configuration::FactoryContext&, const OptionalArg&) override { + ASSERT(false || "this provider does not expect multiple config protos"); + return nullptr; + } }; StaticDummyConfigProvider::StaticDummyConfigProvider( const test::common::config::DummyConfig& config_proto, Server::Configuration::FactoryContext& factory_context, DummyConfigProviderManager& config_provider_manager) - : ImmutableConfigProviderImplBase(factory_context, config_provider_manager, - ConfigProviderInstanceType::Static), + : ImmutableConfigProviderBase(factory_context, config_provider_manager, + ConfigProviderInstanceType::Static, ApiType::Full), config_(std::make_shared(config_proto)), config_proto_(config_proto) {} DummyConfigSubscription::DummyConfigSubscription( const uint64_t manager_identifier, Server::Configuration::FactoryContext& factory_context, DummyConfigProviderManager& config_provider_manager) - : ConfigSubscriptionInstanceBase( + : ConfigSubscriptionInstance( "DummyDS", manager_identifier, config_provider_manager, factory_context.timeSource(), factory_context.timeSource().systemTime(), factory_context.localInfo()) {} @@ -241,7 +248,8 @@ TEST_F(ConfigProviderImplTest, SharedOwnership) { envoy::api::v2::core::ApiConfigSource config_source_proto; config_source_proto.set_api_type(envoy::api::v2::core::ApiConfigSource::GRPC); ConfigProviderPtr provider1 = provider_manager_->createXdsConfigProvider( - config_source_proto, factory_context_, "dummy_prefix"); + config_source_proto, factory_context_, "dummy_prefix", + ConfigProviderManager::NullOptionalArg()); // No config protos have been received via the subscription yet. EXPECT_FALSE(provider1->configProtoInfo().has_value()); @@ -256,7 +264,8 @@ TEST_F(ConfigProviderImplTest, SharedOwnership) { // Check that a newly created provider with the same config source will share // the subscription, config proto and resulting ConfigProvider::Config. ConfigProviderPtr provider2 = provider_manager_->createXdsConfigProvider( - config_source_proto, factory_context_, "dummy_prefix"); + config_source_proto, factory_context_, "dummy_prefix", + ConfigProviderManager::NullOptionalArg()); EXPECT_TRUE(provider2->configProtoInfo().has_value()); EXPECT_EQ(&dynamic_cast(*provider1).subscription(), @@ -269,7 +278,8 @@ TEST_F(ConfigProviderImplTest, SharedOwnership) { // Change the config source and verify that a new subscription is used. config_source_proto.set_api_type(envoy::api::v2::core::ApiConfigSource::REST); ConfigProviderPtr provider3 = provider_manager_->createXdsConfigProvider( - config_source_proto, factory_context_, "dummy_prefix"); + config_source_proto, factory_context_, "dummy_prefix", + ConfigProviderManager::NullOptionalArg()); EXPECT_NE(&dynamic_cast(*provider1).subscription(), &dynamic_cast(*provider3).subscription()); @@ -327,7 +337,8 @@ TEST_F(ConfigProviderImplTest, ConfigDump) { timeSystem().setSystemTime(std::chrono::milliseconds(1234567891234)); ConfigProviderPtr static_config = provider_manager_->createStaticConfigProvider( - parseDummyConfigFromYaml(config_yaml), factory_context_); + parseDummyConfigFromYaml(config_yaml), factory_context_, + ConfigProviderManager::NullOptionalArg()); message_ptr = factory_context_.admin_.config_tracker_.config_tracker_callbacks_["dummy"](); const auto& dummy_config_dump2 = static_cast(*message_ptr); @@ -343,7 +354,8 @@ TEST_F(ConfigProviderImplTest, ConfigDump) { envoy::api::v2::core::ApiConfigSource config_source_proto; config_source_proto.set_api_type(envoy::api::v2::core::ApiConfigSource::GRPC); ConfigProviderPtr dynamic_provider = provider_manager_->createXdsConfigProvider( - config_source_proto, factory_context_, "dummy_prefix"); + config_source_proto, factory_context_, "dummy_prefix", + ConfigProviderManager::NullOptionalArg()); // Static + dynamic config dump. Protobuf::RepeatedPtrField untyped_dummy_configs; @@ -381,7 +393,8 @@ TEST_F(ConfigProviderImplTest, LocalInfoNotDefined) { config_source_proto.set_api_type(envoy::api::v2::core::ApiConfigSource::GRPC); EXPECT_THROW_WITH_MESSAGE( provider_manager_->createXdsConfigProvider(config_source_proto, factory_context_, - "dummy_prefix"), + "dummy_prefix", + ConfigProviderManager::NullOptionalArg()), EnvoyException, "DummyDS: node 'id' and 'cluster' are required. Set it either in 'node' config or " "via --service-node and --service-cluster options."); diff --git a/test/common/http/BUILD b/test/common/http/BUILD index aaf0f6d92024c..55924622050c1 100644 --- a/test/common/http/BUILD +++ b/test/common/http/BUILD @@ -130,6 +130,17 @@ envoy_cc_test_library( ], ) +envoy_cc_test_library( + name = "conn_manager_impl_common_lib", + hdrs = ["conn_manager_impl_common.h"], + deps = [ + "//include/envoy/common:time_interface", + "//include/envoy/config:config_provider_interface", + "//include/envoy/router:rds_interface", + "//test/mocks/router:router_mocks", + ], +) + envoy_proto_library( name = "conn_manager_impl_fuzz_proto", srcs = ["conn_manager_impl_fuzz.proto"], @@ -143,6 +154,7 @@ envoy_cc_fuzz_test( srcs = ["conn_manager_impl_fuzz_test.cc"], corpus = "conn_manager_impl_corpus", deps = [ + ":conn_manager_impl_common_lib", ":conn_manager_impl_fuzz_proto_cc", "//source/common/common:empty_string", "//source/common/http:conn_manager_lib", @@ -168,6 +180,7 @@ envoy_cc_test( name = "conn_manager_impl_test", srcs = ["conn_manager_impl_test.cc"], deps = [ + ":conn_manager_impl_common_lib", "//include/envoy/access_log:access_log_interface", "//include/envoy/buffer:buffer_interface", "//include/envoy/event:dispatcher_interface", diff --git a/test/common/http/conn_manager_impl_common.h b/test/common/http/conn_manager_impl_common.h new file mode 100644 index 0000000000000..1bf4cf01dc6ba --- /dev/null +++ b/test/common/http/conn_manager_impl_common.h @@ -0,0 +1,55 @@ +#pragma once + +#include + +#include "envoy/common/time.h" +#include "envoy/config/config_provider.h" +#include "envoy/router/rds.h" + +#include "test/mocks/router/mocks.h" + +#include "gmock/gmock.h" + +using testing::NiceMock; + +namespace Envoy { +namespace Http { +namespace ConnectionManagerImplHelper { + +// Test RouteConfigProvider that returns a mocked config. +struct RouteConfigProvider : public Router::RouteConfigProvider { + RouteConfigProvider(TimeSource& time_source) : time_source_(time_source) {} + + // Router::RouteConfigProvider + Router::ConfigConstSharedPtr config() override { return route_config_; } + absl::optional configInfo() const override { return {}; } + SystemTime lastUpdated() const override { return time_source_.systemTime(); } + + TimeSource& time_source_; + std::shared_ptr route_config_{new NiceMock()}; +}; + +// Test ScopedRouteConfigProvider that returns a mocked config. +struct ScopedRouteConfigProvider : public Config::ConfigProvider { + ScopedRouteConfigProvider(TimeSource& time_source) + : config_(std::make_shared()), time_source_(time_source) {} + + ~ScopedRouteConfigProvider() override = default; + + // Config::ConfigProvider + SystemTime lastUpdated() const override { return time_source_.systemTime(); } + const Protobuf::Message* getConfigProto() const override { return nullptr; } + const Envoy::Config::ConfigProvider::ConfigProtoVector getConfigProtos() const override { + return {}; + } + std::string getConfigVersion() const override { return ""; } + ConfigConstSharedPtr getConfig() const override { return config_; } + ApiType apiType() const override { return ApiType::Delta; } + + std::shared_ptr config_; + TimeSource& time_source_; +}; + +} // namespace ConnectionManagerImplHelper +} // namespace Http +} // namespace Envoy diff --git a/test/common/http/conn_manager_impl_fuzz_test.cc b/test/common/http/conn_manager_impl_fuzz_test.cc index 07762102ba9ae..4e1820b7b9d1f 100644 --- a/test/common/http/conn_manager_impl_fuzz_test.cc +++ b/test/common/http/conn_manager_impl_fuzz_test.cc @@ -20,6 +20,7 @@ #include "common/network/address_impl.h" #include "common/network/utility.h" +#include "test/common/http/conn_manager_impl_common.h" #include "test/common/http/conn_manager_impl_fuzz.pb.h" #include "test/fuzz/fuzz_runner.h" #include "test/fuzz/utility.h" @@ -44,20 +45,8 @@ namespace Http { class FuzzConfig : public ConnectionManagerConfig { public: - struct RouteConfigProvider : public Router::RouteConfigProvider { - RouteConfigProvider(TimeSource& time_source) : time_source_(time_source) {} - - // Router::RouteConfigProvider - Router::ConfigConstSharedPtr config() override { return route_config_; } - absl::optional configInfo() const override { return {}; } - SystemTime lastUpdated() const override { return time_source_.systemTime(); } - - TimeSource& time_source_; - std::shared_ptr route_config_{new NiceMock()}; - }; - FuzzConfig() - : route_config_provider_(time_system_), + : route_config_provider_(time_system_), scoped_route_config_provider_(time_system_), stats_{{ALL_HTTP_CONN_MAN_STATS(POOL_COUNTER(fake_stats_), POOL_GAUGE(fake_stats_), POOL_HISTOGRAM(fake_stats_))}, "", @@ -95,7 +84,10 @@ class FuzzConfig : public ConnectionManagerConfig { std::chrono::milliseconds streamIdleTimeout() const override { return stream_idle_timeout_; } std::chrono::milliseconds requestTimeout() const override { return request_timeout_; } std::chrono::milliseconds delayedCloseTimeout() const override { return delayed_close_timeout_; } - Router::RouteConfigProvider& routeConfigProvider() override { return route_config_provider_; } + Router::RouteConfigProvider* routeConfigProvider() override { return &route_config_provider_; } + Config::ConfigProvider* scopedRouteConfigProvider() override { + return &scoped_route_config_provider_; + } const std::string& serverName() override { return server_name_; } ConnectionManagerStats& stats() override { return stats_; } ConnectionManagerTracingStats& tracingStats() override { return tracing_stats_; } @@ -126,7 +118,8 @@ class FuzzConfig : public ConnectionManagerConfig { NiceMock filter_factory_; Event::SimulatedTimeSystem time_system_; SlowDateProviderImpl date_provider_{time_system_}; - RouteConfigProvider route_config_provider_; + ConnectionManagerImplHelper::RouteConfigProvider route_config_provider_; + ConnectionManagerImplHelper::ScopedRouteConfigProvider scoped_route_config_provider_; std::string server_name_; Stats::IsolatedStoreImpl fake_stats_; ConnectionManagerStats stats_; diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index a282fc88cda50..ff25ab6c0812a 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -26,6 +26,7 @@ #include "extensions/access_loggers/file/file_access_log_impl.h" +#include "test/common/http/conn_manager_impl_common.h" #include "test/mocks/access_log/mocks.h" #include "test/mocks/buffer/mocks.h" #include "test/mocks/common.h" @@ -62,20 +63,9 @@ namespace Http { class HttpConnectionManagerImplTest : public testing::Test, public ConnectionManagerConfig { public: - struct RouteConfigProvider : public Router::RouteConfigProvider { - RouteConfigProvider(TimeSource& time_source) : time_source_(time_source) {} - - // Router::RouteConfigProvider - Router::ConfigConstSharedPtr config() override { return route_config_; } - absl::optional configInfo() const override { return {}; } - SystemTime lastUpdated() const override { return time_source_.systemTime(); } - - TimeSource& time_source_; - std::shared_ptr route_config_{new NiceMock()}; - }; - HttpConnectionManagerImplTest() - : route_config_provider_(test_time_.timeSystem()), access_log_path_("dummy_path"), + : route_config_provider_(test_time_.timeSystem()), + scoped_route_config_provider_(test_time_.timeSystem()), access_log_path_("dummy_path"), access_logs_{ AccessLog::InstanceSharedPtr{new Extensions::AccessLoggers::File::FileAccessLog( access_log_path_, {}, AccessLog::AccessLogFormatUtils::defaultAccessLogFormatter(), @@ -254,7 +244,10 @@ class HttpConnectionManagerImplTest : public testing::Test, public ConnectionMan std::chrono::milliseconds streamIdleTimeout() const override { return stream_idle_timeout_; } std::chrono::milliseconds requestTimeout() const override { return request_timeout_; } std::chrono::milliseconds delayedCloseTimeout() const override { return delayed_close_timeout_; } - Router::RouteConfigProvider& routeConfigProvider() override { return route_config_provider_; } + Router::RouteConfigProvider* routeConfigProvider() override { return &route_config_provider_; } + Config::ConfigProvider* scopedRouteConfigProvider() override { + return &scoped_route_config_provider_; + } const std::string& serverName() override { return server_name_; } ConnectionManagerStats& stats() override { return stats_; } ConnectionManagerTracingStats& tracingStats() override { return tracing_stats_; } @@ -278,7 +271,8 @@ class HttpConnectionManagerImplTest : public testing::Test, public ConnectionMan bool shouldNormalizePath() const override { return normalize_path_; } DangerousDeprecatedTestTime test_time_; - RouteConfigProvider route_config_provider_; + ConnectionManagerImplHelper::RouteConfigProvider route_config_provider_; + ConnectionManagerImplHelper::ScopedRouteConfigProvider scoped_route_config_provider_; NiceMock tracer_; Stats::IsolatedStoreImpl fake_stats_; Http::ContextImpl http_context_; diff --git a/test/common/http/conn_manager_utility_test.cc b/test/common/http/conn_manager_utility_test.cc index d1bc974e8b8c3..637529509407a 100644 --- a/test/common/http/conn_manager_utility_test.cc +++ b/test/common/http/conn_manager_utility_test.cc @@ -55,7 +55,8 @@ class MockConnectionManagerConfig : public ConnectionManagerConfig { MOCK_CONST_METHOD0(streamIdleTimeout, std::chrono::milliseconds()); MOCK_CONST_METHOD0(requestTimeout, std::chrono::milliseconds()); MOCK_CONST_METHOD0(delayedCloseTimeout, std::chrono::milliseconds()); - MOCK_METHOD0(routeConfigProvider, Router::RouteConfigProvider&()); + MOCK_METHOD0(routeConfigProvider, Router::RouteConfigProvider*()); + MOCK_METHOD0(scopedRouteConfigProvider, Config::ConfigProvider*()); MOCK_METHOD0(serverName, const std::string&()); MOCK_METHOD0(stats, ConnectionManagerStats&()); MOCK_METHOD0(tracingStats, ConnectionManagerTracingStats&()); diff --git a/test/common/router/BUILD b/test/common/router/BUILD index de2a36450bfa3..1b47371bc9851 100644 --- a/test/common/router/BUILD +++ b/test/common/router/BUILD @@ -71,6 +71,22 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "scoped_rds_test", + srcs = ["scoped_rds_test.cc"], + deps = [ + "//source/common/config:utility_lib", + "//source/common/http:message_lib", + "//source/common/json:json_loader_lib", + "//source/common/router:scoped_rds_lib", + "//source/server/http:admin_lib", + "//test/mocks/init:init_mocks", + "//test/mocks/server:server_mocks", + "//test/test_common:simulated_time_system_lib", + "//test/test_common:utility_lib", + ], +) + envoy_cc_test( name = "retry_state_impl_test", srcs = ["retry_state_impl_test.cc"], diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc new file mode 100644 index 0000000000000..e254437e8c83f --- /dev/null +++ b/test/common/router/scoped_rds_test.cc @@ -0,0 +1,411 @@ +#include + +#include "envoy/admin/v2alpha/config_dump.pb.h" +#include "envoy/admin/v2alpha/config_dump.pb.validate.h" +#include "envoy/stats/scope.h" + +#include "common/router/scoped_rds.h" + +#include "test/mocks/server/mocks.h" +#include "test/test_common/simulated_time_system.h" + +#include "absl/strings/string_view.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::InSequence; +using testing::Return; + +namespace Envoy { +namespace Router { +namespace { + +envoy::api::v2::ScopedRouteConfiguration +parseScopedRouteConfigurationFromYaml(const std::string& yaml) { + envoy::api::v2::ScopedRouteConfiguration scoped_route_config; + MessageUtil::loadFromYaml(yaml, scoped_route_config); + return scoped_route_config; +} + +void parseScopedRouteConfigurationFromYaml(ProtobufWkt::Any& scoped_route_config, + const std::string& yaml) { + scoped_route_config.PackFrom(parseScopedRouteConfigurationFromYaml(yaml)); +} + +std::vector> +protosToMessageVec(std::vector&& protos) { + std::vector> messages; + for (const auto& proto : protos) { + Protobuf::Message* message = proto.New(); + message->CopyFrom(proto); + messages.push_back(std::unique_ptr(message)); + } + return messages; +} + +class ScopedRoutesTestBase : public testing::Test { +protected: + ScopedRoutesTestBase() { + EXPECT_CALL(factory_context_.admin_.config_tracker_, add_("route_scopes", _)); + config_provider_manager_ = + std::make_unique(factory_context_.admin_); + + const std::string rds_config_yaml = R"EOF( +api_config_source: + api_type: REST + cluster_names: + - foo_rds_cluster + refresh_delay: { seconds: 1, nanos: 0 } +)EOF"; + MessageUtil::loadFromYaml(rds_config_yaml, rds_config_source_); + } + + ~ScopedRoutesTestBase() override { factory_context_.thread_local_.shutdownThread(); } + + void setupMockClusterMap() { + InSequence s; + cluster_map_.emplace("foo_cluster", cluster_); + EXPECT_CALL(factory_context_.cluster_manager_, clusters()).WillOnce(Return(cluster_map_)); + EXPECT_CALL(cluster_, info()); + EXPECT_CALL(*cluster_.info_, addedViaApi()); + EXPECT_CALL(cluster_, info()); + EXPECT_CALL(*cluster_.info_, type()); + } + + Event::SimulatedTimeSystem& timeSystem() { return time_system_; } + + NiceMock factory_context_; + Upstream::ClusterManager::ClusterInfoMap cluster_map_; + Upstream::MockClusterMockPrioritySet cluster_; + std::unique_ptr config_provider_manager_; + Event::SimulatedTimeSystem time_system_; + envoy::api::v2::core::ConfigSource rds_config_source_; +}; + +class ScopedRdsTest : public ScopedRoutesTestBase { +protected: + ScopedRdsTest() = default; + + ~ScopedRdsTest() override = default; + + void setup() { + InSequence s; + + setupMockClusterMap(); + const std::string config_yaml = R"EOF( +name: foo_scoped_routes +scope_key_builder: + fragments: + - header_value_extractor: { name: X-Google-VIP } +rds_config_source: + api_config_source: + api_type: REST + cluster_names: + - foo_cluster + refresh_delay: { seconds: 1, nanos: 0 } +scoped_rds: + scoped_rds_config_source: + api_config_source: + api_type: REST + cluster_names: + - foo_cluster + refresh_delay: { seconds: 1, nanos: 0 } +)EOF"; + envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes scoped_routes_config; + MessageUtil::loadFromYaml(config_yaml, scoped_routes_config); + provider_ = config_provider_manager_->createXdsConfigProvider( + scoped_routes_config.scoped_rds(), factory_context_, "foo.", + ScopedRoutesConfigProviderManagerOptArg(scoped_routes_config.name(), + scoped_routes_config.rds_config_source(), + scoped_routes_config.scope_key_builder())); + subscription_ = &dynamic_cast(*provider_).subscription(); + } + + ScopedRdsConfigSubscription& subscription() const { return *subscription_; } + + ScopedRdsConfigSubscription* subscription_; + Envoy::Config::ConfigProviderPtr provider_; +}; + +TEST_F(ScopedRdsTest, ValidateFail) { + setup(); + + ScopedRdsConfigSubscription& subscription = + dynamic_cast(*provider_).subscription(); + + // 'name' validation: value must be > 1 byte. + const std::string config_yaml = R"EOF( +name: +route_configuration_name: foo_routes +key: + fragments: + - string_key: x-foo-key +)EOF"; + Protobuf::RepeatedPtrField resources; + parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); + EXPECT_THROW(subscription.onConfigUpdate(resources, "1"), ProtoValidationException); + + // 'route_configuration_name' validation: value must be > 1 byte. + const std::string config_yaml2 = R"EOF( +name: foo_scope +route_configuration_name: +key: + fragments: + - string_key: x-foo-key +)EOF"; + Protobuf::RepeatedPtrField resources2; + parseScopedRouteConfigurationFromYaml(*resources2.Add(), config_yaml2); + EXPECT_THROW(subscription.onConfigUpdate(resources2, "1"), ProtoValidationException); + + // 'key' validation: must define at least 1 fragment. + const std::string config_yaml3 = R"EOF( +name: foo_scope +route_configuration_name: foo_routes +key: +)EOF"; + Protobuf::RepeatedPtrField resources3; + parseScopedRouteConfigurationFromYaml(*resources3.Add(), config_yaml3); + EXPECT_THROW(subscription.onConfigUpdate(resources3, "1"), ProtoValidationException); +} + +// Tests that an empty config update will update the corresponding stat. +TEST_F(ScopedRdsTest, EmptyResource) { + setup(); + + Protobuf::RepeatedPtrField resources; + subscription().onConfigUpdate(resources, "1"); + EXPECT_EQ( + 1UL, + factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.update_empty").value()); +} + +// Tests that multiple uniquely named resources are allowed in config updates. +TEST_F(ScopedRdsTest, MultipleResources) { + setup(); + + const std::string config_yaml = R"EOF( +name: foo_scope +route_configuration_name: foo_routes +key: + fragments: + - string_key: x-foo-key +)EOF"; + Protobuf::RepeatedPtrField resources; + parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); + const std::string config_yaml2 = R"EOF( +name: foo_scope2 +route_configuration_name: foo_routes +key: + fragments: + - string_key: x-foo-key +)EOF"; + parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml2); + EXPECT_NO_THROW(subscription().onConfigUpdate(resources, "1")); + EXPECT_EQ( + 1UL, + factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); +} + +// Tests that only one resource is provided during a config update. +TEST_F(ScopedRdsTest, InvalidDuplicateResource) { + setup(); + + const std::string config_yaml = R"EOF( +name: foo_scope +route_configuration_name: foo_routes +key: + fragments: + - string_key: x-foo-key +)EOF"; + Protobuf::RepeatedPtrField resources; + parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); + parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); + EXPECT_THROW_WITH_MESSAGE(subscription().onConfigUpdate(resources, "1"), EnvoyException, + "duplicate scoped route configuration foo_scope found"); +} + +// Tests that defining an invalid cluster in the SRDS config results in an error. +TEST_F(ScopedRdsTest, UnknownCluster) { + const std::string config_yaml = R"EOF( +name: foo_scoped_routes +scope_key_builder: + fragments: + - header_value_extractor: { name: X-Google-VIP } +rds_config_source: + api_config_source: + api_type: REST + cluster_names: + - foo_cluster + refresh_delay: { seconds: 1, nanos: 0 } +scoped_rds: + scoped_rds_config_source: + api_config_source: + api_type: REST + cluster_names: + - foo_cluster + refresh_delay: { seconds: 1, nanos: 0 } +)EOF"; + envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes scoped_routes_config; + MessageUtil::loadFromYaml(config_yaml, scoped_routes_config); + + Upstream::ClusterManager::ClusterInfoMap cluster_map; + EXPECT_CALL(factory_context_.cluster_manager_, clusters()).WillOnce(Return(cluster_map)); + EXPECT_THROW_WITH_MESSAGE( + config_provider_manager_->createXdsConfigProvider( + scoped_routes_config.scoped_rds(), factory_context_, "foo.", + ScopedRoutesConfigProviderManagerOptArg(scoped_routes_config.name(), + scoped_routes_config.rds_config_source(), + scoped_routes_config.scope_key_builder())), + EnvoyException, + "envoy::api::v2::core::ConfigSource must have a statically defined non-EDS " + "cluster: 'foo_cluster' does not exist, was added via api, or is an " + "EDS cluster"); +} + +class ScopedRoutesConfigProviderManagerTest : public ScopedRoutesTestBase { +public: + ScopedRoutesConfigProviderManagerTest() = default; + + ~ScopedRoutesConfigProviderManagerTest() override = default; +}; + +// Tests that the /config_dump handler returns the corresponding scoped routing config. +TEST_F(ScopedRoutesConfigProviderManagerTest, ConfigDump) { + auto message_ptr = + factory_context_.admin_.config_tracker_.config_tracker_callbacks_["route_scopes"](); + const auto& scoped_routes_config_dump = + MessageUtil::downcastAndValidate( + *message_ptr); + + // No routes at all, no last_updated timestamp + envoy::admin::v2alpha::ScopedRoutesConfigDump expected_config_dump; + MessageUtil::loadFromYaml(R"EOF( +inline_scoped_route_configs: +dynamic_scoped_route_configs: +)EOF", + expected_config_dump); + EXPECT_EQ(expected_config_dump.DebugString(), scoped_routes_config_dump.DebugString()); + + const std::string config_yaml = R"EOF( +name: foo +route_configuration_name: foo-route-config +key: + fragments: { string_key: "172.10.10.10" } +)EOF"; + const std::string config_yaml2 = R"EOF( +name: foo2 +route_configuration_name: foo-route-config2 +key: + fragments: { string_key: "172.10.10.20" } +)EOF"; + std::vector> config_protos = + protosToMessageVec({parseScopedRouteConfigurationFromYaml(config_yaml), + parseScopedRouteConfigurationFromYaml(config_yaml2)}); + + timeSystem().setSystemTime(std::chrono::milliseconds(1234567891234)); + + envoy::config::filter::network::http_connection_manager::v2 ::ScopedRoutes::ScopeKeyBuilder + scope_key_builder; + MessageUtil::loadFromYaml(R"EOF( +fragments: + - header_value_extractor: { name: X-Google-VIP } +)EOF", + scope_key_builder); + // Only load the inline scopes. + Envoy::Config::ConfigProviderPtr inline_config = + config_provider_manager_->createStaticConfigProvider( + std::move(config_protos), factory_context_, + ScopedRoutesConfigProviderManagerOptArg("foo-scoped-routes", rds_config_source_, + scope_key_builder)); + message_ptr = factory_context_.admin_.config_tracker_.config_tracker_callbacks_["route_scopes"](); + const auto& scoped_routes_config_dump2 = + MessageUtil::downcastAndValidate( + *message_ptr); + MessageUtil::loadFromYaml(R"EOF( +inline_scoped_route_configs: + - name: foo-scoped-routes + scoped_route_configs: + - name: foo + route_configuration_name: foo-route-config + key: + fragments: { string_key: "172.10.10.10" } + - name: foo2 + route_configuration_name: foo-route-config2 + key: + fragments: { string_key: "172.10.10.20" } + last_updated: + seconds: 1234567891 + nanos: 234000000 +dynamic_scoped_route_configs: +)EOF", + expected_config_dump); + EXPECT_EQ(expected_config_dump.DebugString(), scoped_routes_config_dump2.DebugString()); + + setupMockClusterMap(); + envoy::config::filter::network::http_connection_manager::v2::ScopedRds scoped_rds_config; + const std::string config_source_yaml = R"EOF( +scoped_rds_config_source: + api_config_source: + api_type: REST + cluster_names: + - foo_cluster + refresh_delay: { seconds: 1, nanos: 0 } +)EOF"; + MessageUtil::loadFromYaml(config_source_yaml, scoped_rds_config); + Envoy::Config::ConfigProviderPtr dynamic_provider = + config_provider_manager_->createXdsConfigProvider( + scoped_rds_config, factory_context_, "foo.", + ScopedRoutesConfigProviderManagerOptArg("foo-dynamic-scoped-routes", rds_config_source_, + scope_key_builder)); + + Protobuf::RepeatedPtrField resources; + resources.Add()->PackFrom(parseScopedRouteConfigurationFromYaml(R"EOF( +name: dynamic-foo +route_configuration_name: dynamic-foo-route-config +key: + fragments: { string_key: "172.30.30.10" } +)EOF")); + + timeSystem().setSystemTime(std::chrono::milliseconds(1234567891567)); + ScopedRdsConfigSubscription& subscription = + dynamic_cast(*dynamic_provider).subscription(); + subscription.onConfigUpdate(resources, "1"); + + MessageUtil::loadFromYaml(R"EOF( +inline_scoped_route_configs: + - name: foo-scoped-routes + scoped_route_configs: + - name: foo + route_configuration_name: foo-route-config + key: + fragments: { string_key: "172.10.10.10" } + - name: foo2 + route_configuration_name: foo-route-config2 + key: + fragments: { string_key: "172.10.10.20" } + last_updated: + seconds: 1234567891 + nanos: 234000000 +dynamic_scoped_route_configs: + - name: foo-dynamic-scoped-routes + scoped_route_configs: + - name: dynamic-foo + route_configuration_name: dynamic-foo-route-config + key: + fragments: { string_key: "172.30.30.10" } + last_updated: + seconds: 1234567891 + nanos: 567000000 + version_info: "1" +)EOF", + expected_config_dump); + message_ptr = factory_context_.admin_.config_tracker_.config_tracker_callbacks_["route_scopes"](); + const auto& scoped_routes_config_dump3 = + MessageUtil::downcastAndValidate( + *message_ptr); + EXPECT_EQ(expected_config_dump.DebugString(), scoped_routes_config_dump3.DebugString()); +} + +} // namespace +} // namespace Router +} // namespace Envoy diff --git a/test/extensions/filters/network/http_connection_manager/config_test.cc b/test/extensions/filters/network/http_connection_manager/config_test.cc index 6feb80d6d37f4..2878fc33499a6 100644 --- a/test/extensions/filters/network/http_connection_manager/config_test.cc +++ b/test/extensions/filters/network/http_connection_manager/config_test.cc @@ -5,6 +5,7 @@ #include "extensions/filters/network/http_connection_manager/config.h" +#include "test/mocks/config/mocks.h" #include "test/mocks/http/mocks.h" #include "test/mocks/network/mocks.h" #include "test/mocks/server/mocks.h" @@ -37,6 +38,7 @@ class HttpConnectionManagerConfigTest : public testing::Test { NiceMock context_; Http::SlowDateProviderImpl date_provider_{context_.dispatcher().timeSource()}; NiceMock route_config_provider_manager_; + NiceMock scoped_routes_config_provider_manager_; }; TEST_F(HttpConnectionManagerConfigTest, ValidateFail) { @@ -68,7 +70,8 @@ stat_prefix: router EXPECT_THROW_WITH_MESSAGE( HttpConnectionManagerConfig(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, - date_provider_, route_config_provider_manager_), + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_), EnvoyException, "Didn't find a registered implementation for name: 'foo'"); } @@ -97,7 +100,8 @@ stat_prefix: router )EOF"; HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, - date_provider_, route_config_provider_manager_); + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); EXPECT_THAT(std::vector({Http::LowerCaseString("foo")}), ContainerEq(config.tracingConfig()->request_headers_for_tags_)); @@ -118,7 +122,8 @@ TEST_F(HttpConnectionManagerConfigTest, UnixSocketInternalAddress) { )EOF"; HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, - date_provider_, route_config_provider_manager_); + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); Network::Address::PipeInstance unixAddress{"/foo"}; Network::Address::Ipv4Instance internalIpAddress{"127.0.0.1", 0}; Network::Address::Ipv4Instance externalIpAddress{"12.0.0.1", 0}; @@ -137,7 +142,8 @@ TEST_F(HttpConnectionManagerConfigTest, MaxRequestHeadersKbDefault) { )EOF"; HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, - date_provider_, route_config_provider_manager_); + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); EXPECT_EQ(60, config.maxRequestHeadersKb()); } @@ -152,7 +158,8 @@ TEST_F(HttpConnectionManagerConfigTest, MaxRequestHeadersKbConfigured) { )EOF"; HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, - date_provider_, route_config_provider_manager_); + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); EXPECT_EQ(16, config.maxRequestHeadersKb()); } @@ -167,7 +174,8 @@ TEST_F(HttpConnectionManagerConfigTest, MaxRequestHeadersKbMaxConfigurable) { )EOF"; HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, - date_provider_, route_config_provider_manager_); + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); EXPECT_EQ(96, config.maxRequestHeadersKb()); } @@ -183,7 +191,8 @@ TEST_F(HttpConnectionManagerConfigTest, DisabledStreamIdleTimeout) { )EOF"; HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, - date_provider_, route_config_provider_manager_); + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); EXPECT_EQ(0, config.streamIdleTimeout().count()); } @@ -202,7 +211,8 @@ TEST_F(HttpConnectionManagerConfigTest, NormalizePathDefault) { .WillOnce(Invoke(&context_.runtime_loader_.snapshot_, &Runtime::MockSnapshot::featureEnabledDefault)); HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, - date_provider_, route_config_provider_manager_); + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); #ifdef ENVOY_NORMALIZE_PATH_BY_DEFAULT EXPECT_TRUE(config.shouldNormalizePath()); #else @@ -224,7 +234,8 @@ TEST_F(HttpConnectionManagerConfigTest, NormalizePathRuntime) { featureEnabled("http_connection_manager.normalize_path", An())) .WillOnce(Return(true)); HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, - date_provider_, route_config_provider_manager_); + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); EXPECT_TRUE(config.shouldNormalizePath()); } @@ -243,7 +254,8 @@ TEST_F(HttpConnectionManagerConfigTest, NormalizePathTrue) { featureEnabled("http_connection_manager.normalize_path", An())) .Times(0); HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, - date_provider_, route_config_provider_manager_); + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); EXPECT_TRUE(config.shouldNormalizePath()); } @@ -262,7 +274,8 @@ TEST_F(HttpConnectionManagerConfigTest, NormalizePathFalse) { featureEnabled("http_connection_manager.normalize_path", An())) .Times(0); HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, - date_provider_, route_config_provider_manager_); + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); EXPECT_FALSE(config.shouldNormalizePath()); } @@ -277,7 +290,8 @@ TEST_F(HttpConnectionManagerConfigTest, ConfiguredRequestTimeout) { )EOF"; HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, - date_provider_, route_config_provider_manager_); + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); EXPECT_EQ(53 * 1000, config.requestTimeout().count()); } @@ -292,7 +306,8 @@ TEST_F(HttpConnectionManagerConfigTest, DisabledRequestTimeout) { )EOF"; HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, - date_provider_, route_config_provider_manager_); + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); EXPECT_EQ(0, config.requestTimeout().count()); } @@ -306,7 +321,8 @@ TEST_F(HttpConnectionManagerConfigTest, UnconfiguredRequestTimeout) { )EOF"; HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, - date_provider_, route_config_provider_manager_); + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); EXPECT_EQ(0, config.requestTimeout().count()); } @@ -483,7 +499,8 @@ stat_prefix: router TEST_F(FilterChainTest, createFilterChain) { HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(basic_config_), context_, - date_provider_, route_config_provider_manager_); + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); Http::MockFilterChainFactoryCallbacks callbacks; EXPECT_CALL(callbacks, addStreamFilter(_)); // Dynamo @@ -497,7 +514,8 @@ TEST_F(FilterChainTest, createUpgradeFilterChain) { hcm_config.add_upgrade_configs()->set_upgrade_type("websocket"); HttpConnectionManagerConfig config(hcm_config, context_, date_provider_, - route_config_provider_manager_); + route_config_provider_manager_, + scoped_routes_config_provider_manager_); NiceMock callbacks; // Check the case where WebSockets are configured in the HCM, and no router @@ -543,7 +561,8 @@ TEST_F(FilterChainTest, createUpgradeFilterChainHCMDisabled) { hcm_config.mutable_upgrade_configs(0)->mutable_enabled()->set_value(false); HttpConnectionManagerConfig config(hcm_config, context_, date_provider_, - route_config_provider_manager_); + route_config_provider_manager_, + scoped_routes_config_provider_manager_); NiceMock callbacks; // Check the case where WebSockets are off in the HCM, and no router config is present. @@ -592,7 +611,8 @@ TEST_F(FilterChainTest, createCustomUpgradeFilterChain) { "envoy.http_dynamo_filter"); HttpConnectionManagerConfig config(hcm_config, context_, date_provider_, - route_config_provider_manager_); + route_config_provider_manager_, + scoped_routes_config_provider_manager_); { Http::MockFilterChainFactoryCallbacks callbacks; @@ -621,7 +641,8 @@ TEST_F(FilterChainTest, invalidConfig) { hcm_config.add_upgrade_configs()->set_upgrade_type("websocket"); EXPECT_THROW_WITH_MESSAGE(HttpConnectionManagerConfig(hcm_config, context_, date_provider_, - route_config_provider_manager_), + route_config_provider_manager_, + scoped_routes_config_provider_manager_), EnvoyException, "Error: multiple upgrade configs with the same name: 'websocket'"); } diff --git a/test/integration/BUILD b/test/integration/BUILD index 8b11b96571885..63e6b6a2c7ffe 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -712,3 +712,20 @@ envoy_cc_fuzz_test( ":h1_fuzz_lib", ], ) + +envoy_cc_test( + name = "scoped_rds_integration_test", + srcs = [ + "scoped_rds_integration_test.cc", + ], + deps = [ + ":http_integration_lib", + "//source/common/config:resources_lib", + "//source/common/event:dispatcher_includes", + "//source/common/event:dispatcher_lib", + "//source/common/network:connection_lib", + "//source/common/network:utility_lib", + "//test/common/grpc:grpc_client_integration_lib", + "//test/test_common:utility_lib", + ], +) diff --git a/test/integration/integration_admin_test.cc b/test/integration/integration_admin_test.cc index 2ce9f3b73b347..c76d4e6bf3cbb 100644 --- a/test/integration/integration_admin_test.cc +++ b/test/integration/integration_admin_test.cc @@ -347,7 +347,9 @@ TEST_P(IntegrationAdminTest, Admin) { "type.googleapis.com/envoy.admin.v2alpha.BootstrapConfigDump", "type.googleapis.com/envoy.admin.v2alpha.ClustersConfigDump", "type.googleapis.com/envoy.admin.v2alpha.ListenersConfigDump", + "type.googleapis.com/envoy.admin.v2alpha.ScopedRoutesConfigDump", "type.googleapis.com/envoy.admin.v2alpha.RoutesConfigDump"}; + for (Json::ObjectSharedPtr obj_ptr : json->getObjectArray("configs")) { EXPECT_TRUE(expected_types[index].compare(obj_ptr->getString("@type")) == 0); index++; @@ -356,11 +358,11 @@ TEST_P(IntegrationAdminTest, Admin) { // Validate we can parse as proto. envoy::admin::v2alpha::ConfigDump config_dump; MessageUtil::loadFromJson(response->body(), config_dump); - EXPECT_EQ(4, config_dump.configs_size()); + EXPECT_EQ(5, config_dump.configs_size()); // .. and that we can unpack one of the entries. envoy::admin::v2alpha::RoutesConfigDump route_config_dump; - config_dump.configs(3).UnpackTo(&route_config_dump); + config_dump.configs(4).UnpackTo(&route_config_dump); EXPECT_EQ("route_config_0", route_config_dump.static_route_configs(0).route_config().name()); } diff --git a/test/integration/scoped_rds_integration_test.cc b/test/integration/scoped_rds_integration_test.cc new file mode 100644 index 0000000000000..afc0897e0b84f --- /dev/null +++ b/test/integration/scoped_rds_integration_test.cc @@ -0,0 +1,198 @@ +#include "envoy/api/v2/srds.pb.h" + +#include "common/config/resources.h" + +#include "test/common/grpc/grpc_client_integration.h" +#include "test/integration/http_integration.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace { + +class ScopedRdsIntegrationTest : public HttpIntegrationTest, + public Grpc::GrpcClientIntegrationParamTest { +protected: + struct FakeUpstreamInfo { + FakeHttpConnectionPtr connection_; + FakeUpstream* upstream_{}; + FakeStreamPtr stream_; + }; + + ScopedRdsIntegrationTest() + : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, ipVersion(), realTime()) {} + + ~ScopedRdsIntegrationTest() override { + resetConnections(); + cleanupUpstreamAndDownstream(); + } + + void initialize() override { + config_helper_.addConfigModifier([](envoy::config::bootstrap::v2::Bootstrap& bootstrap) { + // Add the static cluster to serve SRDS. + auto* scoped_rds_cluster = bootstrap.mutable_static_resources()->add_clusters(); + scoped_rds_cluster->MergeFrom(bootstrap.static_resources().clusters()[0]); + scoped_rds_cluster->set_name("srds_cluster"); + scoped_rds_cluster->mutable_http2_protocol_options(); + + // Add the static cluster to serve RDS. + auto* rds_cluster = bootstrap.mutable_static_resources()->add_clusters(); + rds_cluster->MergeFrom(bootstrap.static_resources().clusters()[0]); + rds_cluster->set_name("rds_cluster"); + rds_cluster->mutable_http2_protocol_options(); + }); + + config_helper_.addConfigModifier( + [this](envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& + http_connection_manager) { + const std::string& scope_key_builder_config_yaml = R"EOF( +fragments: + - header_value_extractor: { name: X-Google-VIP } +)EOF"; + envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder + scope_key_builder; + MessageUtil::loadFromYaml(scope_key_builder_config_yaml, scope_key_builder); + auto* scoped_routes = http_connection_manager.mutable_scoped_routes(); + scoped_routes->set_name("foo-scoped-routes"); + *scoped_routes->mutable_scope_key_builder() = scope_key_builder; + + envoy::api::v2::core::ApiConfigSource* rds_api_config_source = + scoped_routes->mutable_rds_config_source()->mutable_api_config_source(); + rds_api_config_source->set_api_type(envoy::api::v2::core::ApiConfigSource::GRPC); + envoy::api::v2::core::GrpcService* grpc_service = + rds_api_config_source->add_grpc_services(); + setGrpcService(*grpc_service, "rds_cluster", getRdsFakeUpstream().localAddress()); + + envoy::api::v2::core::ApiConfigSource* srds_api_config_source = + scoped_routes->mutable_scoped_rds() + ->mutable_scoped_rds_config_source() + ->mutable_api_config_source(); + srds_api_config_source->set_api_type(envoy::api::v2::core::ApiConfigSource::GRPC); + grpc_service = srds_api_config_source->add_grpc_services(); + setGrpcService(*grpc_service, "srds_cluster", getScopedRdsFakeUpstream().localAddress()); + }); + + HttpIntegrationTest::initialize(); + } + + void createUpstreams() override { + HttpIntegrationTest::createUpstreams(); + // Create the SRDS upstream. + fake_upstreams_.emplace_back(new FakeUpstream(0, FakeHttpConnection::Type::HTTP2, version_, + timeSystem(), enable_half_close_)); + // Create the RDS upstream. + fake_upstreams_.emplace_back(new FakeUpstream(0, FakeHttpConnection::Type::HTTP2, version_, + timeSystem(), enable_half_close_)); + } + + void resetFakeUpstreamInfo(FakeUpstreamInfo* upstream_info) { + ASSERT(upstream_info->upstream_ != nullptr); + + upstream_info->upstream_->set_allow_unexpected_disconnects(true); + AssertionResult result = upstream_info->connection_->close(); + RELEASE_ASSERT(result, result.message()); + result = upstream_info->connection_->waitForDisconnect(); + RELEASE_ASSERT(result, result.message()); + upstream_info->connection_.reset(); + } + + void resetConnections() { + if (rds_upstream_info_.upstream_ != nullptr) { + resetFakeUpstreamInfo(&rds_upstream_info_); + } + resetFakeUpstreamInfo(&scoped_rds_upstream_info_); + } + + FakeUpstream& getRdsFakeUpstream() const { return *fake_upstreams_[2]; } + + FakeUpstream& getScopedRdsFakeUpstream() const { return *fake_upstreams_[1]; } + + void createStream(FakeUpstreamInfo* upstream_info, FakeUpstream& upstream) { + upstream_info->upstream_ = &upstream; + AssertionResult result = + upstream_info->upstream_->waitForHttpConnection(*dispatcher_, upstream_info->connection_); + RELEASE_ASSERT(result, result.message()); + result = upstream_info->connection_->waitForNewStream(*dispatcher_, upstream_info->stream_); + RELEASE_ASSERT(result, result.message()); + upstream_info->stream_->startGrpcStream(); + } + + void createRdsStream() { createStream(&rds_upstream_info_, getRdsFakeUpstream()); } + + void createScopedRdsStream() { + createStream(&scoped_rds_upstream_info_, getScopedRdsFakeUpstream()); + } + + void sendScopedRdsResponse(const std::vector& resource_protos, + const std::string& version) { + ASSERT(scoped_rds_upstream_info_.stream_ != nullptr); + + envoy::api::v2::DiscoveryResponse response; + response.set_version_info(version); + response.set_type_url(Config::TypeUrl::get().ScopedRouteConfiguration); + + for (const auto& resource_proto : resource_protos) { + envoy::api::v2::ScopedRouteConfiguration scoped_route_proto; + MessageUtil::loadFromYaml(resource_proto, scoped_route_proto); + response.add_resources()->PackFrom(scoped_route_proto); + } + + scoped_rds_upstream_info_.stream_->sendGrpcMessage(response); + } + + FakeUpstreamInfo scoped_rds_upstream_info_; + FakeUpstreamInfo rds_upstream_info_; +}; + +INSTANTIATE_TEST_CASE_P(IpVersionsAndGrpcTypes, ScopedRdsIntegrationTest, + GRPC_CLIENT_INTEGRATION_PARAMS); + +TEST_P(ScopedRdsIntegrationTest, BasicSuccess) { + const std::string scope_route1 = R"EOF( +name: foo_scope1 +route_configuration_name: foo_route1 +key: + fragments: + - string_key: x-foo-key +)EOF"; + const std::string scope_route2 = R"EOF( +name: foo_scope2 +route_configuration_name: foo_route2 +key: + fragments: + - string_key: x-foo-key +)EOF"; + + on_server_init_function_ = [this, &scope_route1, &scope_route2]() { + createScopedRdsStream(); + sendScopedRdsResponse({scope_route1, scope_route2}, "1"); + }; + initialize(); + + test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_attempt", 1); + test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_success", 1); + // The version gauge should be set to xxHash64("1"). + test_server_->waitForGaugeEq("http.config_test.scoped_rds.foo-scoped-routes.version", + 13237225503670494420UL); + + const std::string scope_route3 = R"EOF( +name: foo_scope3 +route_configuration_name: foo_route3 +key: + fragments: + - string_key: x-baz-key +)EOF"; + sendScopedRdsResponse({scope_route3}, "2"); + + test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_attempt", 2); + test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_success", 2); + test_server_->waitForGaugeEq("http.config_test.scoped_rds.foo-scoped-routes.version", + 6927017134761466251UL); + + // TODO(AndresGuedez): test actual scoped routing logic; only the config handling is implemented + // at this point. +} + +} // namespace +} // namespace Envoy diff --git a/test/mocks/config/BUILD b/test/mocks/config/BUILD index 4ffbef91dbbfb..4f16cdf5a8afb 100644 --- a/test/mocks/config/BUILD +++ b/test/mocks/config/BUILD @@ -13,6 +13,7 @@ envoy_cc_mock( srcs = ["mocks.cc"], hdrs = ["mocks.h"], deps = [ + "//include/envoy/config:config_provider_manager_interface", "//include/envoy/config:grpc_mux_interface", "//include/envoy/config:subscription_interface", "//include/envoy/config:xds_grpc_context_interface", diff --git a/test/mocks/config/mocks.h b/test/mocks/config/mocks.h index 9cc82f2380834..65df8984321a6 100644 --- a/test/mocks/config/mocks.h +++ b/test/mocks/config/mocks.h @@ -1,6 +1,7 @@ #pragma once #include "envoy/api/v2/eds.pb.h" +#include "envoy/config/config_provider_manager.h" #include "envoy/config/grpc_mux.h" #include "envoy/config/subscription.h" #include "envoy/config/xds_grpc_context.h" @@ -91,5 +92,26 @@ class MockGrpcStreamCallbacks : public GrpcStreamCallbacks>&& config_protos, + Server::Configuration::FactoryContext& factory_context, + const Envoy::Config::ConfigProviderManager::OptionalArg& optarg)); +}; + } // namespace Config } // namespace Envoy diff --git a/test/mocks/router/BUILD b/test/mocks/router/BUILD index 76e2c77be171b..c6e3b9065f71e 100644 --- a/test/mocks/router/BUILD +++ b/test/mocks/router/BUILD @@ -19,6 +19,7 @@ envoy_cc_mock( "//include/envoy/router:route_config_provider_manager_interface", "//include/envoy/router:router_interface", "//include/envoy/router:router_ratelimit_interface", + "//include/envoy/router:scopes_interface", "//include/envoy/router:shadow_writer_interface", "//include/envoy/runtime:runtime_interface", "//include/envoy/stats:stats_interface", diff --git a/test/mocks/router/mocks.h b/test/mocks/router/mocks.h index ba40815aaab49..07a01dc722db3 100644 --- a/test/mocks/router/mocks.h +++ b/test/mocks/router/mocks.h @@ -16,6 +16,7 @@ #include "envoy/router/route_config_provider_manager.h" #include "envoy/router/router.h" #include "envoy/router/router_ratelimit.h" +#include "envoy/router/scopes.h" #include "envoy/router/shadow_writer.h" #include "envoy/runtime/runtime.h" #include "envoy/thread_local/thread_local.h" @@ -356,5 +357,13 @@ class MockRouteConfigProviderManager : public RouteConfigProviderManager { Server::Configuration::FactoryContext& factory_context)); }; +class MockScopedConfig : public ScopedConfig { +public: + MockScopedConfig() = default; + ~MockScopedConfig() override = default; + + MOCK_CONST_METHOD1(getRouterConfig, ConfigConstSharedPtr(const Http::HeaderMap& headers)); +}; + } // namespace Router } // namespace Envoy diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index 1240f39f89a5f..f98c4ef239702 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -224,6 +224,7 @@ SPIFFE SPKI SQL SR +SRDS SRV SS SSL @@ -344,6 +345,7 @@ comparator cond condvar conf +configdump conn conns const @@ -763,6 +765,7 @@ usr util utils validator +validators vanishingly var variadic @@ -772,6 +775,7 @@ verifiers versa versioned vhost +vip vptr vptrs wakeup