From cdae885870383839644a52e9edbc6e893838bb3c Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Tue, 5 Feb 2019 10:32:11 -0500 Subject: [PATCH 01/37] Support scoped routing configuration. This commit introduces: - Support for scoped routing configuration parsing. - Inline and dynamic config providers using the ConfigProvider framework. - Unit and integration testing scaffolding. This is the second step in the PR chain for #4704. Signed-off-by: Andres Guedez --- api/envoy/admin/v2alpha/BUILD | 1 + api/envoy/admin/v2alpha/config_dump.proto | 35 +++ api/envoy/api/v2/BUILD | 21 ++ api/envoy/api/v2/srds.proto | 150 +++++++++ .../network/http_connection_manager/v2/BUILD | 2 + .../v2/http_connection_manager.proto | 41 ++- docs/build.sh | 1 + docs/root/api-v2/http_routes/http_routes.rst | 1 + .../envoy/config/config_provider_manager.h | 18 +- include/envoy/router/BUILD | 10 + include/envoy/router/scopes.h | 30 ++ source/common/config/config_provider_impl.h | 5 +- source/common/config/resources.h | 2 + source/common/http/BUILD | 2 + source/common/http/conn_manager_config.h | 12 +- source/common/http/conn_manager_impl.cc | 41 ++- source/common/http/conn_manager_impl.h | 2 + source/common/router/BUILD | 28 ++ source/common/router/scoped_config_impl.cc | 13 + source/common/router/scoped_config_impl.h | 47 +++ source/common/router/scoped_rds.cc | 191 ++++++++++++ source/common/router/scoped_rds.h | 177 +++++++++++ .../network/http_connection_manager/BUILD | 2 + .../network/http_connection_manager/config.cc | 69 +++- .../network/http_connection_manager/config.h | 13 +- source/server/http/BUILD | 1 + source/server/http/admin.cc | 1 + source/server/http/admin.h | 27 +- .../config/config_provider_impl_test.cc | 29 +- .../http/conn_manager_impl_fuzz_test.cc | 24 +- test/common/http/conn_manager_impl_test.cc | 25 +- test/common/http/conn_manager_utility_test.cc | 3 +- test/common/router/BUILD | 16 + test/common/router/scoped_rds_test.cc | 295 ++++++++++++++++++ .../http_connection_manager/config_test.cc | 178 ++++++++++- test/integration/BUILD | 17 + test/integration/integration_admin_test.cc | 6 +- .../scoped_rds_integration_test.cc | 186 +++++++++++ test/mocks/config/BUILD | 1 + test/mocks/config/mocks.h | 17 + test/mocks/router/BUILD | 1 + test/mocks/router/mocks.h | 9 + 42 files changed, 1681 insertions(+), 69 deletions(-) create mode 100644 api/envoy/api/v2/srds.proto create mode 100644 include/envoy/router/scopes.h create mode 100644 source/common/router/scoped_config_impl.cc create mode 100644 source/common/router/scoped_config_impl.h create mode 100644 source/common/router/scoped_rds.cc create mode 100644 source/common/router/scoped_rds.h create mode 100644 test/common/router/scoped_rds_test.cc create mode 100644 test/integration/scoped_rds_integration_test.cc diff --git a/api/envoy/admin/v2alpha/BUILD b/api/envoy/admin/v2alpha/BUILD index a6b403fdd23ec..bf091cbabc4ae 100644 --- a/api/envoy/admin/v2alpha/BUILD +++ b/api/envoy/admin/v2alpha/BUILD @@ -10,6 +10,7 @@ api_proto_library_internal( "//envoy/api/v2:cds", "//envoy/api/v2:lds", "//envoy/api/v2:rds", + "//envoy/api/v2:srds", "//envoy/config/bootstrap/v2:bootstrap", ], ) diff --git a/api/envoy/admin/v2alpha/config_dump.proto b/api/envoy/admin/v2alpha/config_dump.proto index 0cb1ad21e11ca..0fe2b88ebc043 100644 --- a/api/envoy/admin/v2alpha/config_dump.proto +++ b/api/envoy/admin/v2alpha/config_dump.proto @@ -6,6 +6,7 @@ option java_package = "io.envoyproxy.envoy.admin.v2alpha"; import "envoy/api/v2/cds.proto"; import "envoy/api/v2/lds.proto"; import "envoy/api/v2/rds.proto"; +import "envoy/api/v2/srds.proto"; import "envoy/config/bootstrap/v2/bootstrap.proto"; import "google/protobuf/any.proto"; @@ -175,3 +176,37 @@ message RoutesConfigDump { // The dynamically loaded route configs. repeated DynamicRouteConfig dynamic_route_configs = 3 [(gogoproto.nullable) = false]; } + +// Envoy's scoped RDS implementation fills this message with all currently loaded route +// configuration scopes (defined via ScopedRouteConfigurationsSet protos). This message lists both +// the scopes defined inline with the higher order object (i.e., the HttpConnectionManager) and the +// dynamically obtained scopes via the SRDS API. +message ScopedRoutesConfigDump { + message InlineScopedRoutesConfig { + // The scoped routes config. + envoy.api.v2.ScopedRouteConfigurationsSet scoped_routes_config = 1; + + // The timestamp when the scoped route config set was last updated. + google.protobuf.Timestamp last_updated = 2; + } + + message DynamicScopedRoutesConfig { + // This is the per-resource version information. This version is currently taken from the + // :ref:`version_info ` field at the time that + // the scoped routes configuration was loaded. + string version_info = 1; + + // The scoped routes config. + envoy.api.v2.ScopedRouteConfigurationsSet scoped_routes_config = 2; + + // The timestamp when the scoped route config set was last updated. + google.protobuf.Timestamp last_updated = 3; + } + + // The statically loaded scoped routes configs. + repeated InlineScopedRoutesConfig inline_scoped_routes_configs = 1 [(gogoproto.nullable) = false]; + + // The dynamically loaded scoped routes configs. + repeated DynamicScopedRoutesConfig dynamic_scoped_routes_configs = 2 + [(gogoproto.nullable) = false]; +} diff --git a/api/envoy/api/v2/BUILD b/api/envoy/api/v2/BUILD index 261d140819985..e035644a414eb 100644 --- a/api/envoy/api/v2/BUILD +++ b/api/envoy/api/v2/BUILD @@ -142,3 +142,24 @@ api_go_grpc_library( "//envoy/api/v2/route:route_go_proto", ], ) + +api_proto_library_internal( + name = "srds", + srcs = ["srds.proto"], + has_services = 1, + visibility = [":friends"], + deps = [ + ":discovery", + "//envoy/api/v2/core:base", + "//envoy/api/v2/route", + ], +) + +api_go_grpc_library( + name = "srds", + proto = ":srds", + deps = [ + ":discovery_go_proto", + "//envoy/api/v2/core:base_go_proto", + ], +) diff --git a/api/envoy/api/v2/srds.proto b/api/envoy/api/v2/srds.proto new file mode 100644 index 0000000000000..fcd6563f9ec36 --- /dev/null +++ b/api/envoy/api/v2/srds.proto @@ -0,0 +1,150 @@ +syntax = "proto3"; + +package envoy.api.v2; + +option java_package = "io.envoyproxy.envoy.api.v2"; +option java_multiple_files = true; +option java_generic_services = true; + +import "envoy/api/v2/discovery.proto"; + +import "google/api/annotations.proto"; + +import "validate/validate.proto"; +import "gogoproto/gogo.proto"; + +option (gogoproto.equal_all) = true; + +// [#protodoc-title: HTTP scoped routing configuration] +// * Routing :ref:`architecture overview ` + +// .. attention:: +// +// The Scoped RDS API is not yet fully implemented and *should not* be enabled in +// :ref:`envoy_api_msg_config.filter.network.http_connection_manager.v2.HttpConnectionManager`. + +// The resource_names field in DiscoveryRequest specifies a set of route configuration scopes. Each +// scope points to a route_configuration_name which will be used to request a +// :ref:`RouteConfiguration` via the RDS API. +service ScopedRoutesDiscoveryService { + rpc StreamScopedRoutes(stream DiscoveryRequest) returns (stream DiscoveryResponse) { + } + + rpc IncrementalScopedRoutes(stream IncrementalDiscoveryRequest) + returns (stream IncrementalDiscoveryResponse) { + } + + rpc FetchScopedRoutes(DiscoveryRequest) returns (DiscoveryResponse) { + option (google.api.http) = { + post: "/v2/discovery:scoped-routes" + body: "*" + }; + } +} + +// This configuration represents a set of "scopes", each containing independent routing +// configuration (see :ref:`routing architecture overview `). A scope is +// assigned to each request based on request attributes, such as the value of a header designated +// via this configuration. +// [#comment:next free field: 4] +message ScopedRouteConfigurationsSet { + // The name of the set of route configuration scopes. This will be the + // :ref:`scoped_routes_config_set_name ` + // set in :ref:`envoy_api_msg_config.filter.network.http_connection_manager.v2.ScopedRds`. + string name = 1 [(validate.rules).string.min_bytes = 1]; + + // Specifies the mechanism for constructing keys based on request attributes to match + // :ref:`envoy_api_field_api.v2.ScopedRouteConfigurationsSet.scopes` against. + // + // Upon receiving a request's headers, the Router will build a key using the algorithm specified + // by this message. This key will be used to look up a corresponding + // :ref:`Scope ` in a table containing the set + // of :ref:`scopes `. + message ScopeKeyBuilder { + // Specifies the mechanism for constructing fragments which are composed into keys. + message FragmentBuilder { + // Specifies how the value of a header should be extracted. + // The following example maps the structure of a header to the fields in this message. + // + // .. parsed-literal:: + // + // X-Header: a,b;c,d + // | || | + // | || \\----> + // | || + // | |\\----> + // | | + // | \\----> + // | + // \\----> + message HeaderValueExtractor { + // The name of the header to extract the value from. + string name = 1 [(validate.rules).string.min_bytes = 1]; + + // The element separator (e.g., ';' separates 'a;b;c;d'). + string element_separator = 2; + + // Specifies a key value pair to match on. + message KvElement { + // The separator between key and value (e.g., '=' separates 'k=v;...'). + string separator = 1; + + // The key to match on. + string key = 2; + } + + oneof extract_type { + // Specifies the index of the element to extract. + int32 index = 3; + + // Specifies the key value pair to extract the value from. + KvElement element = 4; + } + } + + oneof type { + option (validate.required) = true; + + // Specifies how a header field's value should be extracted. + HeaderValueExtractor header_value_extractor = 1; + } + } + + // The constructed key consists of the union of these fragments. + repeated FragmentBuilder fragments = 1 [(validate.rules).repeated.min_items = 1]; + } + + // The key construction mechanism. + ScopeKeyBuilder scope_key_builder = 2 [(validate.rules).message.required = true]; + + // Specifies a routing scope, which associates a :ref:`envoy_api_msg_RouteConfiguration` to a + // :ref:`Key ` which is matched against + // each HTTP request. + message Scope { + // Specifies a key which is matched against by the output of a :ref:`ScopeKeyBuilder`. + message Key { + message Fragment { + oneof type { + option (validate.required) = true; + + // A string to match against. + string string_key = 1; + } + } + + // The ordered set of fragments to match against. + repeated Fragment fragments = 1 [(validate.rules).repeated.min_items = 1]; + } + + // The resource name to use for a :ref:`envoy_api_msg_DiscoveryRequest` to an RDS server to + // fetch the :ref:`envoy_api_msg_RouteConfiguration` associated with this scope. + string route_configuration_name = 1 [(validate.rules).string.min_bytes = 1]; + + // The key to match against. + Key key = 2 [(validate.rules).message.required = true]; + } + + // The set of scopes containing :ref:`Key ` + // to :ref:`envoy_api_msg_RouteConfiguration` mappings. + repeated Scope scopes = 3 [(validate.rules).repeated.min_items = 1]; +} diff --git a/api/envoy/config/filter/network/http_connection_manager/v2/BUILD b/api/envoy/config/filter/network/http_connection_manager/v2/BUILD index c89ea09ad2909..3b2937eda289c 100644 --- a/api/envoy/config/filter/network/http_connection_manager/v2/BUILD +++ b/api/envoy/config/filter/network/http_connection_manager/v2/BUILD @@ -7,6 +7,7 @@ api_proto_library_internal( srcs = ["http_connection_manager.proto"], deps = [ "//envoy/api/v2:rds", + "//envoy/api/v2:srds", "//envoy/api/v2/core:base", "//envoy/api/v2/core:config_source", "//envoy/api/v2/core:protocol", @@ -20,6 +21,7 @@ api_go_proto_library( proto = ":http_connection_manager", deps = [ "//envoy/api/v2:rds_go_grpc", + "//envoy/api/v2:srds_go_grpc", "//envoy/api/v2/core:base_go_proto", "//envoy/api/v2/core:config_source_go_proto", "//envoy/api/v2/core:protocol_go_proto", diff --git a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto index 60e7728a86e0c..33554c745750b 100644 --- a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto +++ b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto @@ -7,6 +7,7 @@ option go_package = "v2"; import "envoy/api/v2/core/config_source.proto"; import "envoy/api/v2/core/protocol.proto"; import "envoy/api/v2/rds.proto"; +import "envoy/api/v2/srds.proto"; import "envoy/config/filter/accesslog/v2/accesslog.proto"; import "envoy/type/percent.proto"; @@ -21,7 +22,7 @@ import "gogoproto/gogo.proto"; // [#protodoc-title: HTTP connection manager] // HTTP connection manager :ref:`configuration overview `. -// [#comment:next free field: 30] +// [#comment:next free field: 32] message HttpConnectionManager { enum CodecType { option (gogoproto.goproto_enum_prefix) = false; @@ -60,6 +61,14 @@ message HttpConnectionManager { envoy.api.v2.RouteConfiguration route_config = 4; } + oneof scoped_routes_specifier { + // Configuration for the Scoped RDS API which dynamically loads routing configuration scopes. + ScopedRds scoped_rds = 30; + + // A static set of routing scopes. + envoy.api.v2.ScopedRouteConfigurationsSet scoped_routes_config = 31; + } + // A list of individual HTTP filters that make up the filter chain for // requests made to the connection manager. Order matters as the filters are // processed sequentially as request events happen. @@ -386,11 +395,31 @@ message Rds { envoy.api.v2.core.ConfigSource config_source = 1 [(validate.rules).message.required = true, (gogoproto.nullable) = false]; - // The name of the route configuration. This name will be passed to the RDS - // API. This allows an Envoy configuration with multiple HTTP listeners (and - // associated HTTP connection manager filters) to use different route - // configurations. - string route_config_name = 2 [(validate.rules).string.min_bytes = 1]; + oneof subscription_specifier { + option (validate.required) = true; + + // The name of the route configuration. This name will be passed to the RDS + // API. This allows an Envoy configuration with multiple HTTP listeners (and + // associated HTTP connection manager filters) to use different route + // configurations. + string route_config_name = 2 [(validate.rules).string.min_bytes = 1]; + + // Must be set to true when scoped RDS is enabled. + // If this is enabled, RDS subscriptions will be triggered by scoped RDS config updates using + // this configuration as a template. + bool scoped_rds_template = 3; + } +} + +message ScopedRds { + // Configuration source specifier for scoped RDS. + envoy.api.v2.core.ConfigSource config_source = 1 + [(validate.rules).message.required = true, (gogoproto.nullable) = false]; + + // The name of the set of routing configuration scopes. This name will be passed to the scoped RDS + // API. + // This allows Envoy to segment routing configuration based on a configurable request attribute. + string scoped_routes_config_set_name = 2 [(validate.rules).string.min_bytes = 1]; } message HttpFilter { diff --git a/docs/build.sh b/docs/build.sh index 426ec44422981..07abd21cfc293 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -71,6 +71,7 @@ PROTO_RST=" /envoy/api/v2/cluster/circuit_breaker/envoy/api/v2/cluster/circuit_breaker.proto.rst /envoy/api/v2/rds/envoy/api/v2/rds.proto.rst /envoy/api/v2/route/route/envoy/api/v2/route/route.proto.rst + /envoy/api/v2/srds/envoy/api/v2/srds.proto.rst /envoy/api/v2/lds/envoy/api/v2/lds.proto.rst /envoy/api/v2/listener/listener/envoy/api/v2/listener/listener.proto.rst /envoy/api/v2/ratelimit/ratelimit/envoy/api/v2/ratelimit/ratelimit.proto.rst diff --git a/docs/root/api-v2/http_routes/http_routes.rst b/docs/root/api-v2/http_routes/http_routes.rst index 45a2dbca1d930..241f94601a5c8 100644 --- a/docs/root/api-v2/http_routes/http_routes.rst +++ b/docs/root/api-v2/http_routes/http_routes.rst @@ -6,4 +6,5 @@ HTTP route management :maxdepth: 2 ../api/v2/rds.proto + ../api/v2/srds.proto ../api/v2/route/route.proto diff --git a/include/envoy/config/config_provider_manager.h b/include/envoy/config/config_provider_manager.h index 3c3eddba9c2f0..0202b85b29dbc 100644 --- a/include/envoy/config/config_provider_manager.h +++ b/include/envoy/config/config_provider_manager.h @@ -25,6 +25,17 @@ namespace Config { */ class ConfigProviderManager { public: + class OptionalArg { + public: + virtual ~OptionalArg() = default; + }; + + class NullOptionalArg : public OptionalArg { + public: + NullOptionalArg() = default; + ~NullOptionalArg() override = default; + }; + virtual ~ConfigProviderManager() = default; /** @@ -34,6 +45,7 @@ class ConfigProviderManager { * @param config_source_proto supplies the proto containing the xDS API configuration. * @param factory_context is the context to use for the provider. * @param stat_prefix supplies the prefix to use for statistics. + * @param optarg supplies an optional argument with data specific to the concrete class. * @return ConfigProviderPtr a newly allocated dynamic config provider which shares underlying * data structures with other dynamic providers configured with the same * API source. @@ -41,17 +53,19 @@ class ConfigProviderManager { virtual ConfigProviderPtr createXdsConfigProvider(const Protobuf::Message& config_source_proto, Server::Configuration::FactoryContext& factory_context, - const std::string& stat_prefix) PURE; + const std::string& stat_prefix, const OptionalArg& optarg) PURE; /** * Returns a ConfigProvider associated with a statically specified configuration. * @param config_proto supplies the configuration proto. * @param factory_context is the context to use for the provider. + * @param optarg supplies an optional argument with data specific to the concrete class. * @return ConfigProviderPtr a newly allocated static config provider. */ virtual ConfigProviderPtr createStaticConfigProvider(const Protobuf::Message& config_proto, - Server::Configuration::FactoryContext& factory_context) PURE; + Server::Configuration::FactoryContext& factory_context, + const OptionalArg& optarg) PURE; }; } // namespace Config diff --git a/include/envoy/router/BUILD b/include/envoy/router/BUILD index f47a6db92ec04..c17b0c0236b70 100644 --- a/include/envoy/router/BUILD +++ b/include/envoy/router/BUILD @@ -51,6 +51,16 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "scopes_interface", + hdrs = ["scopes.h"], + deps = [ + ":router_interface", + "//include/envoy/config:config_provider_interface", + "//include/envoy/http:header_map_interface", + ], +) + envoy_cc_library( name = "router_ratelimit_interface", hdrs = ["router_ratelimit.h"], diff --git a/include/envoy/router/scopes.h b/include/envoy/router/scopes.h new file mode 100644 index 0000000000000..ed675fcab8e46 --- /dev/null +++ b/include/envoy/router/scopes.h @@ -0,0 +1,30 @@ +#pragma once + +#include + +#include "envoy/config/config_provider.h" +#include "envoy/router/router.h" + +namespace Envoy { +namespace Router { + +/** + * The scoped routing configuration. + */ +class ScopedConfig : public Envoy::Config::ConfigProvider::Config { +public: + virtual ~ScopedConfig() {} + + /** + * Based on the incoming HTTP request headers, returns the configuration to use for selecting a + * target route (to either a route entry or a direct response entry). + * @param headers the request headers to match the scoped routing configuration against. + * @return ConfigConstSharedPtr the router's Config matching the request headers. + */ + virtual ConfigConstSharedPtr getRouterConfig(const Http::HeaderMap& headers) const PURE; +}; + +using ScopedConfigConstSharedPtr = std::shared_ptr; + +} // namespace Router +} // namespace Envoy diff --git a/source/common/config/config_provider_impl.h b/source/common/config/config_provider_impl.h index 884cbcaf2be31..338f0e3ec07a2 100644 --- a/source/common/config/config_provider_impl.h +++ b/source/common/config/config_provider_impl.h @@ -173,7 +173,10 @@ class ConfigSubscriptionInstanceBase : public Init::Target, * Must be called by derived classes when the onConfigUpdateFailed() callback associated with the * underlying subscription is issued. */ - void onConfigUpdateFailed() { runInitializeCallbackIfAny(); } + void onConfigUpdateFailed() { + setLastUpdated(); + runInitializeCallbackIfAny(); + } /** * Determines whether a configuration proto is a new update, and if so, propagates it to all diff --git a/source/common/config/resources.h b/source/common/config/resources.h index 69ed2d91a46dc..d18ae493c6040 100644 --- a/source/common/config/resources.h +++ b/source/common/config/resources.h @@ -17,6 +17,8 @@ class TypeUrlValues { const std::string ClusterLoadAssignment{"type.googleapis.com/envoy.api.v2.ClusterLoadAssignment"}; const std::string Secret{"type.googleapis.com/envoy.api.v2.auth.Secret"}; const std::string RouteConfiguration{"type.googleapis.com/envoy.api.v2.RouteConfiguration"}; + const std::string ScopedRouteConfigurationsSet{ + "type.googleapis.com/envoy.api.v2.ScopedRouteConfigurationsSet"}; }; typedef ConstSingleton TypeUrl; diff --git a/source/common/http/BUILD b/source/common/http/BUILD index 903f861828657..e493b073138da 100644 --- a/source/common/http/BUILD +++ b/source/common/http/BUILD @@ -119,6 +119,7 @@ envoy_cc_library( hdrs = ["conn_manager_config.h"], deps = [ ":date_provider_lib", + "//include/envoy/config:config_provider_interface", "//include/envoy/http:filter_interface", "//include/envoy/router:rds_interface", "//source/common/network:utility_lib", @@ -157,6 +158,7 @@ envoy_cc_library( "//include/envoy/network:drain_decision_interface", "//include/envoy/network:filter_interface", "//include/envoy/router:rds_interface", + "//include/envoy/router:scopes_interface", "//include/envoy/runtime:runtime_interface", "//include/envoy/server:overload_manager_interface", "//include/envoy/ssl:connection_interface", diff --git a/source/common/http/conn_manager_config.h b/source/common/http/conn_manager_config.h index ee7b486aaf973..b2db5c152c394 100644 --- a/source/common/http/conn_manager_config.h +++ b/source/common/http/conn_manager_config.h @@ -1,5 +1,6 @@ #pragma once +#include "envoy/config/config_provider.h" #include "envoy/http/filter.h" #include "envoy/router/rds.h" #include "envoy/stats/scope.h" @@ -252,10 +253,17 @@ class ConnectionManagerConfig { virtual std::chrono::milliseconds delayedCloseTimeout() const PURE; /** - * @return Router::RouteConfigProvider& the configuration provider used to acquire a route + * @return Router::RouteConfigProvider* the configuration provider used to acquire a route * config for each request flow. */ - virtual Router::RouteConfigProvider& routeConfigProvider() PURE; + virtual Router::RouteConfigProvider* routeConfigProvider() PURE; + + /** + * @return Config::ConfigProvider* the configuration provider used to acquire scoped routing + * configuration for each request flow. Pointer ownership is _not_ transferred to the caller of + * this function. + */ + virtual Config::ConfigProvider* scopedRouteConfigProvider() PURE; /** * @return const std::string& the server name to write into responses. diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index c941694efb99d..a7fe67d3316fa 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -366,11 +366,17 @@ void ConnectionManagerImpl::chargeTracingStats(const Tracing::Reason& tracing_re ConnectionManagerImpl::ActiveStream::ActiveStream(ConnectionManagerImpl& connection_manager) : connection_manager_(connection_manager), - snapped_route_config_(connection_manager.config_.routeConfigProvider().config()), stream_id_(connection_manager.random_generator_.random()), request_response_timespan_(new Stats::Timespan( connection_manager_.stats_.named_.downstream_rq_time_, connection_manager_.timeSystem())), stream_info_(connection_manager_.codec_->protocol(), connection_manager_.timeSystem()) { + if (connection_manager.config_.routeConfigProvider() != nullptr) { + snapped_route_config_ = connection_manager.config_.routeConfigProvider()->config(); + } + if (connection_manager.config_.scopedRouteConfigProvider() != nullptr) { + snapped_scoped_route_config_ = + connection_manager_.config_.scopedRouteConfigProvider()->config(); + } connection_manager_.stats_.named_.downstream_rq_total_.inc(); connection_manager_.stats_.named_.downstream_rq_active_.inc(); if (connection_manager_.codec_->protocol() == Protocol::Http2) { @@ -1047,22 +1053,23 @@ void ConnectionManagerImpl::ActiveStream::sendLocalReply( if (!state_.created_filter_chain_) { createFilterChain(); } - Utility::sendLocalReply(is_grpc_request, - [this, modify_headers](HeaderMapPtr&& headers, bool end_stream) -> void { - if (modify_headers != nullptr) { - modify_headers(*headers); - } - response_headers_ = std::move(headers); - // TODO: Start encoding from the last decoder filter that saw the - // request instead. - encodeHeaders(nullptr, *response_headers_, end_stream); - }, - [this](Buffer::Instance& data, bool end_stream) -> void { - // TODO: Start encoding from the last decoder filter that saw the - // request instead. - encodeData(nullptr, data, end_stream); - }, - state_.destroyed_, code, body, grpc_status, is_head_request); + Utility::sendLocalReply( + is_grpc_request, + [this, modify_headers](HeaderMapPtr&& headers, bool end_stream) -> void { + if (modify_headers != nullptr) { + modify_headers(*headers); + } + response_headers_ = std::move(headers); + // TODO: Start encoding from the last decoder filter that saw the + // request instead. + encodeHeaders(nullptr, *response_headers_, end_stream); + }, + [this](Buffer::Instance& data, bool end_stream) -> void { + // TODO: Start encoding from the last decoder filter that saw the + // request instead. + encodeData(nullptr, data, end_stream); + }, + state_.destroyed_, code, body, grpc_status, is_head_request); } void ConnectionManagerImpl::ActiveStream::encode100ContinueHeaders( diff --git a/source/common/http/conn_manager_impl.h b/source/common/http/conn_manager_impl.h index 982817691beeb..bd53a9c94c426 100644 --- a/source/common/http/conn_manager_impl.h +++ b/source/common/http/conn_manager_impl.h @@ -18,6 +18,7 @@ #include "envoy/network/drain_decision.h" #include "envoy/network/filter.h" #include "envoy/router/rds.h" +#include "envoy/router/scopes.h" #include "envoy/runtime/runtime.h" #include "envoy/server/overload_manager.h" #include "envoy/ssl/connection.h" @@ -399,6 +400,7 @@ class ConnectionManagerImpl : Logger::Loggable, ConnectionManagerImpl& connection_manager_; Router::ConfigConstSharedPtr snapped_route_config_; + Router::ScopedConfigConstSharedPtr snapped_scoped_route_config_; Tracing::SpanPtr active_span_; const uint64_t stream_id_; StreamEncoder* response_encoder_{}; diff --git a/source/common/router/BUILD b/source/common/router/BUILD index b861e6eeb3037..f05d402f76496 100644 --- a/source/common/router/BUILD +++ b/source/common/router/BUILD @@ -93,6 +93,34 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "scoped_config_lib", + srcs = ["scoped_config_impl.cc"], + hdrs = ["scoped_config_impl.h"], + deps = [ + ":config_lib", + "//include/envoy/router:scopes_interface", + "@envoy_api//envoy/api/v2:srds_cc", + ], +) + +envoy_cc_library( + name = "scoped_rds_lib", + srcs = ["scoped_rds.cc"], + hdrs = ["scoped_rds.h"], + deps = [ + ":scoped_config_lib", + "//include/envoy/config:subscription_interface", + "//include/envoy/stats:stats_interface", + "//source/common/common:assert_lib", + "//source/common/common:minimal_logger_lib", + "//source/common/config:config_provider_lib", + "//source/common/config:subscription_factory_lib", + "@envoy_api//envoy/admin/v2alpha:config_dump_cc", + "@envoy_api//envoy/api/v2:srds_cc", + ], +) + envoy_cc_library( name = "retry_state_lib", srcs = ["retry_state_impl.cc"], diff --git a/source/common/router/scoped_config_impl.cc b/source/common/router/scoped_config_impl.cc new file mode 100644 index 0000000000000..3b1dda6160da3 --- /dev/null +++ b/source/common/router/scoped_config_impl.cc @@ -0,0 +1,13 @@ +#include "common/router/scoped_config_impl.h" + +#include "common/router/config_impl.h" + +namespace Envoy { +namespace Router { + +Router::ConfigConstSharedPtr ScopedConfigImpl::getRouterConfig(const Http::HeaderMap&) const { + return std::make_shared(); +} + +} // namespace Router +} // namespace Envoy diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h new file mode 100644 index 0000000000000..46ad8470b4288 --- /dev/null +++ b/source/common/router/scoped_config_impl.h @@ -0,0 +1,47 @@ +#pragma once + +#include "envoy/api/v2/srds.pb.h" +#include "envoy/router/router.h" +#include "envoy/router/scopes.h" + +#include "common/router/config_impl.h" + +namespace Envoy { +namespace Router { + +/** + * The implementation of scoped routing configuration logic. + * NOTE: This is not yet implemented and simply mimics the behavior of the + * NullScopedConfigImpl. + * + * TODO(AndresGuedez): implement scoped routing logic. + */ +class ScopedConfigImpl : public ScopedConfig { +public: + ScopedConfigImpl(const envoy::api::v2::ScopedRouteConfigurationsSet& config_proto) + : name_(config_proto.name()) {} + + ~ScopedConfigImpl() override = default; + + Router::ConfigConstSharedPtr getRouterConfig(const Http::HeaderMap& headers) const override; + +private: + const std::string name_; +}; + +/** + * An empty implementation of the scoped routing configuration. + */ +class NullScopedConfigImpl : public ScopedConfig { +public: + NullScopedConfigImpl() = default; + + ~NullScopedConfigImpl() override = default; + + Router::ConfigConstSharedPtr getRouterConfig(const Http::HeaderMap&) const override { + return std::make_shared(); + } +}; + +} // namespace Router +} // namespace Envoy diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc new file mode 100644 index 0000000000000..e2b9ba991cfee --- /dev/null +++ b/source/common/router/scoped_rds.cc @@ -0,0 +1,191 @@ +#include "common/router/scoped_rds.h" + +#include + +#include "envoy/admin/v2alpha/config_dump.pb.h" +#include "envoy/api/v2/srds.pb.validate.h" + +#include "common/common/assert.h" +#include "common/config/subscription_factory.h" +#include "common/router/scoped_config_impl.h" + +namespace Envoy { +namespace Router { + +Envoy::Config::ConfigProviderPtr ScopedRoutesConfigProviderUtil::maybeCreate( + const envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& + config, + Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, + Envoy::Config::ConfigProviderManager& scoped_routes_config_provider_manager) { + switch (config.scoped_routes_specifier_case()) { + case envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager:: + kScopedRoutesConfig: + return scoped_routes_config_provider_manager.createStaticConfigProvider( + config.scoped_routes_config(), factory_context, + ScopedRoutesConfigProviderManagerOptArg(config.rds())); + + case envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager:: + kScopedRds: + return scoped_routes_config_provider_manager.createXdsConfigProvider( + config.scoped_rds(), factory_context, stat_prefix, + ScopedRoutesConfigProviderManagerOptArg(config.rds())); + + case envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager:: + SCOPED_ROUTES_SPECIFIER_NOT_SET: + return nullptr; + + default: + NOT_REACHED_GCOVR_EXCL_LINE; + } +} + +InlineScopedRoutesConfigProvider::InlineScopedRoutesConfigProvider( + const envoy::api::v2::ScopedRouteConfigurationsSet& config_proto, + Server::Configuration::FactoryContext& factory_context, + ScopedRoutesConfigProviderManager& config_provider_manager, + const envoy::config::filter::network::http_connection_manager::v2::Rds& rds) + : Envoy::Config::ImmutableConfigProviderImplBase( + factory_context, config_provider_manager, + Envoy::Config::ConfigProviderInstanceType::Inline), + config_(std::make_shared()), config_proto_(config_proto), rds_(rds) {} + +ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( + const envoy::config::filter::network::http_connection_manager::v2::ScopedRds& scoped_rds, + const std::string& manager_identifier, Server::Configuration::FactoryContext& factory_context, + const std::string& stat_prefix, ScopedRoutesConfigProviderManager& config_provider_manager) + : ConfigSubscriptionInstanceBase( + "SRDS", manager_identifier, config_provider_manager, factory_context.timeSource(), + factory_context.timeSource().systemTime(), factory_context.localInfo()), + scoped_routes_config_name_(scoped_rds.scoped_routes_config_set_name()), + scope_(factory_context.scope().createScope(stat_prefix + "scoped_rds." + + scoped_routes_config_name_ + ".")), + stats_({ALL_SCOPED_RDS_STATS(POOL_COUNTER(*scope_))}) { + subscription_ = Envoy::Config::SubscriptionFactory::subscriptionFromConfigSource< + envoy::api::v2::ScopedRouteConfigurationsSet>( + scoped_rds.config_source(), factory_context.localInfo(), factory_context.dispatcher(), + factory_context.clusterManager(), factory_context.random(), *scope_, + "envoy.api.v2.ScopedRoutesDiscoveryService.FetchScopedRoutes", + "envoy.api.v2.ScopedRoutesDiscoveryService.StreamScopedRoutes", factory_context.api()); +} + +void ScopedRdsConfigSubscription::onConfigUpdate(const ResourceVector& resources, + const std::string& version_info) { + if (resources.empty()) { + ENVOY_LOG(debug, "Missing ScopedRouteConfigurationsSet for {} in onConfigUpdate()", + scoped_routes_config_name_); + stats_.update_empty_.inc(); + ConfigSubscriptionInstanceBase::onConfigUpdateFailed(); + return; + } + + if (resources.size() != 1) { + throw EnvoyException(fmt::format("Unexpected SRDS resource length: {}", resources.size())); + } + + const auto& scoped_routes_config = resources[0]; + MessageUtil::validate(scoped_routes_config); + // The name we receive in the config must match the name requested. + if (!(scoped_routes_config.name() == scoped_routes_config_name_)) { + throw EnvoyException(fmt::format("Unexpected SRDS configuration (expecting {}): {}", + scoped_routes_config_name_, scoped_routes_config.name())); + } + + if (checkAndApplyConfig(scoped_routes_config, scoped_routes_config_name_, version_info)) { + scoped_routes_proto_ = scoped_routes_config; + stats_.config_reload_.inc(); + } + + ConfigSubscriptionInstanceBase::onConfigUpdate(); +} + +ScopedRdsConfigProvider::ScopedRdsConfigProvider( + ScopedRdsConfigSubscriptionSharedPtr&& subscription, + Envoy::Config::ConfigProvider::ConfigConstSharedPtr initial_config, + Server::Configuration::FactoryContext& factory_context, + const envoy::config::filter::network::http_connection_manager::v2::Rds& rds) + : MutableConfigProviderImplBase(std::move(subscription), factory_context), + subscription_(static_cast( + MutableConfigProviderImplBase::subscription().get())), + rds_(rds) { + initialize(initial_config); +} + +Envoy::Config::ConfigProvider::ConfigConstSharedPtr +ScopedRdsConfigProvider::onConfigProtoUpdate(const Protobuf::Message& config_proto) { + return std::make_shared( + static_cast(config_proto)); +} + +ProtobufTypes::MessagePtr ScopedRoutesConfigProviderManager::dumpConfigs() const { + auto config_dump = std::make_unique(); + for (const auto& element : configSubscriptions()) { + auto subscription = element.second.lock(); + ASSERT(subscription); + + if (subscription->configInfo()) { + auto* dynamic_config = config_dump->mutable_dynamic_scoped_routes_configs()->Add(); + dynamic_config->set_version_info(subscription->configInfo().value().last_config_version_); + dynamic_config->mutable_scoped_routes_config()->MergeFrom( + static_cast(subscription.get())->configProto().value()); + TimestampUtil::systemClockToTimestamp(subscription->lastUpdated(), + *dynamic_config->mutable_last_updated()); + } + } + + for (const auto& provider : + immutableConfigProviders(Envoy::Config::ConfigProviderInstanceType::Inline)) { + ASSERT(provider->configProtoInfo()); + auto* inline_config = config_dump->mutable_inline_scoped_routes_configs()->Add(); + inline_config->mutable_scoped_routes_config()->MergeFrom( + provider->configProtoInfo() + .value() + .config_proto_); + TimestampUtil::systemClockToTimestamp(provider->lastUpdated(), + *inline_config->mutable_last_updated()); + } + + return config_dump; +} + +Envoy::Config::ConfigProviderPtr ScopedRoutesConfigProviderManager::createXdsConfigProvider( + const Protobuf::Message& config_source_proto, + Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, + const Envoy::Config::ConfigProviderManager::OptionalArg& optarg) { + ScopedRdsConfigSubscriptionSharedPtr subscription = + ConfigProviderManagerImplBase::getSubscription( + config_source_proto, factory_context.initManager(), + [&config_source_proto, &factory_context, + &stat_prefix](const std::string& manager_identifier, + ConfigProviderManagerImplBase& config_provider_manager) + -> Envoy::Config::ConfigSubscriptionInstanceBaseSharedPtr { + const auto& scoped_rds_config_source = dynamic_cast< + const envoy::config::filter::network::http_connection_manager::v2::ScopedRds&>( + config_source_proto); + return std::make_shared( + scoped_rds_config_source, manager_identifier, factory_context, stat_prefix, + static_cast(config_provider_manager)); + }); + + Envoy::Config::ConfigProvider::ConfigConstSharedPtr initial_config; + const Envoy::Config::MutableConfigProviderImplBase* first_provider = + subscription->getAnyBoundMutableConfigProvider(); + if (first_provider != nullptr) { + initial_config = first_provider->getConfig(); + } + const auto& typed_optarg = static_cast(optarg); + return std::make_unique(std::move(subscription), initial_config, + factory_context, typed_optarg.rds_); +} + +Envoy::Config::ConfigProviderPtr ScopedRoutesConfigProviderManager::createStaticConfigProvider( + const Protobuf::Message& config_proto, Server::Configuration::FactoryContext& factory_context, + const Envoy::Config::ConfigProviderManager::OptionalArg& optarg) { + const auto& scoped_routes_proto = + dynamic_cast(config_proto); + const auto& typed_optarg = static_cast(optarg); + return absl::make_unique(scoped_routes_proto, factory_context, + *this, typed_optarg.rds_); +} + +} // namespace Router +} // namespace Envoy diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h new file mode 100644 index 0000000000000..eef5f030ca5fd --- /dev/null +++ b/source/common/router/scoped_rds.h @@ -0,0 +1,177 @@ +#pragma once + +#include + +#include "envoy/api/v2/srds.pb.h" +#include "envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.pb.h" +#include "envoy/config/subscription.h" +#include "envoy/stats/scope.h" + +#include "common/common/logger.h" +#include "common/config/config_provider_impl.h" + +namespace Envoy { +namespace Router { + +// Scoped routing configuration utilities. +class ScopedRoutesConfigProviderUtil { +public: + // If enabled in the HttpConnectionManager config, returns a ConfigProvider for scoped routing + // configuration. + static Envoy::Config::ConfigProviderPtr maybeCreate( + const envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& + config, + Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, + Envoy::Config::ConfigProviderManager& scoped_routes_config_provider_manager); +}; + +class ScopedRoutesConfigProviderManager; + +// A ConfigProvider for inline scoped routing configuration. +class InlineScopedRoutesConfigProvider : public Envoy::Config::ImmutableConfigProviderImplBase { +public: + InlineScopedRoutesConfigProvider( + const envoy::api::v2::ScopedRouteConfigurationsSet& config_proto, + Server::Configuration::FactoryContext& factory_context, + ScopedRoutesConfigProviderManager& config_provider_manager, + const envoy::config::filter::network::http_connection_manager::v2::Rds& rds); + + ~InlineScopedRoutesConfigProvider() override = default; + + // Envoy::Config::ConfigProvider + const Protobuf::Message* getConfigProto() const override { return &config_proto_; } + std::string getConfigVersion() const override { return ""; } + ConfigConstSharedPtr getConfig() const override { return config_; } + +private: + ConfigConstSharedPtr config_; + const envoy::api::v2::ScopedRouteConfigurationsSet config_proto_; + const envoy::config::filter::network::http_connection_manager::v2::Rds rds_; +}; + +/** + * All SRDS stats. @see stats_macros.h + */ +// clang-format off +#define ALL_SCOPED_RDS_STATS(COUNTER) \ + COUNTER(config_reload) \ + COUNTER(update_empty) + +// clang-format on + +struct ScopedRdsStats { + ALL_SCOPED_RDS_STATS(GENERATE_COUNTER_STRUCT) +}; + +// A scoped RDS subscription to be used with the dynamic scoped RDS ConfigProvider. +class ScopedRdsConfigSubscription + : public Envoy::Config::ConfigSubscriptionInstanceBase, + Envoy::Config::SubscriptionCallbacks { +public: + ScopedRdsConfigSubscription( + const envoy::config::filter::network::http_connection_manager::v2::ScopedRds& scoped_rds, + const std::string& manager_identifier, Server::Configuration::FactoryContext& factory_context, + const std::string& stat_prefix, ScopedRoutesConfigProviderManager& config_provider_manager); + + ~ScopedRdsConfigSubscription() override = default; + + // Envoy::Config::ConfigSubscriptionInstanceBase + void start() override { subscription_->start({scoped_routes_config_name_}, *this); } + + // Envoy::Config::SubscriptionCallbacks + void onConfigUpdate(const ResourceVector& resources, const std::string& version_info) override; + void onConfigUpdateFailed(const EnvoyException*) override { + ConfigSubscriptionInstanceBase::onConfigUpdateFailed(); + } + std::string resourceName(const ProtobufWkt::Any& resource) override { + return MessageUtil::anyConvert(resource).name(); + } + + const absl::optional& configProto() const { + return scoped_routes_proto_; + } + +private: + std::unique_ptr> + subscription_; + const std::string scoped_routes_config_name_; + Stats::ScopePtr scope_; + ScopedRdsStats stats_; + absl::optional scoped_routes_proto_; +}; + +using ScopedRdsConfigSubscriptionSharedPtr = std::shared_ptr; + +// A ConfigProvider for scoped RDS that dynamically fetches scoped routing configuration via a +// subscription. +class ScopedRdsConfigProvider : public Envoy::Config::MutableConfigProviderImplBase { +public: + ScopedRdsConfigProvider( + ScopedRdsConfigSubscriptionSharedPtr&& subscription, ConfigConstSharedPtr initial_config, + Server::Configuration::FactoryContext& factory_context, + const envoy::config::filter::network::http_connection_manager::v2::Rds& rds); + + ~ScopedRdsConfigProvider() override = default; + + ScopedRdsConfigSubscription& subscription() { return *subscription_; } + + // Envoy::Config::MutableConfigProviderImplBase + Envoy::Config::ConfigProvider::ConfigConstSharedPtr + onConfigProtoUpdate(const Protobuf::Message& config) override; + + // Envoy::Config::ConfigProvider + const Protobuf::Message* getConfigProto() const override { + if (!subscription_->configProto().has_value()) { + return nullptr; + } + return &subscription_->configProto().value(); + } + std::string getConfigVersion() const override { + if (subscription_->configInfo().has_value()) { + return subscription_->configInfo().value().last_config_version_; + } + + return ""; + } + +private: + ScopedRdsConfigSubscription* subscription_; + const envoy::config::filter::network::http_connection_manager::v2::Rds rds_; +}; + +// A ConfigProviderManager for scoped routing configuration that creates static/inline and dynamic +// (xds) config providers. +class ScopedRoutesConfigProviderManager : public Envoy::Config::ConfigProviderManagerImplBase { +public: + ScopedRoutesConfigProviderManager(Server::Admin& admin) + : Envoy::Config::ConfigProviderManagerImplBase(admin, "route_scopes") {} + + ~ScopedRoutesConfigProviderManager() override = default; + + // Envoy::Config::ConfigProviderManagerImplBase + ProtobufTypes::MessagePtr dumpConfigs() const override; + Envoy::Config::ConfigProviderPtr + createXdsConfigProvider(const Protobuf::Message& config_source_proto, + Server::Configuration::FactoryContext& factory_context, + const std::string& stat_prefix, + const Envoy::Config::ConfigProviderManager::OptionalArg& optarg) override; + Envoy::Config::ConfigProviderPtr createStaticConfigProvider( + const Protobuf::Message& config_proto, Server::Configuration::FactoryContext& factory_context, + const Envoy::Config::ConfigProviderManager::OptionalArg& optarg) override; +}; + +// The optional argument passed to the ConfigProviderManager::create*() functions. +class ScopedRoutesConfigProviderManagerOptArg + : public Envoy::Config::ConfigProviderManager::OptionalArg { +public: + ScopedRoutesConfigProviderManagerOptArg( + const envoy::config::filter::network::http_connection_manager::v2::Rds& rds) + : rds_(rds) {} + + ~ScopedRoutesConfigProviderManagerOptArg() override = default; + + const envoy::config::filter::network::http_connection_manager::v2::Rds& rds_; +}; + +} // namespace Router +} // namespace Envoy diff --git a/source/extensions/filters/network/http_connection_manager/BUILD b/source/extensions/filters/network/http_connection_manager/BUILD index 29f6420e60845..1f5474a651e2f 100644 --- a/source/extensions/filters/network/http_connection_manager/BUILD +++ b/source/extensions/filters/network/http_connection_manager/BUILD @@ -17,6 +17,7 @@ envoy_cc_library( srcs = ["config.cc"], hdrs = ["config.h"], deps = [ + "//include/envoy/config:config_provider_manager_interface", "//include/envoy/filesystem:filesystem_interface", "//include/envoy/http:filter_interface", "//include/envoy/registry", @@ -36,6 +37,7 @@ envoy_cc_library( "//source/common/json:json_loader_lib", "//source/common/protobuf:utility_lib", "//source/common/router:rds_lib", + "//source/common/router:scoped_rds_lib", "//source/extensions/filters/network:well_known_names", "//source/extensions/filters/network/common:factory_base_lib", ], diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index 51eb39e237303..193becfa8da67 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -23,6 +23,7 @@ #include "common/json/config_schemas.h" #include "common/protobuf/utility.h" #include "common/router/rds_impl.h" +#include "common/router/scoped_rds.h" namespace Envoy { namespace Extensions { @@ -64,11 +65,48 @@ std::unique_ptr createInternalAddressConfig( return std::make_unique(); } +// Validates that HttpConnectionManager configuration correctly specifies routing configuration. +void validateScopedRoutingAndRds( + const envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& + config) { + // NOTE: This validation can not be done via proto validators due to the conditionals involved. + if (config.scoped_routes_specifier_case() != + envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager:: + SCOPED_ROUTES_SPECIFIER_NOT_SET) { + // When scoped routing is enabled, RDS _must_ be used and subscriptions are dynamically + // generated based on the SRDS config. + if (config.route_specifier_case() != + envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager::kRds) { + throw EnvoyException(fmt::format("Error: RDS must be used when scoped routing is enabled")); + } + + if (config.rds().subscription_specifier_case() != + envoy::config::filter::network::http_connection_manager::v2::Rds::kScopedRdsTemplate || + config.rds().scoped_rds_template() != true) { + throw EnvoyException(fmt::format( + "Error: the RDS subscription specifier must be set to scoped_rds_template=true " + "when scoped routing is enabled")); + } + } else { + // Scoped routing is not enabled, ensure a route configuration name is specified if RDS is + // used. + if (config.route_specifier_case() == + envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager::kRds) { + if (config.rds().subscription_specifier_case() != + envoy::config::filter::network::http_connection_manager::v2::Rds::kRouteConfigName) { + throw EnvoyException(fmt::format( + "Error: RDS must specify a route_config_name when scoped routing is not enabled")); + } + } + } +} + } // namespace // Singleton registration via macro defined in envoy/singleton/manager.h SINGLETON_MANAGER_REGISTRATION(date_provider); SINGLETON_MANAGER_REGISTRATION(route_config_provider_manager); +SINGLETON_MANAGER_REGISTRATION(scoped_routes_config_provider_manager); Network::FilterFactoryCb HttpConnectionManagerFilterConfigFactory::createFilterFactoryFromProtoTyped( @@ -88,14 +126,21 @@ HttpConnectionManagerFilterConfigFactory::createFilterFactoryFromProtoTyped( return std::make_shared(context.admin()); }); + std::shared_ptr scoped_routes_config_provider_manager = + context.singletonManager().getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(scoped_routes_config_provider_manager), [&context] { + return std::make_shared(context.admin()); + }); + std::shared_ptr filter_config(new HttpConnectionManagerConfig( - proto_config, context, *date_provider, *route_config_provider_manager)); + proto_config, context, *date_provider, *route_config_provider_manager, + *scoped_routes_config_provider_manager)); // This lambda captures the shared_ptrs created above, thus preserving the // reference count. Moreover, keep in mind the capture list determines // destruction order. - return [route_config_provider_manager, filter_config, &context, - date_provider](Network::FilterManager& filter_manager) -> void { + return [route_config_provider_manager, scoped_routes_config_provider_manager, filter_config, + &context, date_provider](Network::FilterManager& filter_manager) -> void { filter_manager.addReadFilter(Network::ReadFilterSharedPtr{new Http::ConnectionManagerImpl( *filter_config, context.drainDecision(), context.random(), context.httpContext(), context.runtime(), context.localInfo(), context.clusterManager(), @@ -126,7 +171,8 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( const envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& config, Server::Configuration::FactoryContext& context, Http::DateProvider& date_provider, - Router::RouteConfigProviderManager& route_config_provider_manager) + Router::RouteConfigProviderManager& route_config_provider_manager, + Config::ConfigProviderManager& scoped_routes_config_provider_manager) : context_(context), reverse_encode_order_(PROTOBUF_GET_WRAPPED_OR_DEFAULT( config, bugfix_reverse_encode_order, true)), stats_prefix_(fmt::format("http.{}.", config.stat_prefix())), @@ -138,6 +184,7 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( xff_num_trusted_hops_(config.xff_num_trusted_hops()), skip_xff_append_(config.skip_xff_append()), via_(config.via()), route_config_provider_manager_(route_config_provider_manager), + scoped_routes_config_provider_manager_(scoped_routes_config_provider_manager), http2_settings_(Http::Utility::parseHttp2Settings(config.http2_protocol_options())), http1_settings_(Http::Utility::parseHttp1Settings(config.http_protocol_options())), max_request_headers_size_kb_(PROTOBUF_GET_WRAPPED_OR_DEFAULT( @@ -153,9 +200,19 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( context_.listenerScope())), proxy_100_continue_(config.proxy_100_continue()), delayed_close_timeout_(PROTOBUF_GET_MS_OR_DEFAULT(config, delayed_close_timeout, 1000)) { + // Throws an exception on failure. + validateScopedRoutingAndRds(config); + + // If soped RDS is enabled, avoid creating a route config provider. Route config providers will be + // managed by the scoped routing logic instead. + if (config.rds().subscription_specifier_case() != + envoy::config::filter::network::http_connection_manager::v2::Rds::kScopedRdsTemplate) { + route_config_provider_ = Router::RouteConfigProviderUtil::create( + config, context_, stats_prefix_, route_config_provider_manager_); + } - route_config_provider_ = Router::RouteConfigProviderUtil::create(config, context_, stats_prefix_, - route_config_provider_manager_); + scoped_routes_config_provider_ = Router::ScopedRoutesConfigProviderUtil::maybeCreate( + config, context_, stats_prefix_, scoped_routes_config_provider_manager_); switch (config.forward_client_cert_details()) { case envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager::SANITIZE: diff --git a/source/extensions/filters/network/http_connection_manager/config.h b/source/extensions/filters/network/http_connection_manager/config.h index 05c3c982241df..be7d77c566a0e 100644 --- a/source/extensions/filters/network/http_connection_manager/config.h +++ b/source/extensions/filters/network/http_connection_manager/config.h @@ -7,6 +7,7 @@ #include #include +#include "envoy/config/config_provider_manager.h" #include "envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.pb.validate.h" #include "envoy/http/filter.h" #include "envoy/router/route_config_provider_manager.h" @@ -78,7 +79,8 @@ class HttpConnectionManagerConfig : Logger::Loggable, const envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& config, Server::Configuration::FactoryContext& context, Http::DateProvider& date_provider, - Router::RouteConfigProviderManager& route_config_provider_manager); + Router::RouteConfigProviderManager& route_config_provider_manager, + Config::ConfigProviderManager& scoped_routes_config_provider_manager); // Http::FilterChainFactory void createFilterChain(Http::FilterChainFactoryCallbacks& callbacks) override; @@ -105,7 +107,12 @@ class HttpConnectionManagerConfig : Logger::Loggable, absl::optional idleTimeout() const override { return idle_timeout_; } std::chrono::milliseconds streamIdleTimeout() const override { return stream_idle_timeout_; } std::chrono::milliseconds requestTimeout() const override { return request_timeout_; } - Router::RouteConfigProvider& routeConfigProvider() override { return *route_config_provider_; } + Router::RouteConfigProvider* routeConfigProvider() override { + return route_config_provider_.get(); + } + Config::ConfigProvider* scopedRouteConfigProvider() override { + return scoped_routes_config_provider_.get(); + } const std::string& serverName() override { return server_name_; } Http::ConnectionManagerStats& stats() override { return stats_; } Http::ConnectionManagerTracingStats& tracingStats() override { return tracing_stats_; } @@ -152,6 +159,7 @@ class HttpConnectionManagerConfig : Logger::Loggable, Http::ForwardClientCertType forward_client_cert_; std::vector set_current_client_cert_details_; Router::RouteConfigProviderManager& route_config_provider_manager_; + Config::ConfigProviderManager& scoped_routes_config_provider_manager_; CodecType codec_type_; const Http::Http2Settings http2_settings_; const Http::Http1Settings http1_settings_; @@ -163,6 +171,7 @@ class HttpConnectionManagerConfig : Logger::Loggable, std::chrono::milliseconds stream_idle_timeout_; std::chrono::milliseconds request_timeout_; Router::RouteConfigProviderPtr route_config_provider_; + Config::ConfigProviderPtr scoped_routes_config_provider_; std::chrono::milliseconds drain_timeout_; bool generate_request_id_; Http::DateProvider& date_provider_; diff --git a/source/server/http/BUILD b/source/server/http/BUILD index b6761f2865857..65b8bc56688cd 100644 --- a/source/server/http/BUILD +++ b/source/server/http/BUILD @@ -53,6 +53,7 @@ envoy_cc_library( "//source/common/network:utility_lib", "//source/common/profiler:profiler_lib", "//source/common/router:config_lib", + "//source/common/router:scoped_config_lib", "//source/common/stats:histogram_lib", "//source/common/stats:isolated_store_lib", "//source/common/stats:stats_lib", diff --git a/source/server/http/admin.cc b/source/server/http/admin.cc index c16b7ae5396ba..224781e378dbf 100644 --- a/source/server/http/admin.cc +++ b/source/server/http/admin.cc @@ -1060,6 +1060,7 @@ AdminImpl::AdminImpl(const std::string& profile_path, Server::Instance& server) tracing_stats_( Http::ConnectionManagerImpl::generateTracingStats("http.admin.", no_op_store_)), route_config_provider_(server.timeSystem()), + scoped_route_config_provider_(server.timeSystem()), // TODO(jsedgwick) add /runtime_reset endpoint that removes all admin-set values handlers_{ {"/", "Admin home page", MAKE_ADMIN_HANDLER(handlerAdminHome), false, false}, diff --git a/source/server/http/admin.h b/source/server/http/admin.h index 05b3c1fa840c5..446fa2b63dc43 100644 --- a/source/server/http/admin.h +++ b/source/server/http/admin.h @@ -27,6 +27,7 @@ #include "common/http/default_server_string.h" #include "common/http/utility.h" #include "common/network/raw_buffer_socket.h" +#include "common/router/scoped_config_impl.h" #include "common/stats/isolated_store_impl.h" #include "server/http/config_tracker_impl.h" @@ -104,7 +105,10 @@ class AdminImpl : public Admin, std::chrono::milliseconds streamIdleTimeout() const override { return {}; } std::chrono::milliseconds requestTimeout() const override { return {}; } std::chrono::milliseconds delayedCloseTimeout() const override { return {}; } - Router::RouteConfigProvider& routeConfigProvider() override { return route_config_provider_; } + Router::RouteConfigProvider* routeConfigProvider() override { return &route_config_provider_; } + Config::ConfigProvider* scopedRouteConfigProvider() override { + return &scoped_route_config_provider_; + } const std::string& serverName() override { return Http::DefaultServerString::get(); } Http::ConnectionManagerStats& stats() override { return stats_; } Http::ConnectionManagerTracingStats& tracingStats() override { return tracing_stats_; } @@ -159,6 +163,26 @@ class AdminImpl : public Admin, TimeSource& time_source_; }; + /** + * Implementation of ScopedRouteConfigProvider that returns a null scoped route config. + */ + struct NullScopedRouteConfigProvider : public Config::ConfigProvider { + NullScopedRouteConfigProvider(TimeSource& time_source) + : config_(std::make_shared()), + time_source_(time_source) {} + + ~NullScopedRouteConfigProvider() override = default; + + // Config::ConfigProvider + SystemTime lastUpdated() const override { return time_source_.systemTime(); } + const Protobuf::Message* getConfigProto() const override { return nullptr; } + std::string getConfigVersion() const override { return ""; } + ConfigConstSharedPtr getConfig() const override { return config_; } + + Router::ScopedConfigConstSharedPtr config_; + TimeSource& time_source_; + }; + friend class AdminStatsTest; /** @@ -304,6 +328,7 @@ class AdminImpl : public Admin, Stats::IsolatedStoreImpl no_op_store_; Http::ConnectionManagerTracingStats tracing_stats_; NullRouteConfigProvider route_config_provider_; + NullScopedRouteConfigProvider scoped_route_config_provider_; std::list handlers_; uint32_t max_request_headers_size_kb_{Http::DEFAULT_MAX_REQUEST_HEADERS_SIZE_KB}; absl::optional idle_timeout_; diff --git a/test/common/config/config_provider_impl_test.cc b/test/common/config/config_provider_impl_test.cc index 1f355769cda58..125f5bdc11348 100644 --- a/test/common/config/config_provider_impl_test.cc +++ b/test/common/config/config_provider_impl_test.cc @@ -157,9 +157,11 @@ class DummyConfigProviderManager : public ConfigProviderManagerImplBase { } // Envoy::Config::ConfigProviderManager - ConfigProviderPtr createXdsConfigProvider(const Protobuf::Message& config_source_proto, - Server::Configuration::FactoryContext& factory_context, - const std::string&) override { + ConfigProviderPtr + createXdsConfigProvider(const Protobuf::Message& config_source_proto, + Server::Configuration::FactoryContext& factory_context, + const std::string&, + const Envoy::Config::ConfigProviderManager::OptionalArg&) override { DummyConfigSubscriptionSharedPtr subscription = getSubscription( config_source_proto, factory_context.initManager(), [&factory_context](const std::string& manager_identifier, @@ -183,7 +185,8 @@ class DummyConfigProviderManager : public ConfigProviderManagerImplBase { // Envoy::Config::ConfigProviderManager ConfigProviderPtr createStaticConfigProvider(const Protobuf::Message& config_proto, - Server::Configuration::FactoryContext& factory_context) override { + Server::Configuration::FactoryContext& factory_context, + const Envoy::Config::ConfigProviderManager::OptionalArg&) override { return std::make_unique( dynamic_cast(config_proto), factory_context, *this); @@ -235,7 +238,8 @@ TEST_F(ConfigProviderImplTest, SharedOwnership) { envoy::api::v2::core::ApiConfigSource config_source_proto; config_source_proto.set_api_type(envoy::api::v2::core::ApiConfigSource::GRPC); ConfigProviderPtr provider1 = provider_manager_->createXdsConfigProvider( - config_source_proto, factory_context_, "dummy_prefix"); + config_source_proto, factory_context_, "dummy_prefix", + ConfigProviderManager::NullOptionalArg()); // No config protos have been received via the subscription yet. EXPECT_FALSE(provider1->configProtoInfo().has_value()); @@ -250,7 +254,8 @@ TEST_F(ConfigProviderImplTest, SharedOwnership) { // Check that a newly created provider with the same config source will share // the subscription, config proto and resulting ConfigProvider::Config. ConfigProviderPtr provider2 = provider_manager_->createXdsConfigProvider( - config_source_proto, factory_context_, "dummy_prefix"); + config_source_proto, factory_context_, "dummy_prefix", + ConfigProviderManager::NullOptionalArg()); EXPECT_TRUE(provider2->configProtoInfo().has_value()); EXPECT_EQ(&dynamic_cast(*provider1).subscription(), @@ -263,7 +268,8 @@ TEST_F(ConfigProviderImplTest, SharedOwnership) { // Change the config source and verify that a new subscription is used. config_source_proto.set_api_type(envoy::api::v2::core::ApiConfigSource::REST); ConfigProviderPtr provider3 = provider_manager_->createXdsConfigProvider( - config_source_proto, factory_context_, "dummy_prefix"); + config_source_proto, factory_context_, "dummy_prefix", + ConfigProviderManager::NullOptionalArg()); EXPECT_NE(&dynamic_cast(*provider1).subscription(), &dynamic_cast(*provider3).subscription()); @@ -321,7 +327,8 @@ TEST_F(ConfigProviderImplTest, ConfigDump) { timeSystem().setSystemTime(std::chrono::milliseconds(1234567891234)); ConfigProviderPtr static_config = provider_manager_->createStaticConfigProvider( - parseDummyConfigFromYaml(config_yaml), factory_context_); + parseDummyConfigFromYaml(config_yaml), factory_context_, + ConfigProviderManager::NullOptionalArg()); message_ptr = factory_context_.admin_.config_tracker_.config_tracker_callbacks_["dummy"](); const auto& dummy_config_dump2 = static_cast(*message_ptr); @@ -337,7 +344,8 @@ TEST_F(ConfigProviderImplTest, ConfigDump) { envoy::api::v2::core::ApiConfigSource config_source_proto; config_source_proto.set_api_type(envoy::api::v2::core::ApiConfigSource::GRPC); ConfigProviderPtr dynamic_provider = provider_manager_->createXdsConfigProvider( - config_source_proto, factory_context_, "dummy_prefix"); + config_source_proto, factory_context_, "dummy_prefix", + ConfigProviderManager::NullOptionalArg()); // Static + dynamic config dump. Protobuf::RepeatedPtrField dummy_configs; @@ -375,7 +383,8 @@ TEST_F(ConfigProviderImplTest, LocalInfoNotDefined) { config_source_proto.set_api_type(envoy::api::v2::core::ApiConfigSource::GRPC); EXPECT_THROW_WITH_MESSAGE( provider_manager_->createXdsConfigProvider(config_source_proto, factory_context_, - "dummy_prefix"), + "dummy_prefix", + ConfigProviderManager::NullOptionalArg()), EnvoyException, "DummyDS: node 'id' and 'cluster' are required. Set it either in 'node' config or " "via --service-node and --service-cluster options."); diff --git a/test/common/http/conn_manager_impl_fuzz_test.cc b/test/common/http/conn_manager_impl_fuzz_test.cc index d2c53be04898b..c93a2198a162d 100644 --- a/test/common/http/conn_manager_impl_fuzz_test.cc +++ b/test/common/http/conn_manager_impl_fuzz_test.cc @@ -56,8 +56,24 @@ class FuzzConfig : public ConnectionManagerConfig { std::shared_ptr route_config_{new NiceMock()}; }; + struct ScopedRouteConfigProvider : public Config::ConfigProvider { + ScopedRouteConfigProvider(TimeSource& time_source) + : config_(std::make_shared()), time_source_(time_source) {} + + ~ScopedRouteConfigProvider() override = default; + + // Config::ConfigProvider + SystemTime lastUpdated() const override { return time_source_.systemTime(); } + const Protobuf::Message* getConfigProto() const override { return nullptr; } + std::string getConfigVersion() const override { return ""; } + ConfigConstSharedPtr getConfig() const override { return config_; } + + std::shared_ptr config_; + TimeSource& time_source_; + }; + FuzzConfig() - : route_config_provider_(time_system_), + : route_config_provider_(time_system_), scoped_route_config_provider_(time_system_), stats_{{ALL_HTTP_CONN_MAN_STATS(POOL_COUNTER(fake_stats_), POOL_GAUGE(fake_stats_), POOL_HISTOGRAM(fake_stats_))}, "", @@ -96,7 +112,10 @@ class FuzzConfig : public ConnectionManagerConfig { std::chrono::milliseconds streamIdleTimeout() const override { return stream_idle_timeout_; } std::chrono::milliseconds requestTimeout() const override { return request_timeout_; } std::chrono::milliseconds delayedCloseTimeout() const override { return delayed_close_timeout_; } - Router::RouteConfigProvider& routeConfigProvider() override { return route_config_provider_; } + Router::RouteConfigProvider* routeConfigProvider() override { return &route_config_provider_; } + Config::ConfigProvider* scopedRouteConfigProvider() override { + return &scoped_route_config_provider_; + } const std::string& serverName() override { return server_name_; } ConnectionManagerStats& stats() override { return stats_; } ConnectionManagerTracingStats& tracingStats() override { return tracing_stats_; } @@ -127,6 +146,7 @@ class FuzzConfig : public ConnectionManagerConfig { Event::SimulatedTimeSystem time_system_; SlowDateProviderImpl date_provider_{time_system_}; RouteConfigProvider route_config_provider_; + ScopedRouteConfigProvider scoped_route_config_provider_; std::string server_name_; Stats::IsolatedStoreImpl fake_stats_; ConnectionManagerStats stats_; diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index 527bb7b86578f..982415b6e910a 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -74,8 +74,25 @@ class HttpConnectionManagerImplTest : public TestBase, public ConnectionManagerC std::shared_ptr route_config_{new NiceMock()}; }; + struct ScopedRouteConfigProvider : public Config::ConfigProvider { + ScopedRouteConfigProvider(TimeSource& time_source) + : config_(std::make_shared()), time_source_(time_source) {} + + ~ScopedRouteConfigProvider() override = default; + + // Config::ConfigProvider + SystemTime lastUpdated() const override { return time_source_.systemTime(); } + const Protobuf::Message* getConfigProto() const override { return nullptr; } + std::string getConfigVersion() const override { return ""; } + ConfigConstSharedPtr getConfig() const override { return config_; } + + std::shared_ptr config_; + TimeSource& time_source_; + }; + HttpConnectionManagerImplTest() - : route_config_provider_(test_time_.timeSystem()), access_log_path_("dummy_path"), + : route_config_provider_(test_time_.timeSystem()), + scoped_route_config_provider_(test_time_.timeSystem()), access_log_path_("dummy_path"), access_logs_{ AccessLog::InstanceSharedPtr{new Extensions::AccessLoggers::File::FileAccessLog( access_log_path_, {}, AccessLog::AccessLogFormatUtils::defaultAccessLogFormatter(), @@ -235,7 +252,10 @@ class HttpConnectionManagerImplTest : public TestBase, public ConnectionManagerC std::chrono::milliseconds streamIdleTimeout() const override { return stream_idle_timeout_; } std::chrono::milliseconds requestTimeout() const override { return request_timeout_; } std::chrono::milliseconds delayedCloseTimeout() const override { return delayed_close_timeout_; } - Router::RouteConfigProvider& routeConfigProvider() override { return route_config_provider_; } + Router::RouteConfigProvider* routeConfigProvider() override { return &route_config_provider_; } + Config::ConfigProvider* scopedRouteConfigProvider() override { + return &scoped_route_config_provider_; + } const std::string& serverName() override { return server_name_; } ConnectionManagerStats& stats() override { return stats_; } ConnectionManagerTracingStats& tracingStats() override { return tracing_stats_; } @@ -259,6 +279,7 @@ class HttpConnectionManagerImplTest : public TestBase, public ConnectionManagerC DangerousDeprecatedTestTime test_time_; RouteConfigProvider route_config_provider_; + ScopedRouteConfigProvider scoped_route_config_provider_; NiceMock tracer_; Http::ContextImpl http_context_; NiceMock runtime_; diff --git a/test/common/http/conn_manager_utility_test.cc b/test/common/http/conn_manager_utility_test.cc index 6ede90b769f41..1fae6217df273 100644 --- a/test/common/http/conn_manager_utility_test.cc +++ b/test/common/http/conn_manager_utility_test.cc @@ -56,7 +56,8 @@ class MockConnectionManagerConfig : public ConnectionManagerConfig { MOCK_CONST_METHOD0(streamIdleTimeout, std::chrono::milliseconds()); MOCK_CONST_METHOD0(requestTimeout, std::chrono::milliseconds()); MOCK_CONST_METHOD0(delayedCloseTimeout, std::chrono::milliseconds()); - MOCK_METHOD0(routeConfigProvider, Router::RouteConfigProvider&()); + MOCK_METHOD0(routeConfigProvider, Router::RouteConfigProvider*()); + MOCK_METHOD0(scopedRouteConfigProvider, Config::ConfigProvider*()); MOCK_METHOD0(serverName, const std::string&()); MOCK_METHOD0(stats, ConnectionManagerStats&()); MOCK_METHOD0(tracingStats, ConnectionManagerTracingStats&()); diff --git a/test/common/router/BUILD b/test/common/router/BUILD index 1c303657c3d05..5d69255f1a3bf 100644 --- a/test/common/router/BUILD +++ b/test/common/router/BUILD @@ -72,6 +72,22 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "scoped_rds_test", + srcs = ["scoped_rds_test.cc"], + deps = [ + "//source/common/config:utility_lib", + "//source/common/http:message_lib", + "//source/common/json:json_loader_lib", + "//source/common/router:scoped_rds_lib", + "//source/server/http:admin_lib", + "//test/mocks/init:init_mocks", + "//test/mocks/server:server_mocks", + "//test/test_common:simulated_time_system_lib", + "//test/test_common:utility_lib", + ], +) + envoy_cc_test( name = "retry_state_impl_test", srcs = ["retry_state_impl_test.cc"], diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc new file mode 100644 index 0000000000000..adaa8a5f50b6c --- /dev/null +++ b/test/common/router/scoped_rds_test.cc @@ -0,0 +1,295 @@ +#include + +#include "envoy/admin/v2alpha/config_dump.pb.h" +#include "envoy/admin/v2alpha/config_dump.pb.validate.h" +#include "envoy/stats/scope.h" + +#include "common/router/scoped_rds.h" + +#include "test/mocks/server/mocks.h" +#include "test/test_common/simulated_time_system.h" + +#include "absl/strings/string_view.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::InSequence; +using testing::Return; + +namespace Envoy { +namespace Router { +namespace { + +void parseScopedRoutesConfigFromYaml( + envoy::api::v2::ScopedRouteConfigurationsSet& scoped_routes_proto, const std::string& yaml) { + MessageUtil::loadFromYaml(yaml, scoped_routes_proto); +} + +envoy::api::v2::ScopedRouteConfigurationsSet +parseScopedRoutesConfigFromYaml(const std::string& yaml) { + envoy::api::v2::ScopedRouteConfigurationsSet scoped_routes_proto; + parseScopedRoutesConfigFromYaml(scoped_routes_proto, yaml); + return scoped_routes_proto; +} + +class ScopedRoutesTestBase : public testing::Test { +protected: + ScopedRoutesTestBase() { + EXPECT_CALL(factory_context_.admin_.config_tracker_, add_("route_scopes", _)); + config_provider_manager_ = + std::make_unique(factory_context_.admin_); + + const std::string rds_config_yaml = R"EOF( +config_source: + api_config_source: + api_type: REST + cluster_names: + - foo_rds_cluster + refresh_delay: { seconds: 1, nanos: 0 } +scoped_rds_template: true + )EOF"; + MessageUtil::loadFromYaml(rds_config_yaml, rds_config_); + } + + ~ScopedRoutesTestBase() override { factory_context_.thread_local_.shutdownThread(); } + + Event::SimulatedTimeSystem& timeSystem() { return time_system_; } + + NiceMock factory_context_; + std::unique_ptr config_provider_manager_; + Event::SimulatedTimeSystem time_system_; + envoy::config::filter::network::http_connection_manager::v2::Rds rds_config_; +}; + +class ScopedRdsTest : public ScopedRoutesTestBase { +protected: + ScopedRdsTest() = default; + + ~ScopedRdsTest() override = default; + + void setup() { + InSequence s; + + Upstream::ClusterManager::ClusterInfoMap cluster_map; + Upstream::MockCluster cluster; + cluster_map.emplace("foo_cluster", cluster); + EXPECT_CALL(factory_context_.cluster_manager_, clusters()).WillOnce(Return(cluster_map)); + EXPECT_CALL(cluster, info()); + EXPECT_CALL(*cluster.info_, addedViaApi()); + EXPECT_CALL(cluster, info()); + EXPECT_CALL(*cluster.info_, type()); + + const std::string config_yaml = R"EOF( +config_source: + api_config_source: + api_type: REST + cluster_names: + - foo_cluster + refresh_delay: { seconds: 1, nanos: 0 } +scoped_routes_config_set_name: foo_scope_set + )EOF"; + envoy::config::filter::network::http_connection_manager::v2::ScopedRds scoped_rds_config; + MessageUtil::loadFromYaml(config_yaml, scoped_rds_config); + provider_ = config_provider_manager_->createXdsConfigProvider( + scoped_rds_config, factory_context_, "foo.", + ScopedRoutesConfigProviderManagerOptArg(rds_config_)); + subscription_ = &dynamic_cast(*provider_).subscription(); + } + + ScopedRdsConfigSubscription& subscription() const { return *subscription_; } + + ScopedRdsConfigSubscription* subscription_; + Envoy::Config::ConfigProviderPtr provider_; +}; + +TEST_F(ScopedRdsTest, ValidateFail) { + setup(); + + ScopedRdsConfigSubscription& subscription = + dynamic_cast(*provider_).subscription(); + + // 'name' validation + const std::string config_yaml = R"EOF( +name: +scope_key_builder: + fragments: + - header_value_extractor: { name: element } +scopes: + - route_configuration_name: foo_routes + key: + fragments: + - string_key: x-foo-key +)EOF"; + Protobuf::RepeatedPtrField resources; + parseScopedRoutesConfigFromYaml(*resources.Add(), config_yaml); + EXPECT_THROW(subscription.onConfigUpdate(resources, "1"), ProtoValidationException); + + // 'scope_key_builder.fragments' validation + const std::string config_yaml2 = R"EOF( +name: foo_scope_set +scope_key_builder: + fragments: +scopes: + - route_configuration_name: foo_routes + key: + fragments: + - string_key: x-foo-key +)EOF"; + Protobuf::RepeatedPtrField resources2; + parseScopedRoutesConfigFromYaml(*resources2.Add(), config_yaml2); + EXPECT_THROW(subscription.onConfigUpdate(resources2, "1"), ProtoValidationException); + + // 'scopes.fragments' validation + const std::string config_yaml3 = R"EOF( +name: foo_scope_set +scope_key_builder: + fragments: +scopes: + - route_configuration_name: foo_routes + key: +)EOF"; + Protobuf::RepeatedPtrField resources3; + parseScopedRoutesConfigFromYaml(*resources3.Add(), config_yaml3); + EXPECT_THROW(subscription.onConfigUpdate(resources3, "1"), ProtoValidationException); +} + +// Tests that an empty config update will update the corresponding stat. +TEST_F(ScopedRdsTest, EmptyResource) { + setup(); + + Protobuf::RepeatedPtrField resources; + subscription().onConfigUpdate(resources, "1"); + EXPECT_EQ(1UL, + factory_context_.scope_.counter("foo.scoped_rds.foo_scope_set.update_empty").value()); +} + +// Tests that only one resource is provided during a config update. +TEST_F(ScopedRdsTest, TooManyResources) { + setup(); + + Protobuf::RepeatedPtrField resources; + resources.Add(); + resources.Add(); + EXPECT_THROW(subscription().onConfigUpdate(resources, "1"), EnvoyException); +} + +TEST_F(ScopedRdsTest, ConfigUpdateSuccess) { + setup(); + + ScopedRdsConfigSubscription& subscription = + dynamic_cast(*provider_).subscription(); + + // 'name' validation + const std::string config_yaml = R"EOF( +name: foo_scope_set +scope_key_builder: + fragments: + - header_value_extractor: { name: element } +scopes: + - route_configuration_name: foo_routes + key: + fragments: + - string_key: x-foo-key +)EOF"; + Protobuf::RepeatedPtrField resources; + parseScopedRoutesConfigFromYaml(*resources.Add(), config_yaml); + subscription.onConfigUpdate(resources, "1"); + EXPECT_EQ(1UL, + factory_context_.scope_.counter("foo.scoped_rds.foo_scope_set.config_reload").value()); +} + +// Tests that defining an invalid cluster in the SRDS config results in an error. +TEST_F(ScopedRdsTest, UnknownCluster) { + const std::string config_yaml = R"EOF( +config_source: + api_config_source: + api_type: REST + cluster_names: + - foo_cluster + refresh_delay: { seconds: 1, nanos: 0 } +scoped_routes_config_set_name: foo_scope_set + )EOF"; + envoy::config::filter::network::http_connection_manager::v2::ScopedRds scoped_rds_config; + MessageUtil::loadFromYaml(config_yaml, scoped_rds_config); + + Upstream::ClusterManager::ClusterInfoMap cluster_map; + EXPECT_CALL(factory_context_.cluster_manager_, clusters()).WillOnce(Return(cluster_map)); + EXPECT_THROW_WITH_MESSAGE( + config_provider_manager_->createXdsConfigProvider( + scoped_rds_config, factory_context_, "foo.", + ScopedRoutesConfigProviderManagerOptArg(rds_config_)); + , EnvoyException, + "envoy::api::v2::core::ConfigSource must have a statically defined non-EDS " + "cluster: 'foo_cluster' does not exist, was added via api, or is an " + "EDS cluster"); +} + +class ScopedRoutesConfigProviderManagerTest : public ScopedRoutesTestBase { +public: + ScopedRoutesConfigProviderManagerTest() = default; + + ~ScopedRoutesConfigProviderManagerTest() override = default; +}; + +// Tests that the /configdump handler returns the corresponding scoped routing config. +TEST_F(ScopedRoutesConfigProviderManagerTest, ConfigDump) { + auto message_ptr = + factory_context_.admin_.config_tracker_.config_tracker_callbacks_["route_scopes"](); + const auto& scoped_routes_config_dump = + MessageUtil::downcastAndValidate( + *message_ptr); + + // No routes at all, no last_updated timestamp + envoy::admin::v2alpha::ScopedRoutesConfigDump expected_config_dump; + MessageUtil::loadFromYaml(R"EOF( +inline_scoped_routes_configs: +dynamic_scoped_routes_configs: +)EOF", + expected_config_dump); + EXPECT_EQ(expected_config_dump.DebugString(), scoped_routes_config_dump.DebugString()); + + std::string config_yaml = R"EOF( +name: foo +scope_key_builder: + fragments: + - header_value_extractor: { name: X-Google-VIP } +scopes: + - route_configuration_name: foo-route-config + key: + fragments: { string_key: "172.10.10.10" } +)EOF"; + + timeSystem().setSystemTime(std::chrono::milliseconds(1234567891234)); + + // Only load the inline scopes. + Envoy::Config::ConfigProviderPtr inline_config = + config_provider_manager_->createStaticConfigProvider( + parseScopedRoutesConfigFromYaml(config_yaml), factory_context_, + ScopedRoutesConfigProviderManagerOptArg(rds_config_)); + message_ptr = factory_context_.admin_.config_tracker_.config_tracker_callbacks_["route_scopes"](); + const auto& scoped_routes_config_dump2 = + MessageUtil::downcastAndValidate( + *message_ptr); + MessageUtil::loadFromYaml(R"EOF( +inline_scoped_routes_configs: + - scoped_routes_config: + name: foo + scope_key_builder: + fragments: + - header_value_extractor: { name: X-Google-VIP } + scopes: + - route_configuration_name: foo-route-config + key: + fragments: { string_key: "172.10.10.10" } + last_updated: + seconds: 1234567891 + nanos: 234000000 +dynamic_scoped_routes_configs: +)EOF", + expected_config_dump); + EXPECT_EQ(expected_config_dump.DebugString(), scoped_routes_config_dump2.DebugString()); +} + +} // namespace +} // namespace Router +} // namespace Envoy diff --git a/test/extensions/filters/network/http_connection_manager/config_test.cc b/test/extensions/filters/network/http_connection_manager/config_test.cc index 4fe6b5d7cc92d..4e2d5dbb2c200 100644 --- a/test/extensions/filters/network/http_connection_manager/config_test.cc +++ b/test/extensions/filters/network/http_connection_manager/config_test.cc @@ -6,6 +6,7 @@ #include "extensions/filters/network/http_connection_manager/config.h" +#include "test/mocks/config/mocks.h" #include "test/mocks/http/mocks.h" #include "test/mocks/network/mocks.h" #include "test/mocks/server/mocks.h" @@ -48,6 +49,7 @@ class HttpConnectionManagerConfigTest : public TestBase { NiceMock context_; Http::SlowDateProviderImpl date_provider_{context_.dispatcher().timeSystem()}; NiceMock route_config_provider_manager_; + NiceMock scoped_routes_config_provider_manager_; }; TEST_F(HttpConnectionManagerConfigTest, ValidateFail) { @@ -86,7 +88,8 @@ TEST_F(HttpConnectionManagerConfigTest, InvalidFilterName) { EXPECT_THROW_WITH_MESSAGE( HttpConnectionManagerConfig(parseHttpConnectionManagerFromJson(json_string), context_, - date_provider_, route_config_provider_manager_), + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_), EnvoyException, "Didn't find a registered implementation for name: 'foo'"); } @@ -122,7 +125,8 @@ TEST_F(HttpConnectionManagerConfigTest, MiscConfig) { )EOF"; HttpConnectionManagerConfig config(parseHttpConnectionManagerFromJson(json_string), context_, - date_provider_, route_config_provider_manager_); + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); EXPECT_THAT(std::vector({Http::LowerCaseString("foo")}), ContainerEq(config.tracingConfig()->request_headers_for_tags_)); @@ -143,7 +147,8 @@ TEST_F(HttpConnectionManagerConfigTest, UnixSocketInternalAddress) { )EOF"; HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, - date_provider_, route_config_provider_manager_); + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); Network::Address::PipeInstance unixAddress{"/foo"}; Network::Address::Ipv4Instance internalIpAddress{"127.0.0.1", 0}; Network::Address::Ipv4Instance externalIpAddress{"12.0.0.1", 0}; @@ -162,7 +167,8 @@ TEST_F(HttpConnectionManagerConfigTest, MaxRequestHeadersSizeDefault) { )EOF"; HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, - date_provider_, route_config_provider_manager_); + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); EXPECT_EQ(60, config.maxRequestHeadersSizeKb()); } @@ -177,7 +183,8 @@ TEST_F(HttpConnectionManagerConfigTest, MaxRequestHeadersSizeConfigured) { )EOF"; HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, - date_provider_, route_config_provider_manager_); + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); EXPECT_EQ(16, config.maxRequestHeadersSizeKb()); } @@ -193,7 +200,8 @@ TEST_F(HttpConnectionManagerConfigTest, DisabledStreamIdleTimeout) { )EOF"; HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, - date_provider_, route_config_provider_manager_); + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); EXPECT_EQ(0, config.streamIdleTimeout().count()); } @@ -208,7 +216,8 @@ TEST_F(HttpConnectionManagerConfigTest, ConfiguredRequestTimeout) { )EOF"; HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, - date_provider_, route_config_provider_manager_); + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); EXPECT_EQ(53 * 1000, config.requestTimeout().count()); } @@ -223,7 +232,8 @@ TEST_F(HttpConnectionManagerConfigTest, DisabledRequestTimeout) { )EOF"; HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, - date_provider_, route_config_provider_manager_); + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); EXPECT_EQ(0, config.requestTimeout().count()); } @@ -237,7 +247,8 @@ TEST_F(HttpConnectionManagerConfigTest, UnconfiguredRequestTimeout) { )EOF"; HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, - date_provider_, route_config_provider_manager_); + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); EXPECT_EQ(0, config.requestTimeout().count()); } @@ -447,10 +458,142 @@ TEST_F(HttpConnectionManagerConfigTest, ReversedEncodeOrderConfig) { )EOF"; HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, - date_provider_, route_config_provider_manager_); + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); EXPECT_TRUE(config.reverseEncodeOrder()); } +TEST_F(HttpConnectionManagerConfigTest, ScopedRoutingValidationFailures) { + const std::string config_yaml = R"EOF( +route_config: + name: foo_route +scoped_routes_config: + name: foo_route_scopes +codec_type: auto +stat_prefix: foo +http_filters: + - name: envoy.router +)EOF"; + + EXPECT_THROW_WITH_MESSAGE( + HttpConnectionManagerConfig(parseHttpConnectionManagerFromV2Yaml(config_yaml), context_, + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_), + EnvoyException, "Error: RDS must be used when scoped routing is enabled"); + + const std::string config_yaml2 = R"EOF( +route_config: + name: foo_route +scoped_rds: + config_source: { path: /path/to/file } + scoped_routes_config_set_name: foo_scope +codec_type: auto +stat_prefix: foo +http_filters: + - name: envoy.router +)EOF"; + + EXPECT_THROW_WITH_MESSAGE( + HttpConnectionManagerConfig(parseHttpConnectionManagerFromV2Yaml(config_yaml2), context_, + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_), + EnvoyException, "Error: RDS must be used when scoped routing is enabled"); + + const std::string config_yaml3 = R"EOF( +rds: + config_source: { path: /path/to/file } + route_config_name: foo_routes +scoped_routes_config: + name: foo_route_scopes +codec_type: auto +stat_prefix: foo +http_filters: + - name: envoy.router +)EOF"; + + EXPECT_THROW_WITH_MESSAGE( + HttpConnectionManagerConfig(parseHttpConnectionManagerFromV2Yaml(config_yaml3), context_, + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_), + EnvoyException, + "Error: the RDS subscription specifier must be set to scoped_rds_template=true " + "when scoped routing is enabled"); + + const std::string config_yaml4 = R"EOF( +rds: + config_source: { path: /path/to/file } + scoped_rds_template: true +codec_type: auto +stat_prefix: foo +http_filters: + - name: envoy.router +)EOF"; + + EXPECT_THROW_WITH_MESSAGE( + HttpConnectionManagerConfig(parseHttpConnectionManagerFromV2Yaml(config_yaml4), context_, + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_), + EnvoyException, + "Error: RDS must specify a route_config_name when scoped routing is not enabled"); +} + +TEST_F(HttpConnectionManagerConfigTest, ScopedRoutingValidationSuccess) { + const std::string config_yaml = R"EOF( +rds: + config_source: { path: /path/to/file } + scoped_rds_template: true +scoped_routes_config: + name: foo_route_scopes +codec_type: auto +stat_prefix: foo +http_filters: + - name: envoy.router +)EOF"; + + HttpConnectionManagerConfig(parseHttpConnectionManagerFromV2Yaml(config_yaml), context_, + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); + + const std::string config_yaml2 = R"EOF( +rds: + config_source: { path: /path/to/file } + scoped_rds_template: true +scoped_rds: + config_source: { path: /path/to/file } + scoped_routes_config_set_name: foo_scope +codec_type: auto +stat_prefix: foo +http_filters: + - name: envoy.router +)EOF"; + + HttpConnectionManagerConfig(parseHttpConnectionManagerFromV2Yaml(config_yaml2), context_, + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); +} + +TEST_F(HttpConnectionManagerConfigTest, ScopedRdsAndStatic) { + const std::string config_yaml = R"EOF( +rds: + config_source: { path: /path/to/file } + scoped_rds_template: true +scoped_routes_config: + name: foo_route_scopes +scoped_rds: + config_source: { path: /path/to/file } + scoped_routes_config_set_name: foo_scope +codec_type: auto +stat_prefix: foo +http_filters: + - name: envoy.router +)EOF"; + + EXPECT_THROW(HttpConnectionManagerConfig(parseHttpConnectionManagerFromV2Yaml(config_yaml), + context_, date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_), + EnvoyException); +} + class FilterChainTest : public HttpConnectionManagerConfigTest { public: const std::string basic_config_ = R"EOF( @@ -483,7 +626,8 @@ class FilterChainTest : public HttpConnectionManagerConfigTest { TEST_F(FilterChainTest, createFilterChain) { HttpConnectionManagerConfig config(parseHttpConnectionManagerFromJson(basic_config_), context_, - date_provider_, route_config_provider_manager_); + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); Http::MockFilterChainFactoryCallbacks callbacks; EXPECT_CALL(callbacks, addStreamFilter(_)); // Dynamo @@ -497,7 +641,8 @@ TEST_F(FilterChainTest, createUpgradeFilterChain) { hcm_config.add_upgrade_configs()->set_upgrade_type("websocket"); HttpConnectionManagerConfig config(hcm_config, context_, date_provider_, - route_config_provider_manager_); + route_config_provider_manager_, + scoped_routes_config_provider_manager_); NiceMock callbacks; // Check the case where WebSockets are configured in the HCM, and no router @@ -543,7 +688,8 @@ TEST_F(FilterChainTest, createUpgradeFilterChainHCMDisabled) { hcm_config.mutable_upgrade_configs(0)->mutable_enabled()->set_value(false); HttpConnectionManagerConfig config(hcm_config, context_, date_provider_, - route_config_provider_manager_); + route_config_provider_manager_, + scoped_routes_config_provider_manager_); NiceMock callbacks; // Check the case where WebSockets are off in the HCM, and no router config is present. @@ -592,7 +738,8 @@ TEST_F(FilterChainTest, createCustomUpgradeFilterChain) { "envoy.http_dynamo_filter"); HttpConnectionManagerConfig config(hcm_config, context_, date_provider_, - route_config_provider_manager_); + route_config_provider_manager_, + scoped_routes_config_provider_manager_); { Http::MockFilterChainFactoryCallbacks callbacks; @@ -621,7 +768,8 @@ TEST_F(FilterChainTest, invalidConfig) { hcm_config.add_upgrade_configs()->set_upgrade_type("websocket"); EXPECT_THROW_WITH_MESSAGE(HttpConnectionManagerConfig(hcm_config, context_, date_provider_, - route_config_provider_manager_), + route_config_provider_manager_, + scoped_routes_config_provider_manager_), EnvoyException, "Error: multiple upgrade configs with the same name: 'websocket'"); } diff --git a/test/integration/BUILD b/test/integration/BUILD index 41ea56a0626c2..18a34abff1a78 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -696,3 +696,20 @@ envoy_cc_fuzz_test( ":h1_fuzz_lib", ], ) + +envoy_cc_test( + name = "scoped_rds_integration_test", + srcs = [ + "scoped_rds_integration_test.cc", + ], + deps = [ + ":http_integration_lib", + "//source/common/config:resources_lib", + "//source/common/event:dispatcher_includes", + "//source/common/event:dispatcher_lib", + "//source/common/network:connection_lib", + "//source/common/network:utility_lib", + "//test/common/grpc:grpc_client_integration_lib", + "//test/test_common:utility_lib", + ], +) diff --git a/test/integration/integration_admin_test.cc b/test/integration/integration_admin_test.cc index 0dcdf795a00c1..dafeb0b9c2064 100644 --- a/test/integration/integration_admin_test.cc +++ b/test/integration/integration_admin_test.cc @@ -347,7 +347,9 @@ TEST_P(IntegrationAdminTest, Admin) { "type.googleapis.com/envoy.admin.v2alpha.BootstrapConfigDump", "type.googleapis.com/envoy.admin.v2alpha.ClustersConfigDump", "type.googleapis.com/envoy.admin.v2alpha.ListenersConfigDump", + "type.googleapis.com/envoy.admin.v2alpha.ScopedRoutesConfigDump", "type.googleapis.com/envoy.admin.v2alpha.RoutesConfigDump"}; + for (Json::ObjectSharedPtr obj_ptr : json->getObjectArray("configs")) { EXPECT_TRUE(expected_types[index].compare(obj_ptr->getString("@type")) == 0); index++; @@ -356,11 +358,11 @@ TEST_P(IntegrationAdminTest, Admin) { // Validate we can parse as proto. envoy::admin::v2alpha::ConfigDump config_dump; MessageUtil::loadFromJson(response->body(), config_dump); - EXPECT_EQ(4, config_dump.configs_size()); + EXPECT_EQ(5, config_dump.configs_size()); // .. and that we can unpack one of the entries. envoy::admin::v2alpha::RoutesConfigDump route_config_dump; - config_dump.configs(3).UnpackTo(&route_config_dump); + config_dump.configs(4).UnpackTo(&route_config_dump); EXPECT_EQ("route_config_0", route_config_dump.static_route_configs(0).route_config().name()); } diff --git a/test/integration/scoped_rds_integration_test.cc b/test/integration/scoped_rds_integration_test.cc new file mode 100644 index 0000000000000..590cae3fccc39 --- /dev/null +++ b/test/integration/scoped_rds_integration_test.cc @@ -0,0 +1,186 @@ +#include "envoy/api/v2/srds.pb.h" + +#include "common/config/resources.h" + +#include "test/common/grpc/grpc_client_integration.h" +#include "test/integration/http_integration.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace { + +class ScopedRdsIntegrationTest : public HttpIntegrationTest, + public Grpc::GrpcClientIntegrationParamTest { +protected: + struct FakeUpstreamInfo { + FakeHttpConnectionPtr connection_; + FakeUpstream* upstream_{}; + FakeStreamPtr stream_; + }; + + ScopedRdsIntegrationTest() + : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, ipVersion(), realTime()) {} + + ~ScopedRdsIntegrationTest() { + resetConnections(); + cleanupUpstreamAndDownstream(); + } + + void initialize() override { + config_helper_.addConfigModifier([this](envoy::config::bootstrap::v2::Bootstrap& bootstrap) { + // Add the static cluster to serve SRDS. + auto* scoped_rds_cluster = bootstrap.mutable_static_resources()->add_clusters(); + scoped_rds_cluster->MergeFrom(bootstrap.static_resources().clusters()[0]); + scoped_rds_cluster->set_name("srds_cluster"); + scoped_rds_cluster->mutable_http2_protocol_options(); + + // Add the static cluster to serve RDS. + auto* rds_cluster = bootstrap.mutable_static_resources()->add_clusters(); + rds_cluster->MergeFrom(bootstrap.static_resources().clusters()[0]); + rds_cluster->set_name("rds_cluster"); + rds_cluster->mutable_http2_protocol_options(); + }); + + config_helper_.addConfigModifier( + [this](envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& + http_connection_manager) { + // RDS must be enabled along when SRDS is enabled. + auto* rds = http_connection_manager.mutable_rds(); + rds->set_scoped_rds_template(true); + auto* api_config_source = rds->mutable_config_source()->mutable_api_config_source(); + api_config_source->set_api_type(envoy::api::v2::core::ApiConfigSource::GRPC); + auto* grpc_service = api_config_source->add_grpc_services(); + setGrpcService(*grpc_service, "rds_cluster", fake_upstreams_[2]->localAddress()); + + // Configure SRDS. + auto* scoped_rds = http_connection_manager.mutable_scoped_rds(); + scoped_rds->set_scoped_routes_config_set_name("foo_scope_set"); + api_config_source = scoped_rds->mutable_config_source()->mutable_api_config_source(); + api_config_source->set_api_type(envoy::api::v2::core::ApiConfigSource::GRPC); + grpc_service = api_config_source->add_grpc_services(); + setGrpcService(*grpc_service, "srds_cluster", fake_upstreams_[1]->localAddress()); + }); + + HttpIntegrationTest::initialize(); + } + + void createUpstreams() override { + HttpIntegrationTest::createUpstreams(); + // Create the SRDS upstream. + fake_upstreams_.emplace_back(new FakeUpstream(0, FakeHttpConnection::Type::HTTP2, version_, + timeSystem(), enable_half_close_)); + // Create the RDS upstream. + fake_upstreams_.emplace_back(new FakeUpstream(0, FakeHttpConnection::Type::HTTP2, version_, + timeSystem(), enable_half_close_)); + } + + void resetFakeUpstreamInfo(FakeUpstreamInfo* upstream_info) { + ASSERT(upstream_info->upstream_ != nullptr); + + upstream_info->upstream_->set_allow_unexpected_disconnects(true); + AssertionResult result = upstream_info->connection_->close(); + RELEASE_ASSERT(result, result.message()); + result = upstream_info->connection_->waitForDisconnect(); + RELEASE_ASSERT(result, result.message()); + upstream_info->connection_.reset(); + } + + void resetConnections() { + if (rds_upstream_info_.upstream_ != nullptr) { + resetFakeUpstreamInfo(&rds_upstream_info_); + } + resetFakeUpstreamInfo(&scoped_rds_upstream_info_); + } + + FakeUpstream& getRdsFakeUpstream() const { return *fake_upstreams_[2]; } + + FakeUpstream& getScopedRdsFakeUpstream() const { return *fake_upstreams_[1]; } + + void createStream(FakeUpstreamInfo* upstream_info, FakeUpstream& upstream) { + upstream_info->upstream_ = &upstream; + AssertionResult result = + upstream_info->upstream_->waitForHttpConnection(*dispatcher_, upstream_info->connection_); + RELEASE_ASSERT(result, result.message()); + result = upstream_info->connection_->waitForNewStream(*dispatcher_, upstream_info->stream_); + RELEASE_ASSERT(result, result.message()); + upstream_info->stream_->startGrpcStream(); + } + + void createRdsStream() { createStream(&rds_upstream_info_, getRdsFakeUpstream()); } + + void createScopedRdsStream() { + createStream(&scoped_rds_upstream_info_, getScopedRdsFakeUpstream()); + } + + void sendScopedRdsResponse(const std::string& response_yaml, const std::string& version) { + ASSERT(scoped_rds_upstream_info_.stream_ != nullptr); + + envoy::api::v2::DiscoveryResponse response; + response.set_version_info(version); + response.set_type_url(Config::TypeUrl::get().ScopedRouteConfigurationsSet); + + envoy::api::v2::ScopedRouteConfigurationsSet scoped_routes_proto; + MessageUtil::loadFromYaml(response_yaml, scoped_routes_proto); + response.add_resources()->PackFrom(scoped_routes_proto); + + scoped_rds_upstream_info_.stream_->sendGrpcMessage(response); + } + + FakeUpstreamInfo scoped_rds_upstream_info_; + FakeUpstreamInfo rds_upstream_info_; +}; + +INSTANTIATE_TEST_CASE_P(IpVersionsAndGrpcTypes, ScopedRdsIntegrationTest, + GRPC_CLIENT_INTEGRATION_PARAMS); + +TEST_P(ScopedRdsIntegrationTest, BasicSuccess) { + const std::string response_yaml = R"EOF( +name: foo_scope_set +scope_key_builder: + fragments: + - header_value_extractor: { name: element } +scopes: + - route_configuration_name: foo_routes + key: + fragments: + - string_key: x-foo-key +)EOF"; + + pre_worker_start_test_steps_ = [this, &response_yaml]() { + createScopedRdsStream(); + sendScopedRdsResponse(response_yaml, "1"); + }; + initialize(); + + test_server_->waitForCounterGe("http.config_test.scoped_rds.foo_scope_set.update_attempt", 1); + test_server_->waitForCounterGe("http.config_test.scoped_rds.foo_scope_set.update_success", 1); + // The version gauge should be set to xxHash64("1"). + test_server_->waitForGaugeEq("http.config_test.scoped_rds.foo_scope_set.version", + 13237225503670494420UL); + + const std::string response_yaml2 = R"EOF( +name: foo_scope_set +scope_key_builder: + fragments: + - header_value_extractor: { name: element } +scopes: + - route_configuration_name: foo_routes2 + key: + fragments: + - string_key: x-baz-key +)EOF"; + sendScopedRdsResponse(response_yaml, "2"); + + test_server_->waitForCounterGe("http.config_test.scoped_rds.foo_scope_set.update_attempt", 2); + test_server_->waitForCounterGe("http.config_test.scoped_rds.foo_scope_set.update_success", 2); + test_server_->waitForGaugeEq("http.config_test.scoped_rds.foo_scope_set.version", + 6927017134761466251UL); + + // TODO(AndresGuedez): test actual scoped routing logic; only the config hanlding is implemented + // at this point. +} + +} // namespace +} // namespace Envoy diff --git a/test/mocks/config/BUILD b/test/mocks/config/BUILD index f23e0337aa505..773d0abd1ec06 100644 --- a/test/mocks/config/BUILD +++ b/test/mocks/config/BUILD @@ -13,6 +13,7 @@ envoy_cc_mock( srcs = ["mocks.cc"], hdrs = ["mocks.h"], deps = [ + "//include/envoy/config:config_provider_manager_interface", "//include/envoy/config:grpc_mux_interface", "//include/envoy/config:subscription_interface", "//source/common/config:resources_lib", diff --git a/test/mocks/config/mocks.h b/test/mocks/config/mocks.h index 99014dad6defb..713c1882b24ed 100644 --- a/test/mocks/config/mocks.h +++ b/test/mocks/config/mocks.h @@ -1,6 +1,7 @@ #pragma once #include "envoy/api/v2/eds.pb.h" +#include "envoy/config/config_provider_manager.h" #include "envoy/config/grpc_mux.h" #include "envoy/config/subscription.h" @@ -74,5 +75,21 @@ class MockGrpcMuxCallbacks : public GrpcMuxCallbacks { MOCK_METHOD1(resourceName, std::string(const ProtobufWkt::Any& resource)); }; +class MockConfigProviderManager : public ConfigProviderManager { +public: + MockConfigProviderManager() = default; + ~MockConfigProviderManager() override = default; + + MOCK_METHOD4(createXdsConfigProvider, + ConfigProviderPtr(const Protobuf::Message& config_source_proto, + Server::Configuration::FactoryContext& factory_context, + const std::string& stat_prefix, + const Envoy::Config::ConfigProviderManager::OptionalArg& optarg)); + MOCK_METHOD3(createStaticConfigProvider, + ConfigProviderPtr(const Protobuf::Message& config_proto, + Server::Configuration::FactoryContext& factory_context, + const Envoy::Config::ConfigProviderManager::OptionalArg& optarg)); +}; + } // namespace Config } // namespace Envoy diff --git a/test/mocks/router/BUILD b/test/mocks/router/BUILD index 75e4a07c6f393..e5a6b0d44d59a 100644 --- a/test/mocks/router/BUILD +++ b/test/mocks/router/BUILD @@ -20,6 +20,7 @@ envoy_cc_mock( "//include/envoy/router:route_config_provider_manager_interface", "//include/envoy/router:router_interface", "//include/envoy/router:router_ratelimit_interface", + "//include/envoy/router:scopes_interface", "//include/envoy/router:shadow_writer_interface", "//include/envoy/runtime:runtime_interface", "//include/envoy/stats:stats_interface", diff --git a/test/mocks/router/mocks.h b/test/mocks/router/mocks.h index c07459c63114a..fcc48e2c85a6e 100644 --- a/test/mocks/router/mocks.h +++ b/test/mocks/router/mocks.h @@ -17,6 +17,7 @@ #include "envoy/router/route_config_provider_manager.h" #include "envoy/router/router.h" #include "envoy/router/router_ratelimit.h" +#include "envoy/router/scopes.h" #include "envoy/router/shadow_writer.h" #include "envoy/runtime/runtime.h" #include "envoy/thread_local/thread_local.h" @@ -339,5 +340,13 @@ class MockRouteConfigProviderManager : public RouteConfigProviderManager { Server::Configuration::FactoryContext& factory_context)); }; +class MockScopedConfig : public ScopedConfig { +public: + MockScopedConfig() = default; + ~MockScopedConfig() override = default; + + MOCK_CONST_METHOD1(getRouterConfig, ConfigConstSharedPtr(const Http::HeaderMap& headers)); +}; + } // namespace Router } // namespace Envoy From db484505529a6506bcd7fb03b11745b80f3c0054 Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Tue, 5 Feb 2019 10:42:31 -0500 Subject: [PATCH 02/37] Derive test class from TestBase. Signed-off-by: Andres Guedez --- test/common/router/scoped_rds_test.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index adaa8a5f50b6c..b4d22e85dcbcb 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -8,6 +8,7 @@ #include "test/mocks/server/mocks.h" #include "test/test_common/simulated_time_system.h" +#include "test/test_common/test_base.h" #include "absl/strings/string_view.h" #include "gmock/gmock.h" @@ -32,7 +33,7 @@ parseScopedRoutesConfigFromYaml(const std::string& yaml) { return scoped_routes_proto; } -class ScopedRoutesTestBase : public testing::Test { +class ScopedRoutesTestBase : public TestBase { protected: ScopedRoutesTestBase() { EXPECT_CALL(factory_context_.admin_.config_tracker_, add_("route_scopes", _)); From 756afcd62bcb4dfcf39fd70eda62c9cc77cf8fde Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Tue, 5 Feb 2019 10:45:48 -0500 Subject: [PATCH 03/37] fix_format fixes. Signed-off-by: Andres Guedez --- api/envoy/api/v2/srds.proto | 13 +++++----- source/common/http/conn_manager_impl.cc | 33 ++++++++++++------------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/api/envoy/api/v2/srds.proto b/api/envoy/api/v2/srds.proto index fcd6563f9ec36..acdcf6e9a9c70 100644 --- a/api/envoy/api/v2/srds.proto +++ b/api/envoy/api/v2/srds.proto @@ -49,7 +49,8 @@ service ScopedRoutesDiscoveryService { // [#comment:next free field: 4] message ScopedRouteConfigurationsSet { // The name of the set of route configuration scopes. This will be the - // :ref:`scoped_routes_config_set_name ` + // :ref:`scoped_routes_config_set_name + // ` // set in :ref:`envoy_api_msg_config.filter.network.http_connection_manager.v2.ScopedRds`. string name = 1 [(validate.rules).string.min_bytes = 1]; @@ -58,7 +59,7 @@ message ScopedRouteConfigurationsSet { // // Upon receiving a request's headers, the Router will build a key using the algorithm specified // by this message. This key will be used to look up a corresponding - // :ref:`Scope ` in a table containing the set + // :ref:`Scope ` in a table containing the set // of :ref:`scopes `. message ScopeKeyBuilder { // Specifies the mechanism for constructing fragments which are composed into keys. @@ -111,14 +112,14 @@ message ScopedRouteConfigurationsSet { } // The constructed key consists of the union of these fragments. - repeated FragmentBuilder fragments = 1 [(validate.rules).repeated.min_items = 1]; + repeated FragmentBuilder fragments = 1 [(validate.rules).repeated .min_items = 1]; } // The key construction mechanism. ScopeKeyBuilder scope_key_builder = 2 [(validate.rules).message.required = true]; // Specifies a routing scope, which associates a :ref:`envoy_api_msg_RouteConfiguration` to a - // :ref:`Key ` which is matched against + // :ref:`Key ` which is matched against // each HTTP request. message Scope { // Specifies a key which is matched against by the output of a :ref:`ScopeKeyBuilder`. @@ -133,7 +134,7 @@ message ScopedRouteConfigurationsSet { } // The ordered set of fragments to match against. - repeated Fragment fragments = 1 [(validate.rules).repeated.min_items = 1]; + repeated Fragment fragments = 1 [(validate.rules).repeated .min_items = 1]; } // The resource name to use for a :ref:`envoy_api_msg_DiscoveryRequest` to an RDS server to @@ -146,5 +147,5 @@ message ScopedRouteConfigurationsSet { // The set of scopes containing :ref:`Key ` // to :ref:`envoy_api_msg_RouteConfiguration` mappings. - repeated Scope scopes = 3 [(validate.rules).repeated.min_items = 1]; + repeated Scope scopes = 3 [(validate.rules).repeated .min_items = 1]; } diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index a7fe67d3316fa..89edcc11ed375 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -1053,23 +1053,22 @@ void ConnectionManagerImpl::ActiveStream::sendLocalReply( if (!state_.created_filter_chain_) { createFilterChain(); } - Utility::sendLocalReply( - is_grpc_request, - [this, modify_headers](HeaderMapPtr&& headers, bool end_stream) -> void { - if (modify_headers != nullptr) { - modify_headers(*headers); - } - response_headers_ = std::move(headers); - // TODO: Start encoding from the last decoder filter that saw the - // request instead. - encodeHeaders(nullptr, *response_headers_, end_stream); - }, - [this](Buffer::Instance& data, bool end_stream) -> void { - // TODO: Start encoding from the last decoder filter that saw the - // request instead. - encodeData(nullptr, data, end_stream); - }, - state_.destroyed_, code, body, grpc_status, is_head_request); + Utility::sendLocalReply(is_grpc_request, + [this, modify_headers](HeaderMapPtr&& headers, bool end_stream) -> void { + if (modify_headers != nullptr) { + modify_headers(*headers); + } + response_headers_ = std::move(headers); + // TODO: Start encoding from the last decoder filter that saw the + // request instead. + encodeHeaders(nullptr, *response_headers_, end_stream); + }, + [this](Buffer::Instance& data, bool end_stream) -> void { + // TODO: Start encoding from the last decoder filter that saw the + // request instead. + encodeData(nullptr, data, end_stream); + }, + state_.destroyed_, code, body, grpc_status, is_head_request); } void ConnectionManagerImpl::ActiveStream::encode100ContinueHeaders( From 98149e5c5cd185e93a4c7c355dd8a9469a5781ab Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Tue, 5 Feb 2019 11:28:30 -0500 Subject: [PATCH 04/37] Minor cleanup. Signed-off-by: Andres Guedez --- test/integration/scoped_rds_integration_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/scoped_rds_integration_test.cc b/test/integration/scoped_rds_integration_test.cc index 590cae3fccc39..83111bc23c54e 100644 --- a/test/integration/scoped_rds_integration_test.cc +++ b/test/integration/scoped_rds_integration_test.cc @@ -29,7 +29,7 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, } void initialize() override { - config_helper_.addConfigModifier([this](envoy::config::bootstrap::v2::Bootstrap& bootstrap) { + config_helper_.addConfigModifier([](envoy::config::bootstrap::v2::Bootstrap& bootstrap) { // Add the static cluster to serve SRDS. auto* scoped_rds_cluster = bootstrap.mutable_static_resources()->add_clusters(); scoped_rds_cluster->MergeFrom(bootstrap.static_resources().clusters()[0]); From 76b71960ea29f94a7b58619ac2d1aa1a95af1718 Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Tue, 5 Feb 2019 16:55:30 -0500 Subject: [PATCH 05/37] Fix srds.proto docs. Signed-off-by: Andres Guedez --- api/envoy/api/v2/srds.proto | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/api/envoy/api/v2/srds.proto b/api/envoy/api/v2/srds.proto index acdcf6e9a9c70..ce8b5482e721b 100644 --- a/api/envoy/api/v2/srds.proto +++ b/api/envoy/api/v2/srds.proto @@ -2,6 +2,7 @@ syntax = "proto3"; package envoy.api.v2; +option java_outer_classname = "SrdsProto"; option java_package = "io.envoyproxy.envoy.api.v2"; option java_multiple_files = true; option java_generic_services = true; @@ -50,16 +51,16 @@ service ScopedRoutesDiscoveryService { message ScopedRouteConfigurationsSet { // The name of the set of route configuration scopes. This will be the // :ref:`scoped_routes_config_set_name - // ` + // ` // set in :ref:`envoy_api_msg_config.filter.network.http_connection_manager.v2.ScopedRds`. string name = 1 [(validate.rules).string.min_bytes = 1]; // Specifies the mechanism for constructing keys based on request attributes to match - // :ref:`envoy_api_field_api.v2.ScopedRouteConfigurationsSet.scopes` against. + // :ref:`scopes ` against. // // Upon receiving a request's headers, the Router will build a key using the algorithm specified // by this message. This key will be used to look up a corresponding - // :ref:`Scope ` in a table containing the set + // :ref:`Scope ` in a table containing the set // of :ref:`scopes `. message ScopeKeyBuilder { // Specifies the mechanism for constructing fragments which are composed into keys. @@ -122,7 +123,8 @@ message ScopedRouteConfigurationsSet { // :ref:`Key ` which is matched against // each HTTP request. message Scope { - // Specifies a key which is matched against by the output of a :ref:`ScopeKeyBuilder`. + // Specifies a key which is matched against by the output of a :ref:`ScopeKeyBuilder + // `. message Key { message Fragment { oneof type { From f924eb7c13e4f87a0e606ed22a60c0c164fd209d Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Wed, 6 Feb 2019 09:40:02 -0500 Subject: [PATCH 06/37] Pendatic spelling and docs cleanup. Signed-off-by: Andres Guedez --- api/envoy/api/v2/srds.proto | 4 ++-- .../filters/network/http_connection_manager/config.cc | 2 +- test/integration/scoped_rds_integration_test.cc | 2 +- tools/spelling_dictionary.txt | 3 +++ 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/api/envoy/api/v2/srds.proto b/api/envoy/api/v2/srds.proto index ce8b5482e721b..1b5f500ed4776 100644 --- a/api/envoy/api/v2/srds.proto +++ b/api/envoy/api/v2/srds.proto @@ -18,7 +18,7 @@ option (gogoproto.equal_all) = true; // [#protodoc-title: HTTP scoped routing configuration] // * Routing :ref:`architecture overview ` - +// // .. attention:: // // The Scoped RDS API is not yet fully implemented and *should not* be enabled in @@ -68,7 +68,7 @@ message ScopedRouteConfigurationsSet { // Specifies how the value of a header should be extracted. // The following example maps the structure of a header to the fields in this message. // - // .. parsed-literal:: + // .. code:: // // X-Header: a,b;c,d // | || | diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index babc8fbb70edb..cf1c731dd68c8 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -203,7 +203,7 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( // Throws an exception on failure. validateScopedRoutingAndRds(config); - // If soped RDS is enabled, avoid creating a route config provider. Route config providers will be + // If scoped RDS is enabled, avoid creating a route config provider. Route config providers will be // managed by the scoped routing logic instead. if (config.rds().subscription_specifier_case() != envoy::config::filter::network::http_connection_manager::v2::Rds::kScopedRdsTemplate) { diff --git a/test/integration/scoped_rds_integration_test.cc b/test/integration/scoped_rds_integration_test.cc index 83111bc23c54e..a7b35dcc82bf7 100644 --- a/test/integration/scoped_rds_integration_test.cc +++ b/test/integration/scoped_rds_integration_test.cc @@ -178,7 +178,7 @@ name: foo_scope_set test_server_->waitForGaugeEq("http.config_test.scoped_rds.foo_scope_set.version", 6927017134761466251UL); - // TODO(AndresGuedez): test actual scoped routing logic; only the config hanlding is implemented + // TODO(AndresGuedez): test actual scoped routing logic; only the config handling is implemented // at this point. } diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index f7a4911917f8a..644b900e5ae6d 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -207,6 +207,7 @@ SPIFFE SPKI SQL SR +SRDS SRV SS SSL @@ -319,6 +320,7 @@ comparator cond condvar conf +configdump conn conns const @@ -718,6 +720,7 @@ usr util utils validator +validators vanishingly var variadic From 51d8dfff99faa008a36c566bdfc21c3a9f3408db Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Wed, 6 Feb 2019 09:41:18 -0500 Subject: [PATCH 07/37] fix_format. Signed-off-by: Andres Guedez --- .../filters/network/http_connection_manager/config.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index cf1c731dd68c8..7f1d91db23577 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -203,8 +203,8 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( // Throws an exception on failure. validateScopedRoutingAndRds(config); - // If scoped RDS is enabled, avoid creating a route config provider. Route config providers will be - // managed by the scoped routing logic instead. + // If scoped RDS is enabled, avoid creating a route config provider. Route config providers will + // be managed by the scoped routing logic instead. if (config.rds().subscription_specifier_case() != envoy::config::filter::network::http_connection_manager::v2::Rds::kScopedRdsTemplate) { route_config_provider_ = Router::RouteConfigProviderUtil::create( From ccca775ddf22f09b2a01497a7a546b6d56e5c88e Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Wed, 27 Feb 2019 11:37:00 -0500 Subject: [PATCH 08/37] More detailed documentation/examples for SRDS. Signed-off-by: Andres Guedez --- api/envoy/api/v2/srds.proto | 94 +++++++++++++++---- .../v2/http_connection_manager.proto | 5 +- include/envoy/router/scopes.h | 2 +- source/common/router/scoped_config_impl.h | 6 -- source/common/router/scoped_rds.cc | 2 +- .../network/http_connection_manager/config.cc | 2 +- test/common/http/BUILD | 13 +++ test/common/http/conn_manager_impl_common.h | 51 ++++++++++ .../http/conn_manager_impl_fuzz_test.cc | 33 +------ test/common/http/conn_manager_impl_test.cc | 33 +------ test/common/router/scoped_rds_test.cc | 8 +- 11 files changed, 157 insertions(+), 92 deletions(-) create mode 100644 test/common/http/conn_manager_impl_common.h diff --git a/api/envoy/api/v2/srds.proto b/api/envoy/api/v2/srds.proto index 1b5f500ed4776..9032ad6e4ad05 100644 --- a/api/envoy/api/v2/srds.proto +++ b/api/envoy/api/v2/srds.proto @@ -23,10 +23,14 @@ option (gogoproto.equal_all) = true; // // The Scoped RDS API is not yet fully implemented and *should not* be enabled in // :ref:`envoy_api_msg_config.filter.network.http_connection_manager.v2.HttpConnectionManager`. +// +// TODO(AndresGuedez): Update :ref:`arch_overview_http_routing` with scoped routing overview and +// configuration details. -// The resource_names field in DiscoveryRequest specifies a set of route configuration scopes. Each -// scope points to a route_configuration_name which will be used to request a -// :ref:`RouteConfiguration` via the RDS API. +// The `ref:resource_names` field in +// DiscoveryRequest specifies a set of route configuration scopes. Each scope points to a +// route_configuration_name which will be used to request a +// :ref:`RouteConfiguration` via the RDS API. service ScopedRoutesDiscoveryService { rpc StreamScopedRoutes(stream DiscoveryRequest) returns (stream DiscoveryResponse) { } @@ -43,10 +47,62 @@ service ScopedRoutesDiscoveryService { } } -// This configuration represents a set of "scopes", each containing independent routing -// configuration (see :ref:`routing architecture overview `). A scope is -// assigned to each request based on request attributes, such as the value of a header designated -// via this configuration. +// This configuration represents a set of routing 'scopes', each containing independent routing +// configuration (see :ref:`routing architecture overview `). +// +// A ScopedRouteConfigurationsSet is associated with an +// :ref:`HttpConnectionManager` +// via the +// :ref:`scoped_rds` +// or +// :ref:`scoped_routes_config` +// fields. When scoped routing is enabled, the Envoy router will first attempt to determine the +// routing scope assigned to a request by building a +// :ref:`Scope.Key` based on request header +// attributes (such as the value of a header field) defined by the +// :ref:`ScopeKeyBuilder` +// this message. +// +// The key will be looked up in a table maintained by the router, containing the +// :ref:`Scope.Key` to +// :ref:`RouteConfiguration` mappings specified by the +// :ref:`scopes` field in this message. The +// routing rules specified by the resulting RouteConfiguration will be used to determine the route +// for the request. +// +// For example, with the following YAML configuration: +// +// .. code:: +// +// scope_key_builder: +// fragments: +// - header_element: +// name: X-Route-Selector +// element_separator: , +// element: +// separator: = +// key: vip +// scopes: +// - route_configuration_name: route-config1 +// key: +// fragments: +// - string_key: 172.10.10.20 +// - route_configuration_name: route-config2 +// key: +// fragments: +// - string_key: 172.20.20.30 +// +// A request from a client such as: +// +// .. code:: +// +// GET / HTTP/1.1 +// Host: foo.com +// X-Route-Selector: vip=172.10.10.20 +// +// Would result in the `route-config1` RouteConfiguration being assigned to the request/stream, +// which Envoy would fetch via RDS. +// // [#comment:next free field: 4] message ScopedRouteConfigurationsSet { // The name of the set of route configuration scopes. This will be the @@ -61,7 +117,7 @@ message ScopedRouteConfigurationsSet { // Upon receiving a request's headers, the Router will build a key using the algorithm specified // by this message. This key will be used to look up a corresponding // :ref:`Scope ` in a table containing the set - // of :ref:`scopes `. + // of :ref:`scopes ` in this message. message ScopeKeyBuilder { // Specifies the mechanism for constructing fragments which are composed into keys. message FragmentBuilder { @@ -70,23 +126,25 @@ message ScopedRouteConfigurationsSet { // // .. code:: // - // X-Header: a,b;c,d + // X-Header: a=b;c=d // | || | - // | || \\----> + // | || \----> // | || - // | |\\----> + // | |\----> // | | - // | \\----> + // | \----> // | - // \\----> + // \----> + // + // Each 'a=b' key-value pair constitutes an 'element' of the header field. message HeaderValueExtractor { - // The name of the header to extract the value from. + // The name of the header field to extract the value from. string name = 1 [(validate.rules).string.min_bytes = 1]; // The element separator (e.g., ';' separates 'a;b;c;d'). string element_separator = 2; - // Specifies a key value pair to match on. + // Specifies a header field's key value pair to match on. message KvElement { // The separator between key and value (e.g., '=' separates 'k=v;...'). string separator = 1; @@ -112,7 +170,7 @@ message ScopedRouteConfigurationsSet { } } - // The constructed key consists of the union of these fragments. + // The constructed key consists of the ordered union of these fragments. repeated FragmentBuilder fragments = 1 [(validate.rules).repeated .min_items = 1]; } @@ -123,7 +181,7 @@ message ScopedRouteConfigurationsSet { // :ref:`Key ` which is matched against // each HTTP request. message Scope { - // Specifies a key which is matched against by the output of a :ref:`ScopeKeyBuilder + // Specifies a key which is matched against the output of a :ref:`ScopeKeyBuilder // `. message Key { message Fragment { @@ -147,7 +205,7 @@ message ScopedRouteConfigurationsSet { Key key = 2 [(validate.rules).message.required = true]; } - // The set of scopes containing :ref:`Key ` + // The set of scopes, containing :ref:`Key ` // to :ref:`envoy_api_msg_RouteConfiguration` mappings. repeated Scope scopes = 3 [(validate.rules).repeated .min_items = 1]; } diff --git a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto index 09df08fe1bf04..7350dc2dd73c2 100644 --- a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto +++ b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto @@ -409,7 +409,10 @@ message Rds { // Must be set to true when scoped RDS is enabled. // If this is enabled, RDS subscriptions will be triggered by scoped RDS config updates using - // this configuration as a template. + // this configuration as a template; i.e., the + // :ref:`config_source` + // from this message will be used to specify the config source for subscriptions initiated by + // scoped RDS. bool scoped_rds_template = 3; } } diff --git a/include/envoy/router/scopes.h b/include/envoy/router/scopes.h index ed675fcab8e46..fcdee3c38472d 100644 --- a/include/envoy/router/scopes.h +++ b/include/envoy/router/scopes.h @@ -17,7 +17,7 @@ class ScopedConfig : public Envoy::Config::ConfigProvider::Config { /** * Based on the incoming HTTP request headers, returns the configuration to use for selecting a - * target route (to either a route entry or a direct response entry). + * target route. * @param headers the request headers to match the scoped routing configuration against. * @return ConfigConstSharedPtr the router's Config matching the request headers. */ diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h index 46ad8470b4288..603be142b06ed 100644 --- a/source/common/router/scoped_config_impl.h +++ b/source/common/router/scoped_config_impl.h @@ -21,8 +21,6 @@ class ScopedConfigImpl : public ScopedConfig { ScopedConfigImpl(const envoy::api::v2::ScopedRouteConfigurationsSet& config_proto) : name_(config_proto.name()) {} - ~ScopedConfigImpl() override = default; - Router::ConfigConstSharedPtr getRouterConfig(const Http::HeaderMap& headers) const override; private: @@ -34,10 +32,6 @@ class ScopedConfigImpl : public ScopedConfig { */ class NullScopedConfigImpl : public ScopedConfig { public: - NullScopedConfigImpl() = default; - - ~NullScopedConfigImpl() override = default; - Router::ConfigConstSharedPtr getRouterConfig(const Http::HeaderMap&) const override { return std::make_shared(); } diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index e2b9ba991cfee..5bf8ff8ce3300 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -85,7 +85,7 @@ void ScopedRdsConfigSubscription::onConfigUpdate(const ResourceVector& resources const auto& scoped_routes_config = resources[0]; MessageUtil::validate(scoped_routes_config); // The name we receive in the config must match the name requested. - if (!(scoped_routes_config.name() == scoped_routes_config_name_)) { + if (scoped_routes_config.name() != scoped_routes_config_name_) { throw EnvoyException(fmt::format("Unexpected SRDS configuration (expecting {}): {}", scoped_routes_config_name_, scoped_routes_config.name())); } diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index 7f1d91db23577..be70ba7dd470a 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -200,7 +200,7 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( context_.listenerScope())), proxy_100_continue_(config.proxy_100_continue()), delayed_close_timeout_(PROTOBUF_GET_MS_OR_DEFAULT(config, delayed_close_timeout, 1000)) { - // Throws an exception on failure. + // Throws an exception on invalid config. validateScopedRoutingAndRds(config); // If scoped RDS is enabled, avoid creating a route config provider. Route config providers will diff --git a/test/common/http/BUILD b/test/common/http/BUILD index d810ea68df6dc..bc7661e2d22e7 100644 --- a/test/common/http/BUILD +++ b/test/common/http/BUILD @@ -109,6 +109,17 @@ envoy_cc_test_library( ], ) +envoy_cc_test_library( + name = "conn_manager_impl_common_lib", + hdrs = ["conn_manager_impl_common.h"], + deps = [ + "//include/envoy/common:time_interface", + "//include/envoy/config:config_provider_interface", + "//include/envoy/router:rds_interface", + "//test/mocks/router:router_mocks", + ], +) + envoy_proto_library( name = "conn_manager_impl_fuzz_proto", srcs = ["conn_manager_impl_fuzz.proto"], @@ -122,6 +133,7 @@ envoy_cc_fuzz_test( srcs = ["conn_manager_impl_fuzz_test.cc"], corpus = "conn_manager_impl_corpus", deps = [ + ":conn_manager_impl_common_lib", ":conn_manager_impl_fuzz_proto_cc", "//source/common/common:empty_string", "//source/common/http:conn_manager_lib", @@ -147,6 +159,7 @@ envoy_cc_test( name = "conn_manager_impl_test", srcs = ["conn_manager_impl_test.cc"], deps = [ + ":conn_manager_impl_common_lib", "//include/envoy/access_log:access_log_interface", "//include/envoy/buffer:buffer_interface", "//include/envoy/event:dispatcher_interface", diff --git a/test/common/http/conn_manager_impl_common.h b/test/common/http/conn_manager_impl_common.h new file mode 100644 index 0000000000000..a363d7d816fcf --- /dev/null +++ b/test/common/http/conn_manager_impl_common.h @@ -0,0 +1,51 @@ +#pragma once + +#include + +#include "envoy/common/time.h" +#include "envoy/config/config_provider.h" +#include "envoy/router/rds.h" + +#include "test/mocks/router/mocks.h" + +#include "gmock/gmock.h" + +using testing::NiceMock; + +namespace Envoy { +namespace Http { +namespace ConnectionManagerImplHelper { + +// Test RouteConfigProvider that returns a mocked config. +struct RouteConfigProvider : public Router::RouteConfigProvider { + RouteConfigProvider(TimeSource& time_source) : time_source_(time_source) {} + + // Router::RouteConfigProvider + Router::ConfigConstSharedPtr config() override { return route_config_; } + absl::optional configInfo() const override { return {}; } + SystemTime lastUpdated() const override { return time_source_.systemTime(); } + + TimeSource& time_source_; + std::shared_ptr route_config_{new NiceMock()}; +}; + +// Test ScopedRouteConfigProvider that returns a mocked config. +struct ScopedRouteConfigProvider : public Config::ConfigProvider { + ScopedRouteConfigProvider(TimeSource& time_source) + : config_(std::make_shared()), time_source_(time_source) {} + + ~ScopedRouteConfigProvider() override = default; + + // Config::ConfigProvider + SystemTime lastUpdated() const override { return time_source_.systemTime(); } + const Protobuf::Message* getConfigProto() const override { return nullptr; } + std::string getConfigVersion() const override { return ""; } + ConfigConstSharedPtr getConfig() const override { return config_; } + + std::shared_ptr config_; + TimeSource& time_source_; +}; + +} // namespace ConnectionManagerImplHelper +} // namespace Http +} // namespace Envoy diff --git a/test/common/http/conn_manager_impl_fuzz_test.cc b/test/common/http/conn_manager_impl_fuzz_test.cc index 47af30cfff355..5a8a4e67f38a2 100644 --- a/test/common/http/conn_manager_impl_fuzz_test.cc +++ b/test/common/http/conn_manager_impl_fuzz_test.cc @@ -20,6 +20,7 @@ #include "common/network/address_impl.h" #include "common/network/utility.h" +#include "test/common/http/conn_manager_impl_common.h" #include "test/common/http/conn_manager_impl_fuzz.pb.h" #include "test/fuzz/fuzz_runner.h" #include "test/fuzz/utility.h" @@ -44,34 +45,6 @@ namespace Http { class FuzzConfig : public ConnectionManagerConfig { public: - struct RouteConfigProvider : public Router::RouteConfigProvider { - RouteConfigProvider(TimeSource& time_source) : time_source_(time_source) {} - - // Router::RouteConfigProvider - Router::ConfigConstSharedPtr config() override { return route_config_; } - absl::optional configInfo() const override { return {}; } - SystemTime lastUpdated() const override { return time_source_.systemTime(); } - - TimeSource& time_source_; - std::shared_ptr route_config_{new NiceMock()}; - }; - - struct ScopedRouteConfigProvider : public Config::ConfigProvider { - ScopedRouteConfigProvider(TimeSource& time_source) - : config_(std::make_shared()), time_source_(time_source) {} - - ~ScopedRouteConfigProvider() override = default; - - // Config::ConfigProvider - SystemTime lastUpdated() const override { return time_source_.systemTime(); } - const Protobuf::Message* getConfigProto() const override { return nullptr; } - std::string getConfigVersion() const override { return ""; } - ConfigConstSharedPtr getConfig() const override { return config_; } - - std::shared_ptr config_; - TimeSource& time_source_; - }; - FuzzConfig() : route_config_provider_(time_system_), scoped_route_config_provider_(time_system_), stats_{{ALL_HTTP_CONN_MAN_STATS(POOL_COUNTER(fake_stats_), POOL_GAUGE(fake_stats_), @@ -145,8 +118,8 @@ class FuzzConfig : public ConnectionManagerConfig { NiceMock filter_factory_; Event::SimulatedTimeSystem time_system_; SlowDateProviderImpl date_provider_{time_system_}; - RouteConfigProvider route_config_provider_; - ScopedRouteConfigProvider scoped_route_config_provider_; + ConnectionManagerImplHelper::RouteConfigProvider route_config_provider_; + ConnectionManagerImplHelper::ScopedRouteConfigProvider scoped_route_config_provider_; std::string server_name_; Stats::IsolatedStoreImpl fake_stats_; ConnectionManagerStats stats_; diff --git a/test/common/http/conn_manager_impl_test.cc b/test/common/http/conn_manager_impl_test.cc index 2b6426b72bf79..e3fb66b08bec2 100644 --- a/test/common/http/conn_manager_impl_test.cc +++ b/test/common/http/conn_manager_impl_test.cc @@ -26,6 +26,7 @@ #include "extensions/access_loggers/file/file_access_log_impl.h" +#include "test/common/http/conn_manager_impl_common.h" #include "test/mocks/access_log/mocks.h" #include "test/mocks/buffer/mocks.h" #include "test/mocks/common.h" @@ -62,34 +63,6 @@ namespace Http { class HttpConnectionManagerImplTest : public TestBase, public ConnectionManagerConfig { public: - struct RouteConfigProvider : public Router::RouteConfigProvider { - RouteConfigProvider(TimeSource& time_source) : time_source_(time_source) {} - - // Router::RouteConfigProvider - Router::ConfigConstSharedPtr config() override { return route_config_; } - absl::optional configInfo() const override { return {}; } - SystemTime lastUpdated() const override { return time_source_.systemTime(); } - - TimeSource& time_source_; - std::shared_ptr route_config_{new NiceMock()}; - }; - - struct ScopedRouteConfigProvider : public Config::ConfigProvider { - ScopedRouteConfigProvider(TimeSource& time_source) - : config_(std::make_shared()), time_source_(time_source) {} - - ~ScopedRouteConfigProvider() override = default; - - // Config::ConfigProvider - SystemTime lastUpdated() const override { return time_source_.systemTime(); } - const Protobuf::Message* getConfigProto() const override { return nullptr; } - std::string getConfigVersion() const override { return ""; } - ConfigConstSharedPtr getConfig() const override { return config_; } - - std::shared_ptr config_; - TimeSource& time_source_; - }; - HttpConnectionManagerImplTest() : route_config_provider_(test_time_.timeSystem()), scoped_route_config_provider_(test_time_.timeSystem()), access_log_path_("dummy_path"), @@ -278,8 +251,8 @@ class HttpConnectionManagerImplTest : public TestBase, public ConnectionManagerC const Http::Http1Settings& http1Settings() const override { return http1_settings_; } DangerousDeprecatedTestTime test_time_; - RouteConfigProvider route_config_provider_; - ScopedRouteConfigProvider scoped_route_config_provider_; + ConnectionManagerImplHelper::RouteConfigProvider route_config_provider_; + ConnectionManagerImplHelper::ScopedRouteConfigProvider scoped_route_config_provider_; NiceMock tracer_; Http::ContextImpl http_context_; NiceMock runtime_; diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index b4d22e85dcbcb..af1bacf03722d 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -109,7 +109,7 @@ TEST_F(ScopedRdsTest, ValidateFail) { ScopedRdsConfigSubscription& subscription = dynamic_cast(*provider_).subscription(); - // 'name' validation + // 'name' validation: value must be > 1 byte. const std::string config_yaml = R"EOF( name: scope_key_builder: @@ -125,7 +125,7 @@ TEST_F(ScopedRdsTest, ValidateFail) { parseScopedRoutesConfigFromYaml(*resources.Add(), config_yaml); EXPECT_THROW(subscription.onConfigUpdate(resources, "1"), ProtoValidationException); - // 'scope_key_builder.fragments' validation + // 'scope_key_builder.fragments' validation: must define at least 1 item in the repeated field. const std::string config_yaml2 = R"EOF( name: foo_scope_set scope_key_builder: @@ -140,7 +140,7 @@ name: foo_scope_set parseScopedRoutesConfigFromYaml(*resources2.Add(), config_yaml2); EXPECT_THROW(subscription.onConfigUpdate(resources2, "1"), ProtoValidationException); - // 'scopes.fragments' validation + // 'scopes.fragments' validation: must define at least 1 item in the repeated field. const std::string config_yaml3 = R"EOF( name: foo_scope_set scope_key_builder: @@ -232,7 +232,7 @@ class ScopedRoutesConfigProviderManagerTest : public ScopedRoutesTestBase { ~ScopedRoutesConfigProviderManagerTest() override = default; }; -// Tests that the /configdump handler returns the corresponding scoped routing config. +// Tests that the /config_dump handler returns the corresponding scoped routing config. TEST_F(ScopedRoutesConfigProviderManagerTest, ConfigDump) { auto message_ptr = factory_context_.admin_.config_tracker_.config_tracker_callbacks_["route_scopes"](); From e5d87db2d9fcf4408a3f74665e78e8ed0f50f1d5 Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Wed, 27 Feb 2019 11:39:09 -0500 Subject: [PATCH 09/37] fix_format. Signed-off-by: Andres Guedez --- api/envoy/api/v2/srds.proto | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/envoy/api/v2/srds.proto b/api/envoy/api/v2/srds.proto index 9032ad6e4ad05..12a6564fec78c 100644 --- a/api/envoy/api/v2/srds.proto +++ b/api/envoy/api/v2/srds.proto @@ -54,16 +54,16 @@ service ScopedRoutesDiscoveryService { // :ref:`HttpConnectionManager` // via the // :ref:`scoped_rds` -// or +// or // :ref:`scoped_routes_config` // fields. When scoped routing is enabled, the Envoy router will first attempt to determine the // routing scope assigned to a request by building a // :ref:`Scope.Key` based on request header -// attributes (such as the value of a header field) defined by the +// attributes (such as the value of a header field) defined by the // :ref:`ScopeKeyBuilder` // this message. // -// The key will be looked up in a table maintained by the router, containing the +// The key will be looked up in a table maintained by the router, containing the // :ref:`Scope.Key` to // :ref:`RouteConfiguration` mappings specified by the // :ref:`scopes` field in this message. The From 9c3d45b6e2b50159c1a5693fc623151a4a65cd76 Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Wed, 27 Feb 2019 13:55:44 -0500 Subject: [PATCH 10/37] Update pedantic spelling dictionary. Signed-off-by: Andres Guedez --- tools/spelling_dictionary.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index 644b900e5ae6d..7205fb657835f 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -730,6 +730,7 @@ verifiers versa versioned vhost +vip vptr vptrs wakeup From afcc2a5d15f06e51aeb374572eb2e56a449e4d72 Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Wed, 27 Feb 2019 14:17:17 -0500 Subject: [PATCH 11/37] Clarify Scope.Key documentation. Signed-off-by: Andres Guedez --- api/envoy/api/v2/srds.proto | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/api/envoy/api/v2/srds.proto b/api/envoy/api/v2/srds.proto index 12a6564fec78c..668caf8a92045 100644 --- a/api/envoy/api/v2/srds.proto +++ b/api/envoy/api/v2/srds.proto @@ -177,12 +177,15 @@ message ScopedRouteConfigurationsSet { // The key construction mechanism. ScopeKeyBuilder scope_key_builder = 2 [(validate.rules).message.required = true]; - // Specifies a routing scope, which associates a :ref:`envoy_api_msg_RouteConfiguration` to a - // :ref:`Key ` which is matched against - // each HTTP request. + // Specifies a routing scope, which associates a :ref:`envoy_api_msg_RouteConfiguration` + // (identified by its resource name) to a + // :ref:`Key` which is matched against each + // HTTP request. message Scope { - // Specifies a key which is matched against the output of a :ref:`ScopeKeyBuilder - // `. + // Specifies a key which is matched against the output of the + // :ref:`scope_key_builder` to + // obtain the route_configuration_name/RouteConfiguration associated with the Scope. The + // matching is dependent on the order of the fragments contained in the Key. message Key { message Fragment { oneof type { From c774f00bc82b15a5ae5993b15bc6d28ec9effc74 Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Wed, 27 Feb 2019 14:47:36 -0500 Subject: [PATCH 12/37] Use EXPECT_NO_THROW() to validate successful allocation/construction. Signed-off-by: Andres Guedez --- .../network/http_connection_manager/config_test.cc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/test/extensions/filters/network/http_connection_manager/config_test.cc b/test/extensions/filters/network/http_connection_manager/config_test.cc index 4d7ed21d6119f..74df165865592 100644 --- a/test/extensions/filters/network/http_connection_manager/config_test.cc +++ b/test/extensions/filters/network/http_connection_manager/config_test.cc @@ -550,9 +550,10 @@ stat_prefix: foo - name: envoy.router )EOF"; - HttpConnectionManagerConfig(parseHttpConnectionManagerFromV2Yaml(config_yaml), context_, - date_provider_, route_config_provider_manager_, - scoped_routes_config_provider_manager_); + EXPECT_NO_THROW(HttpConnectionManagerConfig(parseHttpConnectionManagerFromV2Yaml(config_yaml), + context_, date_provider_, + route_config_provider_manager_, + scoped_routes_config_provider_manager_)); const std::string config_yaml2 = R"EOF( rds: @@ -567,9 +568,10 @@ stat_prefix: foo - name: envoy.router )EOF"; - HttpConnectionManagerConfig(parseHttpConnectionManagerFromV2Yaml(config_yaml2), context_, - date_provider_, route_config_provider_manager_, - scoped_routes_config_provider_manager_); + EXPECT_NO_THROW(HttpConnectionManagerConfig(parseHttpConnectionManagerFromV2Yaml(config_yaml2), + context_, date_provider_, + route_config_provider_manager_, + scoped_routes_config_provider_manager_)); } TEST_F(HttpConnectionManagerConfigTest, ScopedRdsAndStatic) { From 8cc1b13557191e301417038218ace018d1c49cc3 Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Wed, 27 Feb 2019 14:51:27 -0500 Subject: [PATCH 13/37] fix_format. Signed-off-by: Andres Guedez --- .../network/http_connection_manager/config_test.cc | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/test/extensions/filters/network/http_connection_manager/config_test.cc b/test/extensions/filters/network/http_connection_manager/config_test.cc index 74df165865592..4f87c8fc2dce2 100644 --- a/test/extensions/filters/network/http_connection_manager/config_test.cc +++ b/test/extensions/filters/network/http_connection_manager/config_test.cc @@ -550,10 +550,9 @@ stat_prefix: foo - name: envoy.router )EOF"; - EXPECT_NO_THROW(HttpConnectionManagerConfig(parseHttpConnectionManagerFromV2Yaml(config_yaml), - context_, date_provider_, - route_config_provider_manager_, - scoped_routes_config_provider_manager_)); + EXPECT_NO_THROW(HttpConnectionManagerConfig( + parseHttpConnectionManagerFromV2Yaml(config_yaml), context_, date_provider_, + route_config_provider_manager_, scoped_routes_config_provider_manager_)); const std::string config_yaml2 = R"EOF( rds: @@ -568,10 +567,9 @@ stat_prefix: foo - name: envoy.router )EOF"; - EXPECT_NO_THROW(HttpConnectionManagerConfig(parseHttpConnectionManagerFromV2Yaml(config_yaml2), - context_, date_provider_, - route_config_provider_manager_, - scoped_routes_config_provider_manager_)); + EXPECT_NO_THROW(HttpConnectionManagerConfig( + parseHttpConnectionManagerFromV2Yaml(config_yaml2), context_, date_provider_, + route_config_provider_manager_, scoped_routes_config_provider_manager_)); } TEST_F(HttpConnectionManagerConfigTest, ScopedRdsAndStatic) { From 71f2a742a4d8756c19f114300f1130be57c6dc36 Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Fri, 22 Mar 2019 22:19:36 -0400 Subject: [PATCH 14/37] Modify SRDS to support delta updates and ease transition to incremental. - Top level SRDS resource is now called a 'ScopedRouteConfiguration'. - SRDS now operates like CDS where the full set of resources received via the DS builds the complete configuration state. - Modified the ConfigProvider framework to support delta APIs such as SRDS and CDS - Scoped RDS unit test mostly passing, integration tests have not yet been migrated to new API Signed-off-by: Andres Guedez --- api/envoy/admin/v2alpha/config_dump.proto | 20 +- api/envoy/api/v2/srds.proto | 120 ++------- .../v2/http_connection_manager.proto | 122 +++++++-- include/envoy/config/config_provider.h | 33 +++ .../envoy/config/config_provider_manager.h | 13 + source/common/config/config_provider_impl.cc | 35 ++- source/common/config/config_provider_impl.h | 54 +++- source/common/config/utility.h | 37 +++ source/common/router/BUILD | 10 + source/common/router/scoped_config_impl.cc | 15 +- source/common/router/scoped_config_impl.h | 37 ++- source/common/router/scoped_rds.cc | 198 ++++++++++----- source/common/router/scoped_rds.h | 105 +++++--- .../router/thread_local_scoped_config.h | 25 ++ .../network/http_connection_manager/config.cc | 43 +--- source/server/http/admin.h | 2 + .../config/config_provider_impl_test.cc | 12 +- test/common/router/scoped_rds_test.cc | 238 ++++++++++-------- .../http_connection_manager/config_test.cc | 131 ---------- test/mocks/config/mocks.h | 5 + 20 files changed, 727 insertions(+), 528 deletions(-) create mode 100644 source/common/router/thread_local_scoped_config.h diff --git a/api/envoy/admin/v2alpha/config_dump.proto b/api/envoy/admin/v2alpha/config_dump.proto index 24808213f9906..93b4c83f1bb9d 100644 --- a/api/envoy/admin/v2alpha/config_dump.proto +++ b/api/envoy/admin/v2alpha/config_dump.proto @@ -185,31 +185,31 @@ message RoutesConfigDump { // the scopes defined inline with the higher order object (i.e., the HttpConnectionManager) and the // dynamically obtained scopes via the SRDS API. message ScopedRoutesConfigDump { - message InlineScopedRoutesConfig { - // The scoped routes config. - envoy.api.v2.ScopedRouteConfigurationsSet scoped_routes_config = 1; + message InlineScopedRouteConfigs { + // The scoped route configurations. + repeated envoy.api.v2.ScopedRouteConfiguration scoped_route_configs = 1; // The timestamp when the scoped route config set was last updated. google.protobuf.Timestamp last_updated = 2; } - message DynamicScopedRoutesConfig { + message DynamicScopedRouteConfigs { // This is the per-resource version information. This version is currently taken from the // :ref:`version_info ` field at the time that // the scoped routes configuration was loaded. string version_info = 1; - // The scoped routes config. - envoy.api.v2.ScopedRouteConfigurationsSet scoped_routes_config = 2; + // The scoped route configurations. + repeated envoy.api.v2.ScopedRouteConfiguration scoped_route_configs = 2; // The timestamp when the scoped route config set was last updated. google.protobuf.Timestamp last_updated = 3; } - // The statically loaded scoped routes configs. - repeated InlineScopedRoutesConfig inline_scoped_routes_configs = 1 [(gogoproto.nullable) = false]; + // The statically loaded scoped route configs. + repeated InlineScopedRouteConfigs inline_scoped_route_configs = 1 [(gogoproto.nullable) = false]; - // The dynamically loaded scoped routes configs. - repeated DynamicScopedRoutesConfig dynamic_scoped_routes_configs = 2 + // The dynamically loaded scoped route configs. + repeated DynamicScopedRouteConfigs dynamic_scoped_route_configs = 2 [(gogoproto.nullable) = false]; } diff --git a/api/envoy/api/v2/srds.proto b/api/envoy/api/v2/srds.proto index 668caf8a92045..8dcd086b3fcae 100644 --- a/api/envoy/api/v2/srds.proto +++ b/api/envoy/api/v2/srds.proto @@ -104,111 +104,39 @@ service ScopedRoutesDiscoveryService { // which Envoy would fetch via RDS. // // [#comment:next free field: 4] -message ScopedRouteConfigurationsSet { - // The name of the set of route configuration scopes. This will be the - // :ref:`scoped_routes_config_set_name - // ` - // set in :ref:`envoy_api_msg_config.filter.network.http_connection_manager.v2.ScopedRds`. - string name = 1 [(validate.rules).string.min_bytes = 1]; - // Specifies the mechanism for constructing keys based on request attributes to match - // :ref:`scopes ` against. - // - // Upon receiving a request's headers, the Router will build a key using the algorithm specified - // by this message. This key will be used to look up a corresponding - // :ref:`Scope ` in a table containing the set - // of :ref:`scopes ` in this message. - message ScopeKeyBuilder { - // Specifies the mechanism for constructing fragments which are composed into keys. - message FragmentBuilder { - // Specifies how the value of a header should be extracted. - // The following example maps the structure of a header to the fields in this message. - // - // .. code:: - // - // X-Header: a=b;c=d - // | || | - // | || \----> - // | || - // | |\----> - // | | - // | \----> - // | - // \----> - // - // Each 'a=b' key-value pair constitutes an 'element' of the header field. - message HeaderValueExtractor { - // The name of the header field to extract the value from. - string name = 1 [(validate.rules).string.min_bytes = 1]; - - // The element separator (e.g., ';' separates 'a;b;c;d'). - string element_separator = 2; - - // Specifies a header field's key value pair to match on. - message KvElement { - // The separator between key and value (e.g., '=' separates 'k=v;...'). - string separator = 1; - - // The key to match on. - string key = 2; - } - - oneof extract_type { - // Specifies the index of the element to extract. - int32 index = 3; - - // Specifies the key value pair to extract the value from. - KvElement element = 4; - } - } +// TODO(AndresGuedez): +// Specifies a routing scope, which associates a :ref:`envoy_api_msg_RouteConfiguration` +// (identified by its resource name) to a +// :ref:`Key` which is matched against each +// HTTP request. +message ScopedRouteConfiguration { + // TODO(AndresGuedez): + // The name assigned to the routing scope. + string name = 1 [(validate.rules).string.min_bytes = 1]; + // Specifies a key which is matched against the output of the + // :ref:`scope_key_builder` to + // obtain the route_configuration_name/RouteConfiguration associated with the Scope. The + // matching is dependent on the order of the fragments contained in the Key. + message Key { + message Fragment { oneof type { option (validate.required) = true; - // Specifies how a header field's value should be extracted. - HeaderValueExtractor header_value_extractor = 1; + // A string to match against. + string string_key = 1; } } - // The constructed key consists of the ordered union of these fragments. - repeated FragmentBuilder fragments = 1 [(validate.rules).repeated .min_items = 1]; + // The ordered set of fragments to match against. + repeated Fragment fragments = 1 [(validate.rules).repeated .min_items = 1]; } - // The key construction mechanism. - ScopeKeyBuilder scope_key_builder = 2 [(validate.rules).message.required = true]; - - // Specifies a routing scope, which associates a :ref:`envoy_api_msg_RouteConfiguration` - // (identified by its resource name) to a - // :ref:`Key` which is matched against each - // HTTP request. - message Scope { - // Specifies a key which is matched against the output of the - // :ref:`scope_key_builder` to - // obtain the route_configuration_name/RouteConfiguration associated with the Scope. The - // matching is dependent on the order of the fragments contained in the Key. - message Key { - message Fragment { - oneof type { - option (validate.required) = true; - - // A string to match against. - string string_key = 1; - } - } - - // The ordered set of fragments to match against. - repeated Fragment fragments = 1 [(validate.rules).repeated .min_items = 1]; - } - - // The resource name to use for a :ref:`envoy_api_msg_DiscoveryRequest` to an RDS server to - // fetch the :ref:`envoy_api_msg_RouteConfiguration` associated with this scope. - string route_configuration_name = 1 [(validate.rules).string.min_bytes = 1]; - - // The key to match against. - Key key = 2 [(validate.rules).message.required = true]; - } + // The resource name to use for a :ref:`envoy_api_msg_DiscoveryRequest` to an RDS server to + // fetch the :ref:`envoy_api_msg_RouteConfiguration` associated with this scope. + string route_configuration_name = 2 [(validate.rules).string.min_bytes = 1]; - // The set of scopes, containing :ref:`Key ` - // to :ref:`envoy_api_msg_RouteConfiguration` mappings. - repeated Scope scopes = 3 [(validate.rules).repeated .min_items = 1]; + // The key to match against. + Key key = 3 [(validate.rules).message.required = true]; } diff --git a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto index 7350dc2dd73c2..5409cefb45a00 100644 --- a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto +++ b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto @@ -62,14 +62,11 @@ message HttpConnectionManager { // The route table for the connection manager is static and is specified in this property. envoy.api.v2.RouteConfiguration route_config = 4; - } - - oneof scoped_routes_specifier { - // Configuration for the Scoped RDS API which dynamically loads routing configuration scopes. - ScopedRds scoped_rds = 30; - // A static set of routing scopes. - envoy.api.v2.ScopedRouteConfigurationsSet scoped_routes_config = 31; + // A route table will be dynamically assigned to each request based on request attributes + // (e.g., the value of a header). The "routing scopes" (i.e., route tables) and scope keys are + // specified in this message. + ScopedRoutes scoped_routes = 30; } // A list of individual HTTP filters that make up the filter chain for @@ -398,34 +395,115 @@ message Rds { envoy.api.v2.core.ConfigSource config_source = 1 [(validate.rules).message.required = true, (gogoproto.nullable) = false]; - oneof subscription_specifier { - option (validate.required) = true; - // The name of the route configuration. This name will be passed to the RDS // API. This allows an Envoy configuration with multiple HTTP listeners (and // associated HTTP connection manager filters) to use different route // configurations. string route_config_name = 2 [(validate.rules).string.min_bytes = 1]; +} + +// This message is used to work around the limitations with 'oneof' and repeated fields. +message ScopedRouteConfigurationsList { + repeated envoy.api.v2.ScopedRouteConfiguration scoped_route_configurations = 1 + [(validate.rules).repeated .min_items = 1]; +} + +message ScopedRoutes { + // TODO(AndresGuedez): + // Specifies the mechanism for constructing keys based on request attributes to match + // :ref:`scopes ` against. + // + // Upon receiving a request's headers, the Router will build a key using the algorithm specified + // by this message. This key will be used to look up a corresponding + // :ref:`Scope ` in a table containing the set + // of :ref:`scopes ` in this message. + message ScopeKeyBuilder { + // Specifies the mechanism for constructing fragments which are composed into keys. + message FragmentBuilder { + // Specifies how the value of a header should be extracted. + // The following example maps the structure of a header to the fields in this message. + // + // .. code:: + // + // X-Header: a=b;c=d + // | || | + // | || \----> + // | || + // | |\----> + // | | + // | \----> + // | + // \----> + // + // Each 'a=b' key-value pair constitutes an 'element' of the header field. + message HeaderValueExtractor { + // The name of the header field to extract the value from. + string name = 1 [(validate.rules).string.min_bytes = 1]; + + // The element separator (e.g., ';' separates 'a;b;c;d'). + string element_separator = 2; + + // Specifies a header field's key value pair to match on. + message KvElement { + // The separator between key and value (e.g., '=' separates 'k=v;...'). + string separator = 1; + + // The key to match on. + string key = 2; + } + + oneof extract_type { + // Specifies the index of the element to extract. + int32 index = 3; + + // Specifies the key value pair to extract the value from. + KvElement element = 4; + } + } + + oneof type { + option (validate.required) = true; + + // Specifies how a header field's value should be extracted. + HeaderValueExtractor header_value_extractor = 1; + } + } - // Must be set to true when scoped RDS is enabled. - // If this is enabled, RDS subscriptions will be triggered by scoped RDS config updates using - // this configuration as a template; i.e., the - // :ref:`config_source` - // from this message will be used to specify the config source for subscriptions initiated by - // scoped RDS. - bool scoped_rds_template = 3; + // The constructed key consists of the ordered union of these fragments. + repeated FragmentBuilder fragments = 1 [(validate.rules).repeated .min_items = 1]; + } + + ScopeKeyBuilder scope_key_builder = 1 [(validate.rules).message.required = true]; + + // Configuration source specifier for RDS. + // This config source is used to subscribe to RouteConfiguration resources specified in + // ScopedRouteConfiguration messages. + envoy.api.v2.core.ConfigSource rds_config_source = 2 + [(validate.rules).message.required = true, (gogoproto.nullable) = false]; + + oneof config_specifier { + option (validate.required) = true; + + // The set of routing scopes corresponding to the HCM. A scope is assigned to a request by + // matching a key constructed from the request's attributes according to the algorithm specified + // by the :ref:`ScopeKeyBuilder` in this message. + ScopedRouteConfigurationsList scoped_route_configurations_list = 3; + + // The connection manager's route table will be dynamically loaded via the SRDS API; the + // :ref:`` received via + // SRDS will specify a set of routing scopes, each containing . + ScopedRds scoped_rds = 4; } } message ScopedRds { // Configuration source specifier for scoped RDS. - envoy.api.v2.core.ConfigSource config_source = 1 + envoy.api.v2.core.ConfigSource scoped_rds_config_source = 1 [(validate.rules).message.required = true, (gogoproto.nullable) = false]; - // The name of the set of routing configuration scopes. This name will be passed to the scoped RDS - // API. - // This allows Envoy to segment routing configuration based on a configurable request attribute. - string scoped_routes_config_set_name = 2 [(validate.rules).string.min_bytes = 1]; + // This is a hint to the xDS server containing the list of resources that Envoy is interested in + // tracking updates for. + // repeated string scoped_route_config_names = 3; } message HttpFilter { diff --git a/include/envoy/config/config_provider.h b/include/envoy/config/config_provider.h index a42f512dea4e9..9e404dd9b7868 100644 --- a/include/envoy/config/config_provider.h +++ b/include/envoy/config/config_provider.h @@ -41,6 +41,11 @@ class ConfigProvider { }; using ConfigConstSharedPtr = std::shared_ptr; + /** + * + */ + enum class ApiType { Full, Delta }; + /** * Stores the config proto as well as the associated version. */ @@ -51,8 +56,23 @@ class ConfigProvider { std::string version_; }; + /** + * + */ + struct ConfigProtoInfoVec { + const std::vector config_protos_; + + // Only populated by dynamic config providers. + std::string version_; + }; + virtual ~ConfigProvider() = default; + /** + * + */ + virtual ApiType apiType() const PURE; + /** * Returns a ConfigProtoInfo associated with the provider. * @return absl::optional> an optional ConfigProtoInfo; the value is set when a @@ -69,6 +89,14 @@ class ConfigProvider { return ConfigProtoInfo

{*config_proto, getConfigVersion()}; } + absl::optional configProtoInfoVec() const { + std::vector config_protos = getConfigProtos(); + if (config_protos.empty()) { + return absl::nullopt; + } + return ConfigProtoInfoVec{std::move(config_protos), getConfigVersion()}; + } + /** * Returns the Config corresponding to the provider. * @return std::shared_ptr a shared pointer to the Config. @@ -94,6 +122,11 @@ class ConfigProvider { */ virtual const Protobuf::Message* getConfigProto() const PURE; + /** + * + */ + virtual const std::vector getConfigProtos() const PURE; + /** * Returns the config version associated with the provider. * @return std::string the config version. diff --git a/include/envoy/config/config_provider_manager.h b/include/envoy/config/config_provider_manager.h index 0202b85b29dbc..48aaaea5516dd 100644 --- a/include/envoy/config/config_provider_manager.h +++ b/include/envoy/config/config_provider_manager.h @@ -66,6 +66,19 @@ class ConfigProviderManager { createStaticConfigProvider(const Protobuf::Message& config_proto, Server::Configuration::FactoryContext& factory_context, const OptionalArg& optarg) PURE; + + /** + * Returns a ConfigProvider associated with a statically specified configuration. This is intended + * to be used when a set of configuration protos is required to build the full configuration. + * @param config_protos supplies a vector of configuration protos. + * @param factory_context is the context to use for the provider. + * @param optarg supplies an optional argument with data specific to the concrete class. + * @return ConfigProviderPtr a newly allocated static config provider. + */ + virtual ConfigProviderPtr + createStaticConfigProvider(std::vector>&& config_protos, + Server::Configuration::FactoryContext& factory_context, + const OptionalArg& optarg) PURE; }; } // namespace Config diff --git a/source/common/config/config_provider_impl.cc b/source/common/config/config_provider_impl.cc index 541c767412aac..6baa89ee7ef7d 100644 --- a/source/common/config/config_provider_impl.cc +++ b/source/common/config/config_provider_impl.cc @@ -5,9 +5,11 @@ namespace Config { ImmutableConfigProviderImplBase::ImmutableConfigProviderImplBase( Server::Configuration::FactoryContext& factory_context, - ConfigProviderManagerImplBase& config_provider_manager, ConfigProviderInstanceType type) + ConfigProviderManagerImplBase& config_provider_manager, + ConfigProviderInstanceType instance_type, ApiType api_type) : last_updated_(factory_context.timeSource().systemTime()), - config_provider_manager_(config_provider_manager), type_(type) { + config_provider_manager_(config_provider_manager), instance_type_(instance_type), + api_type_(api_type) { config_provider_manager_.bindImmutableConfigProvider(this); } @@ -31,8 +33,11 @@ bool ConfigSubscriptionInstanceBase::checkAndApplyConfig(const Protobuf::Message const std::string& config_name, const std::string& version_info) { const uint64_t new_hash = MessageUtil::hash(config_proto); - if (config_info_ && config_info_.value().last_config_hash_ == new_hash) { - return false; + if (config_info_) { + ASSERT(config_info_.value().last_config_hash_.has_value()); + if (config_info_.value().last_config_hash_.value() == new_hash) { + return false; + } } config_info_ = {new_hash, version_info}; @@ -57,6 +62,14 @@ bool ConfigSubscriptionInstanceBase::checkAndApplyConfig(const Protobuf::Message return true; } +void ConfigSubscriptionInstanceBase::propagateDeltaConfigUpdate( + std::function updateFn) { + for (auto* provider : mutable_config_providers_) { + ConfigProvider::ConfigConstSharedPtr config = provider->getConfig(); + provider->onDeltaConfigUpdate([config, updateFn]() { updateFn(config); }); + } +} + void ConfigSubscriptionInstanceBase::bindConfigProvider(MutableConfigProviderImplBase* provider) { // All config providers bound to a ConfigSubscriptionInstanceBase must be of the same concrete // type; this is assumed by checkAndApplyConfig() and is verified by the assertion below. @@ -95,13 +108,13 @@ ConfigProviderManagerImplBase::immutableConfigProviders(ConfigProviderInstanceTy void ConfigProviderManagerImplBase::bindImmutableConfigProvider( ImmutableConfigProviderImplBase* provider) { - ASSERT(provider->type() == ConfigProviderInstanceType::Static || - provider->type() == ConfigProviderInstanceType::Inline); + ASSERT(provider->instanceType() == ConfigProviderInstanceType::Static || + provider->instanceType() == ConfigProviderInstanceType::Inline); ConfigProviderMap::iterator it; - if ((it = immutable_config_providers_map_.find(provider->type())) == + if ((it = immutable_config_providers_map_.find(provider->instanceType())) == immutable_config_providers_map_.end()) { immutable_config_providers_map_.insert(std::make_pair( - provider->type(), + provider->instanceType(), std::make_unique(std::initializer_list({provider})))); } else { it->second->insert(provider); @@ -110,9 +123,9 @@ void ConfigProviderManagerImplBase::bindImmutableConfigProvider( void ConfigProviderManagerImplBase::unbindImmutableConfigProvider( ImmutableConfigProviderImplBase* provider) { - ASSERT(provider->type() == ConfigProviderInstanceType::Static || - provider->type() == ConfigProviderInstanceType::Inline); - auto it = immutable_config_providers_map_.find(provider->type()); + ASSERT(provider->instanceType() == ConfigProviderInstanceType::Static || + provider->instanceType() == ConfigProviderInstanceType::Inline); + auto it = immutable_config_providers_map_.find(provider->instanceType()); ASSERT(it != immutable_config_providers_map_.end()); it->second->erase(provider); } diff --git a/source/common/config/config_provider_impl.h b/source/common/config/config_provider_impl.h index 338f0e3ec07a2..5c1422b2ac46f 100644 --- a/source/common/config/config_provider_impl.h +++ b/source/common/config/config_provider_impl.h @@ -100,18 +100,20 @@ class ImmutableConfigProviderImplBase : public ConfigProvider { // Envoy::Config::ConfigProvider SystemTime lastUpdated() const override { return last_updated_; } + ApiType apiType() const override { return api_type_; } - ConfigProviderInstanceType type() const { return type_; } + ConfigProviderInstanceType instanceType() const { return instance_type_; } protected: ImmutableConfigProviderImplBase(Server::Configuration::FactoryContext& factory_context, ConfigProviderManagerImplBase& config_provider_manager, - ConfigProviderInstanceType type); + ConfigProviderInstanceType instance_type, ApiType api_type); private: SystemTime last_updated_; ConfigProviderManagerImplBase& config_provider_manager_; - ConfigProviderInstanceType type_; + ConfigProviderInstanceType instance_type_; + ApiType api_type_; }; class MutableConfigProviderImplBase; @@ -137,7 +139,7 @@ class ConfigSubscriptionInstanceBase : public Init::Target, protected Logger::Loggable { public: struct LastConfigInfo { - uint64_t last_config_hash_; + absl::optional last_config_hash_; std::string last_config_version_; }; @@ -211,8 +213,18 @@ class ConfigSubscriptionInstanceBase : public Init::Target, void setLastUpdated() { last_updated_ = time_source_.systemTime(); } + void setLastConfigInfo(absl::optional&& config_info) { + config_info_ = std::move(config_info); + } + void runInitializeCallbackIfAny(); + /** + * TODO(AndresGuedez): + */ + void + propagateDeltaConfigUpdate(std::function updateFn); + private: void registerInitTarget(Init::Manager& init_manager) { init_manager.registerTarget(*this); } @@ -257,11 +269,10 @@ class MutableConfigProviderImplBase : public ConfigProvider { // Envoy::Config::ConfigProvider SystemTime lastUpdated() const override { return subscription_->lastUpdated(); } - - // Envoy::Config::ConfigProvider ConfigConstSharedPtr getConfig() const override { return tls_->getTyped().config_; } + ApiType apiType() const override { return api_type_; } /** * Called when a new config proto is received via an xDS subscription. @@ -271,6 +282,7 @@ class MutableConfigProviderImplBase : public ConfigProvider { * Note that this function is called _once_ across all shared config providers per xDS * subscription config update. * @param config_proto supplies the configuration proto. + * @param version_info supplies the version associated with the config. * @return ConfigConstSharedPtr the ConfigProvider::Config to share with other providers. */ virtual ConfigConstSharedPtr onConfigProtoUpdate(const Protobuf::Message& config_proto) PURE; @@ -280,28 +292,49 @@ class MutableConfigProviderImplBase : public ConfigProvider { * @param initial_config supplies an initial Envoy::Config::ConfigProvider::Config associated with * the underlying subscription. */ - void initialize(const ConfigConstSharedPtr& initial_config) { + virtual void initialize(const ConfigConstSharedPtr& initial_config) { subscription_->bindConfigProvider(this); tls_->set([initial_config](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { return std::make_shared(initial_config); }); } + /** + * TODO(AndresGuedez): + */ + virtual void initialize(ThreadLocal::Slot::InitializeCb initializeCb) { + subscription_->bindConfigProvider(this); + tls_->set(initializeCb); + } + /** * Propagates a newly instantiated Envoy::Config::ConfigProvider::Config to all workers. * @param config supplies the newly instantiated config. */ - void onConfigUpdate(const ConfigConstSharedPtr& config) { + virtual void onConfigUpdate(const ConfigConstSharedPtr& config) { + if (getConfig() == config) { + return; + } tls_->runOnAllThreads( [this, config]() -> void { tls_->getTyped().config_ = config; }); } + /** + * TODO(AndresGuedez): + */ + virtual void onDeltaConfigUpdate(Envoy::Event::PostCb updateCb) { + tls_->runOnAllThreads(updateCb); + } + protected: MutableConfigProviderImplBase(ConfigSubscriptionInstanceBaseSharedPtr&& subscription, - Server::Configuration::FactoryContext& factory_context) - : subscription_(subscription), tls_(factory_context.threadLocal().allocateSlot()) {} + Server::Configuration::FactoryContext& factory_context, + ApiType api_type) + : subscription_(subscription), tls_(factory_context.threadLocal().allocateSlot()), + api_type_(api_type) {} const ConfigSubscriptionInstanceBaseSharedPtr& subscription() const { return subscription_; } + ThreadLocal::Slot* tls() const { return tls_.get(); } private: struct ThreadLocalConfig : public ThreadLocal::ThreadLocalObject { @@ -313,6 +346,7 @@ class MutableConfigProviderImplBase : public ConfigProvider { ConfigSubscriptionInstanceBaseSharedPtr subscription_; ThreadLocal::SlotPtr tls_; + ApiType api_type_; }; /** diff --git a/source/common/config/utility.h b/source/common/config/utility.h index 60c36db3197cc..ce6f07b7faac4 100644 --- a/source/common/config/utility.h +++ b/source/common/config/utility.h @@ -1,5 +1,7 @@ #pragma once +#include + #include "envoy/api/api.h" #include "envoy/api/v2/core/base.pb.h" #include "envoy/config/bootstrap/v2/bootstrap.pb.h" @@ -321,6 +323,41 @@ class Utility { static void translateOpaqueConfig(const ProtobufWkt::Any& typed_config, const ProtobufWkt::Struct& config, Protobuf::Message& out_proto); + +#if 0 + template + static bool diffResourcesAndUpdateConfig( + const std::string& api_name, const ResourceVector& resources, const std::string& version_info, + std::function addOrUpdateFn, + std::function removeFn, + std::function validateFn) { + std::unordered_set resource_names; + for (const auto& resource : resources) { + if (!resource_names.insert(resource.name()).second) { + throw EnvoyException(fmt::format("duplicate resource {} found", resource.name())); + } + } + for (const auto& resource : resources) { + validateFn(resource); + } + // We need to keep track of which clusters we might need to remove. + ClusterManager::ClusterInfoMap clusters_to_remove = cm_.clusters(); + for (auto& cluster : resources) { + const std::string cluster_name = cluster.name(); + clusters_to_remove.erase(cluster_name); + if (cm_.addOrUpdateCluster(cluster, version_info)) { + ENVOY_LOG(debug, "cds: add/update cluster '{}'", cluster_name); + } + } + + for (auto cluster : clusters_to_remove) { + const std::string cluster_name = cluster.first; + if (cm_.removeCluster(cluster_name)) { + ENVOY_LOG(debug, "cds: remove cluster '{}'", cluster_name); + } + } + } +#endif }; } // namespace Config diff --git a/source/common/router/BUILD b/source/common/router/BUILD index f05d402f76496..55e7f104493aa 100644 --- a/source/common/router/BUILD +++ b/source/common/router/BUILD @@ -93,14 +93,24 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "thread_local_scoped_config_lib", + hdrs = ["thread_local_scoped_config.h"], + deps = [ + "//include/envoy/thread_local:thread_local_interface", + ], +) + envoy_cc_library( name = "scoped_config_lib", srcs = ["scoped_config_impl.cc"], hdrs = ["scoped_config_impl.h"], deps = [ ":config_lib", + ":thread_local_scoped_config_lib", "//include/envoy/router:scopes_interface", "@envoy_api//envoy/api/v2:srds_cc", + "@envoy_api//envoy/config/filter/network/http_connection_manager/v2:http_connection_manager_cc", ], ) diff --git a/source/common/router/scoped_config_impl.cc b/source/common/router/scoped_config_impl.cc index 3b1dda6160da3..ba5b1736e0b57 100644 --- a/source/common/router/scoped_config_impl.cc +++ b/source/common/router/scoped_config_impl.cc @@ -5,7 +5,20 @@ namespace Envoy { namespace Router { -Router::ConfigConstSharedPtr ScopedConfigImpl::getRouterConfig(const Http::HeaderMap&) const { +ScopedRouteInfoConstSharedPtr +ScopedConfigManager::addOrUpdateRoutingScope(const envoy::api::v2::ScopedRouteConfiguration&, + const std::string&) { + return std::make_shared(); +} + +bool ScopedConfigManager::removeRoutingScope(const std::string&) { return true; } + +void ThreadLocalScopedConfigImpl::addOrUpdateRoutingScope(ScopedRouteInfoConstSharedPtr) {} + +void ThreadLocalScopedConfigImpl::removeRoutingScope(const std::string&) {} + +Router::ConfigConstSharedPtr +ThreadLocalScopedConfigImpl::getRouterConfig(const Http::HeaderMap&) const { return std::make_shared(); } diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h index 603be142b06ed..e003e501172a0 100644 --- a/source/common/router/scoped_config_impl.h +++ b/source/common/router/scoped_config_impl.h @@ -1,14 +1,36 @@ #pragma once #include "envoy/api/v2/srds.pb.h" +#include "envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.pb.h" #include "envoy/router/router.h" #include "envoy/router/scopes.h" #include "common/router/config_impl.h" +#include "common/router/thread_local_scoped_config.h" namespace Envoy { namespace Router { +// TODO(AndresGuedez): +class ScopedConfigManager { +public: + using ScopedRouteMap = std::map; + + // Adds/updates a routing scope specified via the Scoped RDS API. This scope will be added to the + // set of scopes matched against the scope keys built for each HTTP request. + ScopedRouteInfoConstSharedPtr + addOrUpdateRoutingScope(const envoy::api::v2::ScopedRouteConfiguration& scoped_route_config, + const std::string& version_info); + + // Removes a routing scope from the set of scopes matched against each HTTP request. + bool removeRoutingScope(const std::string& scope_name); + + const ScopedRouteMap& scopedRouteMap() const { return scoped_route_map_; } + +private: + ScopedRouteMap scoped_route_map_; +}; + /** * The implementation of scoped routing configuration logic. * NOTE: This is not yet implemented and simply mimics the behavior of the @@ -16,15 +38,22 @@ namespace Router { * * TODO(AndresGuedez): implement scoped routing logic. */ -class ScopedConfigImpl : public ScopedConfig { +class ThreadLocalScopedConfigImpl : public ScopedConfig, public ThreadLocalScopedConfig { public: - ScopedConfigImpl(const envoy::api::v2::ScopedRouteConfigurationsSet& config_proto) - : name_(config_proto.name()) {} + ThreadLocalScopedConfigImpl(const envoy::config::filter::network::http_connection_manager::v2:: + ScopedRoutes::ScopeKeyBuilder& scope_key_builder) + : scope_key_builder_(scope_key_builder) {} + + // Envoy::Router::ThreadLocalScopedConfig + void addOrUpdateRoutingScope(ScopedRouteInfoConstSharedPtr scoped_route_info) override; + void removeRoutingScope(const std::string& scope_name) override; + // Envoy::Router::ScopedConfig Router::ConfigConstSharedPtr getRouterConfig(const Http::HeaderMap& headers) const override; private: - const std::string name_; + const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder + scope_key_builder_; }; /** diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 5bf8ff8ce3300..67397a710b615 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -7,7 +7,6 @@ #include "common/common/assert.h" #include "common/config/subscription_factory.h" -#include "common/router/scoped_config_impl.h" namespace Envoy { namespace Router { @@ -17,21 +16,40 @@ Envoy::Config::ConfigProviderPtr ScopedRoutesConfigProviderUtil::maybeCreate( config, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, Envoy::Config::ConfigProviderManager& scoped_routes_config_provider_manager) { - switch (config.scoped_routes_specifier_case()) { - case envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager:: - kScopedRoutesConfig: + if (config.route_specifier_case() != envoy::config::filter::network::http_connection_manager::v2:: + HttpConnectionManager::kScopedRoutes) { + return nullptr; + } + + switch (config.scoped_routes().config_specifier_case()) { + case envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: + kScopedRouteConfigurationsList: { + const envoy::config::filter::network::http_connection_manager::v2:: + ScopedRouteConfigurationsList& scoped_route_list = + config.scoped_routes().scoped_route_configurations_list(); + std::vector> config_protos( + scoped_route_list.scoped_route_configurations().size()); + for (auto it = scoped_route_list.scoped_route_configurations().begin(); + it != scoped_route_list.scoped_route_configurations().end(); ++it) { + Protobuf::Message* clone = (*it).New(); + clone->CopyFrom(*it); + config_protos.push_back(std::unique_ptr(clone)); + } + return scoped_routes_config_provider_manager.createStaticConfigProvider( - config.scoped_routes_config(), factory_context, - ScopedRoutesConfigProviderManagerOptArg(config.rds())); + std::move(config_protos), factory_context, + ScopedRoutesConfigProviderManagerOptArg(config.scoped_routes().rds_config_source(), + config.scoped_routes().scope_key_builder())); + } - case envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager:: - kScopedRds: + case envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::kScopedRds: return scoped_routes_config_provider_manager.createXdsConfigProvider( - config.scoped_rds(), factory_context, stat_prefix, - ScopedRoutesConfigProviderManagerOptArg(config.rds())); + config.scoped_routes().scoped_rds(), factory_context, stat_prefix, + ScopedRoutesConfigProviderManagerOptArg(config.scoped_routes().rds_config_source(), + config.scoped_routes().scope_key_builder())); - case envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager:: - SCOPED_ROUTES_SPECIFIER_NOT_SET: + case envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: + CONFIG_SPECIFIER_NOT_SET: return nullptr; default: @@ -40,14 +58,20 @@ Envoy::Config::ConfigProviderPtr ScopedRoutesConfigProviderUtil::maybeCreate( } InlineScopedRoutesConfigProvider::InlineScopedRoutesConfigProvider( - const envoy::api::v2::ScopedRouteConfigurationsSet& config_proto, + std::vector>&& config_protos, Server::Configuration::FactoryContext& factory_context, ScopedRoutesConfigProviderManager& config_provider_manager, - const envoy::config::filter::network::http_connection_manager::v2::Rds& rds) + const envoy::api::v2::core::ConfigSource& rds_config_source, + const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: + ScopeKeyBuilder& scope_key_builder) : Envoy::Config::ImmutableConfigProviderImplBase( factory_context, config_provider_manager, - Envoy::Config::ConfigProviderInstanceType::Inline), - config_(std::make_shared()), config_proto_(config_proto), rds_(rds) {} + Envoy::Config::ConfigProviderInstanceType::Inline, + Envoy::Config::ConfigProvider::ApiType::Delta), + config_(std::make_shared(scope_key_builder)), + config_protos_(std::make_move_iterator(config_protos.begin()), + std::make_move_iterator(config_protos.end())), + rds_config_source_(rds_config_source) {} ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( const envoy::config::filter::network::http_connection_manager::v2::ScopedRds& scoped_rds, @@ -56,64 +80,98 @@ ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( : ConfigSubscriptionInstanceBase( "SRDS", manager_identifier, config_provider_manager, factory_context.timeSource(), factory_context.timeSource().systemTime(), factory_context.localInfo()), - scoped_routes_config_name_(scoped_rds.scoped_routes_config_set_name()), - scope_(factory_context.scope().createScope(stat_prefix + "scoped_rds." + - scoped_routes_config_name_ + ".")), + scope_(factory_context.scope().createScope(stat_prefix + "scoped_rds.")), stats_({ALL_SCOPED_RDS_STATS(POOL_COUNTER(*scope_))}) { subscription_ = Envoy::Config::SubscriptionFactory::subscriptionFromConfigSource< - envoy::api::v2::ScopedRouteConfigurationsSet>( - scoped_rds.config_source(), factory_context.localInfo(), factory_context.dispatcher(), - factory_context.clusterManager(), factory_context.random(), *scope_, - "envoy.api.v2.ScopedRoutesDiscoveryService.FetchScopedRoutes", + envoy::api::v2::ScopedRouteConfiguration>( + scoped_rds.scoped_rds_config_source(), factory_context.localInfo(), + factory_context.dispatcher(), factory_context.clusterManager(), factory_context.random(), + *scope_, "envoy.api.v2.ScopedRoutesDiscoveryService.FetchScopedRoutes", "envoy.api.v2.ScopedRoutesDiscoveryService.StreamScopedRoutes", factory_context.api()); } void ScopedRdsConfigSubscription::onConfigUpdate(const ResourceVector& resources, const std::string& version_info) { if (resources.empty()) { - ENVOY_LOG(debug, "Missing ScopedRouteConfigurationsSet for {} in onConfigUpdate()", - scoped_routes_config_name_); + ENVOY_LOG(debug, "Empty resources in scoped RDS onConfigUpdate()"); stats_.update_empty_.inc(); ConfigSubscriptionInstanceBase::onConfigUpdateFailed(); return; } - if (resources.size() != 1) { - throw EnvoyException(fmt::format("Unexpected SRDS resource length: {}", resources.size())); + std::unordered_set resource_names; + for (const auto& scoped_route : resources) { + if (!resource_names.insert(scoped_route.name()).second) { + throw EnvoyException( + fmt::format("duplicate scoped route configuration {} found", scoped_route.name())); + } + } + for (const auto& scoped_route : resources) { + MessageUtil::validate(scoped_route); } - const auto& scoped_routes_config = resources[0]; - MessageUtil::validate(scoped_routes_config); - // The name we receive in the config must match the name requested. - if (scoped_routes_config.name() != scoped_routes_config_name_) { - throw EnvoyException(fmt::format("Unexpected SRDS configuration (expecting {}): {}", - scoped_routes_config_name_, scoped_routes_config.name())); + std::vector exception_msgs; + // We need to keep track of which scoped routes we might need to remove. + ScopedConfigManager::ScopedRouteMap scoped_routes_to_remove = + scoped_config_manager_.scopedRouteMap(); + for (auto& scoped_route : resources) { + const std::string scoped_route_name = scoped_route.name(); + try { + scoped_routes_to_remove.erase(scoped_route_name); + ScopedRouteInfoConstSharedPtr scoped_route_info = + scoped_config_manager_.addOrUpdateRoutingScope(scoped_route, version_info); + if (scoped_route_info == nullptr) { + throw EnvoyException( + fmt::format("failed to create/update global routing scope {}", scoped_route_name)); + } + ENVOY_LOG(debug, "srds: add/update scoped_route '{}'", scoped_route_name); + propagateDeltaConfigUpdate( + [scoped_route_info](Envoy::Config::ConfigProvider::ConfigConstSharedPtr config) { + ThreadLocalScopedConfigImpl* thread_local_scoped_config = + const_cast( + static_cast(config.get())); + thread_local_scoped_config->addOrUpdateRoutingScope(scoped_route_info); + }); + } catch (const EnvoyException& ex) { + exception_msgs.push_back(fmt::format("{}: {}", scoped_route_name, ex.what())); + } } - if (checkAndApplyConfig(scoped_routes_config, scoped_routes_config_name_, version_info)) { - scoped_routes_proto_ = scoped_routes_config; - stats_.config_reload_.inc(); + for (auto scoped_route : scoped_routes_to_remove) { + const std::string scoped_route_name = scoped_route.first; + ENVOY_LOG(debug, "srds: remove scoped route '{}'", scoped_route_name); + propagateDeltaConfigUpdate( + [scoped_route_name](Envoy::Config::ConfigProvider::ConfigConstSharedPtr config) { + ThreadLocalScopedConfigImpl* thread_local_scoped_config = + const_cast( + static_cast(config.get())); + thread_local_scoped_config->removeRoutingScope(scoped_route_name); + }); } ConfigSubscriptionInstanceBase::onConfigUpdate(); + setLastConfigInfo(absl::optional({absl::nullopt, version_info})); + if (!exception_msgs.empty()) { + throw EnvoyException(fmt::format("Error adding/updating scoped route(s) {}", + StringUtil::join(exception_msgs, ", "))); + } + stats_.config_reload_.inc(); } ScopedRdsConfigProvider::ScopedRdsConfigProvider( ScopedRdsConfigSubscriptionSharedPtr&& subscription, - Envoy::Config::ConfigProvider::ConfigConstSharedPtr initial_config, Server::Configuration::FactoryContext& factory_context, - const envoy::config::filter::network::http_connection_manager::v2::Rds& rds) - : MutableConfigProviderImplBase(std::move(subscription), factory_context), + const envoy::api::v2::core::ConfigSource& rds_config_source, + const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: + ScopeKeyBuilder& scope_key_builder) + : MutableConfigProviderImplBase(std::move(subscription), factory_context, + Envoy::Config::ConfigProvider::ApiType::Delta), subscription_(static_cast( MutableConfigProviderImplBase::subscription().get())), - rds_(rds) { - initialize(initial_config); -} - -Envoy::Config::ConfigProvider::ConfigConstSharedPtr -ScopedRdsConfigProvider::onConfigProtoUpdate(const Protobuf::Message& config_proto) { - return std::make_shared( - static_cast(config_proto)); + rds_config_source_(rds_config_source) { + initialize([scope_key_builder](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { + return std::make_shared(scope_key_builder); + }); } ProtobufTypes::MessagePtr ScopedRoutesConfigProviderManager::dumpConfigs() const { @@ -123,10 +181,15 @@ ProtobufTypes::MessagePtr ScopedRoutesConfigProviderManager::dumpConfigs() const ASSERT(subscription); if (subscription->configInfo()) { - auto* dynamic_config = config_dump->mutable_dynamic_scoped_routes_configs()->Add(); + auto* dynamic_config = config_dump->mutable_dynamic_scoped_route_configs()->Add(); dynamic_config->set_version_info(subscription->configInfo().value().last_config_version_); - dynamic_config->mutable_scoped_routes_config()->MergeFrom( - static_cast(subscription.get())->configProto().value()); + const ScopedRdsConfigSubscription::ScopedRouteConfigurationMap& scoped_route_configurations = + static_cast(subscription.get()) + ->scopedRouteConfigurations(); + for (auto it = scoped_route_configurations.begin(); it != scoped_route_configurations.end(); + ++it) { + dynamic_config->mutable_scoped_route_configs()->Add()->MergeFrom(it->second); + } TimestampUtil::systemClockToTimestamp(subscription->lastUpdated(), *dynamic_config->mutable_last_updated()); } @@ -134,12 +197,14 @@ ProtobufTypes::MessagePtr ScopedRoutesConfigProviderManager::dumpConfigs() const for (const auto& provider : immutableConfigProviders(Envoy::Config::ConfigProviderInstanceType::Inline)) { - ASSERT(provider->configProtoInfo()); - auto* inline_config = config_dump->mutable_inline_scoped_routes_configs()->Add(); - inline_config->mutable_scoped_routes_config()->MergeFrom( - provider->configProtoInfo() - .value() - .config_proto_); + ASSERT(provider->configProtoInfoVec().has_value()); + auto* inline_config = config_dump->mutable_inline_scoped_route_configs()->Add(); + const std::vector& scoped_route_configurations = + provider->configProtoInfoVec().value().config_protos_; + for (auto it = scoped_route_configurations.begin(); it != scoped_route_configurations.end(); + ++it) { + inline_config->mutable_scoped_route_configs()->Add()->MergeFrom(**it); + } TimestampUtil::systemClockToTimestamp(provider->lastUpdated(), *inline_config->mutable_last_updated()); } @@ -166,25 +231,20 @@ Envoy::Config::ConfigProviderPtr ScopedRoutesConfigProviderManager::createXdsCon static_cast(config_provider_manager)); }); - Envoy::Config::ConfigProvider::ConfigConstSharedPtr initial_config; - const Envoy::Config::MutableConfigProviderImplBase* first_provider = - subscription->getAnyBoundMutableConfigProvider(); - if (first_provider != nullptr) { - initial_config = first_provider->getConfig(); - } const auto& typed_optarg = static_cast(optarg); - return std::make_unique(std::move(subscription), initial_config, - factory_context, typed_optarg.rds_); + return std::make_unique(std::move(subscription), factory_context, + typed_optarg.rds_config_source_, + typed_optarg.scope_key_builder_); } Envoy::Config::ConfigProviderPtr ScopedRoutesConfigProviderManager::createStaticConfigProvider( - const Protobuf::Message& config_proto, Server::Configuration::FactoryContext& factory_context, + std::vector>&& config_protos, + Server::Configuration::FactoryContext& factory_context, const Envoy::Config::ConfigProviderManager::OptionalArg& optarg) { - const auto& scoped_routes_proto = - dynamic_cast(config_proto); const auto& typed_optarg = static_cast(optarg); - return absl::make_unique(scoped_routes_proto, factory_context, - *this, typed_optarg.rds_); + return absl::make_unique( + std::move(config_protos), factory_context, *this, typed_optarg.rds_config_source_, + typed_optarg.scope_key_builder_); } } // namespace Router diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index eef5f030ca5fd..fd659311c5323 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -9,6 +9,7 @@ #include "common/common/logger.h" #include "common/config/config_provider_impl.h" +#include "common/router/scoped_config_impl.h" namespace Envoy { namespace Router { @@ -31,22 +32,33 @@ class ScopedRoutesConfigProviderManager; class InlineScopedRoutesConfigProvider : public Envoy::Config::ImmutableConfigProviderImplBase { public: InlineScopedRoutesConfigProvider( - const envoy::api::v2::ScopedRouteConfigurationsSet& config_proto, + std::vector>&& config_protos, Server::Configuration::FactoryContext& factory_context, ScopedRoutesConfigProviderManager& config_provider_manager, - const envoy::config::filter::network::http_connection_manager::v2::Rds& rds); + const envoy::api::v2::core::ConfigSource& rds_config_source, + const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: + ScopeKeyBuilder& scope_key_builder); ~InlineScopedRoutesConfigProvider() override = default; // Envoy::Config::ConfigProvider - const Protobuf::Message* getConfigProto() const override { return &config_proto_; } + const Protobuf::Message* getConfigProto() const override { return nullptr; } + const std::vector getConfigProtos() const override { + std::vector ret_protos(config_protos_.size()); + std::for_each(config_protos_.begin(), config_protos_.end(), + [&ret_protos](const std::unique_ptr& message) { + ret_protos.push_back(message.get()); + }); + return ret_protos; + } + std::string getConfigVersion() const override { return ""; } ConfigConstSharedPtr getConfig() const override { return config_; } private: ConfigConstSharedPtr config_; - const envoy::api::v2::ScopedRouteConfigurationsSet config_proto_; - const envoy::config::filter::network::http_connection_manager::v2::Rds rds_; + const std::vector> config_protos_; + const envoy::api::v2::core::ConfigSource rds_config_source_; }; /** @@ -66,8 +78,11 @@ struct ScopedRdsStats { // A scoped RDS subscription to be used with the dynamic scoped RDS ConfigProvider. class ScopedRdsConfigSubscription : public Envoy::Config::ConfigSubscriptionInstanceBase, - Envoy::Config::SubscriptionCallbacks { + Envoy::Config::SubscriptionCallbacks { public: + using ScopedRouteConfigurationMap = + std::map; + ScopedRdsConfigSubscription( const envoy::config::filter::network::http_connection_manager::v2::ScopedRds& scoped_rds, const std::string& manager_identifier, Server::Configuration::FactoryContext& factory_context, @@ -76,7 +91,7 @@ class ScopedRdsConfigSubscription ~ScopedRdsConfigSubscription() override = default; // Envoy::Config::ConfigSubscriptionInstanceBase - void start() override { subscription_->start({scoped_routes_config_name_}, *this); } + void start() override { subscription_->start({}, *this); } // Envoy::Config::SubscriptionCallbacks void onConfigUpdate(const ResourceVector& resources, const std::string& version_info) override; @@ -84,20 +99,20 @@ class ScopedRdsConfigSubscription ConfigSubscriptionInstanceBase::onConfigUpdateFailed(); } std::string resourceName(const ProtobufWkt::Any& resource) override { - return MessageUtil::anyConvert(resource).name(); + return MessageUtil::anyConvert(resource).name(); } - const absl::optional& configProto() const { - return scoped_routes_proto_; + const ScopedRouteConfigurationMap& scopedRouteConfigurations() const { + return scoped_route_configurations_; } private: - std::unique_ptr> + std::unique_ptr> subscription_; - const std::string scoped_routes_config_name_; Stats::ScopePtr scope_; ScopedRdsStats stats_; - absl::optional scoped_routes_proto_; + ScopedRouteConfigurationMap scoped_route_configurations_; + ScopedConfigManager scoped_config_manager_; }; using ScopedRdsConfigSubscriptionSharedPtr = std::shared_ptr; @@ -106,25 +121,35 @@ using ScopedRdsConfigSubscriptionSharedPtr = std::shared_ptrconfigProto().has_value()) { - return nullptr; + const Protobuf::Message* getConfigProto() const override { return nullptr; } + const std::vector getConfigProtos() const override { + const ScopedRdsConfigSubscription::ScopedRouteConfigurationMap& scoped_route_configurations = + subscription_->scopedRouteConfigurations(); + if (scoped_route_configurations.empty()) { + return {}; + } + + std::vector config_protos(scoped_route_configurations.size()); + for (auto it = scoped_route_configurations.begin(); it != scoped_route_configurations.end(); + ++it) { + config_protos.push_back(&it->second); } - return &subscription_->configProto().value(); + return config_protos; } std::string getConfigVersion() const override { if (subscription_->configInfo().has_value()) { @@ -133,10 +158,13 @@ class ScopedRdsConfigProvider : public Envoy::Config::MutableConfigProviderImplB return ""; } + ConfigConstSharedPtr getConfig() const override { + return std::dynamic_pointer_cast(tls()->get()); + } private: ScopedRdsConfigSubscription* subscription_; - const envoy::config::filter::network::http_connection_manager::v2::Rds rds_; + const envoy::api::v2::core::ConfigSource rds_config_source_; }; // A ConfigProviderManager for scoped routing configuration that creates static/inline and dynamic @@ -150,13 +178,24 @@ class ScopedRoutesConfigProviderManager : public Envoy::Config::ConfigProviderMa // Envoy::Config::ConfigProviderManagerImplBase ProtobufTypes::MessagePtr dumpConfigs() const override; + + // Envoy::Config::ConfigProviderManager Envoy::Config::ConfigProviderPtr createXdsConfigProvider(const Protobuf::Message& config_source_proto, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, const Envoy::Config::ConfigProviderManager::OptionalArg& optarg) override; + Envoy::Config::ConfigProviderPtr + createStaticConfigProvider(const Protobuf::Message&, Server::Configuration::FactoryContext&, + const Envoy::Config::ConfigProviderManager::OptionalArg&) override { + ASSERT(false || + "SRDS supports delta updates and requires the use of the createStaticConfigProvider() " + "overload that accepts a config proto set as an argument."); + return nullptr; + } Envoy::Config::ConfigProviderPtr createStaticConfigProvider( - const Protobuf::Message& config_proto, Server::Configuration::FactoryContext& factory_context, + std::vector>&& config_protos, + Server::Configuration::FactoryContext& factory_context, const Envoy::Config::ConfigProviderManager::OptionalArg& optarg) override; }; @@ -165,12 +204,14 @@ class ScopedRoutesConfigProviderManagerOptArg : public Envoy::Config::ConfigProviderManager::OptionalArg { public: ScopedRoutesConfigProviderManagerOptArg( - const envoy::config::filter::network::http_connection_manager::v2::Rds& rds) - : rds_(rds) {} - - ~ScopedRoutesConfigProviderManagerOptArg() override = default; - - const envoy::config::filter::network::http_connection_manager::v2::Rds& rds_; + const envoy::api::v2::core::ConfigSource& rds_config_source, + const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: + ScopeKeyBuilder& scope_key_builder) + : rds_config_source_(rds_config_source), scope_key_builder_(scope_key_builder) {} + + const envoy::api::v2::core::ConfigSource& rds_config_source_; + const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder& + scope_key_builder_; }; } // namespace Router diff --git a/source/common/router/thread_local_scoped_config.h b/source/common/router/thread_local_scoped_config.h new file mode 100644 index 0000000000000..39a80f27068c8 --- /dev/null +++ b/source/common/router/thread_local_scoped_config.h @@ -0,0 +1,25 @@ +#pragma once + +#include +#include + +#include "envoy/thread_local/thread_local.h" + +namespace Envoy { +namespace Router { + +class ScopedRouteInfo {}; +using ScopedRouteInfoConstSharedPtr = std::shared_ptr; + +class ThreadLocalScopedConfig : public ThreadLocal::ThreadLocalObject { +public: + virtual ~ThreadLocalScopedConfig() {} + + virtual void addOrUpdateRoutingScope(ScopedRouteInfoConstSharedPtr scoped_route_info) PURE; + + // Removes a routing scope from the set of scopes matched against each HTTP request. + virtual void removeRoutingScope(const std::string& scope_name) PURE; +}; + +} // namespace Router +} // namespace Envoy diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index be70ba7dd470a..1f9d6a44d4c93 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -65,42 +65,6 @@ std::unique_ptr createInternalAddressConfig( return std::make_unique(); } -// Validates that HttpConnectionManager configuration correctly specifies routing configuration. -void validateScopedRoutingAndRds( - const envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& - config) { - // NOTE: This validation can not be done via proto validators due to the conditionals involved. - if (config.scoped_routes_specifier_case() != - envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager:: - SCOPED_ROUTES_SPECIFIER_NOT_SET) { - // When scoped routing is enabled, RDS _must_ be used and subscriptions are dynamically - // generated based on the SRDS config. - if (config.route_specifier_case() != - envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager::kRds) { - throw EnvoyException(fmt::format("Error: RDS must be used when scoped routing is enabled")); - } - - if (config.rds().subscription_specifier_case() != - envoy::config::filter::network::http_connection_manager::v2::Rds::kScopedRdsTemplate || - config.rds().scoped_rds_template() != true) { - throw EnvoyException(fmt::format( - "Error: the RDS subscription specifier must be set to scoped_rds_template=true " - "when scoped routing is enabled")); - } - } else { - // Scoped routing is not enabled, ensure a route configuration name is specified if RDS is - // used. - if (config.route_specifier_case() == - envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager::kRds) { - if (config.rds().subscription_specifier_case() != - envoy::config::filter::network::http_connection_manager::v2::Rds::kRouteConfigName) { - throw EnvoyException(fmt::format( - "Error: RDS must specify a route_config_name when scoped routing is not enabled")); - } - } - } -} - } // namespace // Singleton registration via macro defined in envoy/singleton/manager.h @@ -200,13 +164,10 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( context_.listenerScope())), proxy_100_continue_(config.proxy_100_continue()), delayed_close_timeout_(PROTOBUF_GET_MS_OR_DEFAULT(config, delayed_close_timeout, 1000)) { - // Throws an exception on invalid config. - validateScopedRoutingAndRds(config); - // If scoped RDS is enabled, avoid creating a route config provider. Route config providers will // be managed by the scoped routing logic instead. - if (config.rds().subscription_specifier_case() != - envoy::config::filter::network::http_connection_manager::v2::Rds::kScopedRdsTemplate) { + if (config.route_specifier_case() != envoy::config::filter::network::http_connection_manager::v2:: + HttpConnectionManager::kScopedRoutes) { route_config_provider_ = Router::RouteConfigProviderUtil::create( config, context_, stats_prefix_, route_config_provider_manager_); } diff --git a/source/server/http/admin.h b/source/server/http/admin.h index 5c6b9694f2ed3..3bb5fefb51d59 100644 --- a/source/server/http/admin.h +++ b/source/server/http/admin.h @@ -178,6 +178,8 @@ class AdminImpl : public Admin, const Protobuf::Message* getConfigProto() const override { return nullptr; } std::string getConfigVersion() const override { return ""; } ConfigConstSharedPtr getConfig() const override { return config_; } + ApiType apiType() const override { return ApiType::Full; } + const std::vector getConfigProtos() const { return {}; } Router::ScopedConfigConstSharedPtr config_; TimeSource& time_source_; diff --git a/test/common/config/config_provider_impl_test.cc b/test/common/config/config_provider_impl_test.cc index 125f5bdc11348..30c111e52413b 100644 --- a/test/common/config/config_provider_impl_test.cc +++ b/test/common/config/config_provider_impl_test.cc @@ -26,6 +26,7 @@ class StaticDummyConfigProvider : public ImmutableConfigProviderImplBase { // Envoy::Config::ConfigProvider const Protobuf::Message* getConfigProto() const override { return &config_proto_; } + const std::vector getConfigProtos() const override { return {}; } // Envoy::Config::ConfigProvider std::string getConfigVersion() const override { return ""; } @@ -87,7 +88,7 @@ class DummyDynamicConfigProvider : public MutableConfigProviderImplBase { DummyDynamicConfigProvider(DummyConfigSubscriptionSharedPtr&& subscription, ConfigConstSharedPtr initial_config, Server::Configuration::FactoryContext& factory_context) - : MutableConfigProviderImplBase(std::move(subscription), factory_context), + : MutableConfigProviderImplBase(std::move(subscription), factory_context, ApiType::Full), subscription_(static_cast( MutableConfigProviderImplBase::subscription().get())) { initialize(initial_config); @@ -111,6 +112,7 @@ class DummyDynamicConfigProvider : public MutableConfigProviderImplBase { } return &subscription_->config_proto().value(); } + const std::vector getConfigProtos() const override { return {}; } // Envoy::Config::ConfigProvider std::string getConfigVersion() const override { return ""; } @@ -191,6 +193,12 @@ class DummyConfigProviderManager : public ConfigProviderManagerImplBase { dynamic_cast(config_proto), factory_context, *this); } + ConfigProviderPtr + createStaticConfigProvider(std::vector>&&, + Server::Configuration::FactoryContext&, const OptionalArg&) override { + ASSERT(false || "this provider does not expect multiple config protos"); + return nullptr; + } }; StaticDummyConfigProvider::StaticDummyConfigProvider( @@ -198,7 +206,7 @@ StaticDummyConfigProvider::StaticDummyConfigProvider( Server::Configuration::FactoryContext& factory_context, DummyConfigProviderManager& config_provider_manager) : ImmutableConfigProviderImplBase(factory_context, config_provider_manager, - ConfigProviderInstanceType::Static), + ConfigProviderInstanceType::Static, ApiType::Full), config_(std::make_shared(config_proto)), config_proto_(config_proto) {} DummyConfigSubscription::DummyConfigSubscription( diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index af1bacf03722d..5ba004b1a045b 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -21,16 +21,27 @@ namespace Envoy { namespace Router { namespace { -void parseScopedRoutesConfigFromYaml( - envoy::api::v2::ScopedRouteConfigurationsSet& scoped_routes_proto, const std::string& yaml) { - MessageUtil::loadFromYaml(yaml, scoped_routes_proto); +void parseScopedRouteConfigurationFromYaml( + envoy::api::v2::ScopedRouteConfiguration& scoped_route_config, const std::string& yaml) { + MessageUtil::loadFromYaml(yaml, scoped_route_config); } -envoy::api::v2::ScopedRouteConfigurationsSet -parseScopedRoutesConfigFromYaml(const std::string& yaml) { - envoy::api::v2::ScopedRouteConfigurationsSet scoped_routes_proto; - parseScopedRoutesConfigFromYaml(scoped_routes_proto, yaml); - return scoped_routes_proto; +envoy::api::v2::ScopedRouteConfiguration +parseScopedRouteConfigurationFromYaml(const std::string& yaml) { + envoy::api::v2::ScopedRouteConfiguration scoped_route_config; + parseScopedRouteConfigurationFromYaml(scoped_route_config, yaml); + return scoped_route_config; +} + +std::vector> +concreteProtosToMessageVec(std::vector protos) { + std::vector> messages; + for (const auto& proto : protos) { + Protobuf::Message* message = proto.New(); + message->CopyFrom(proto); + messages.push_back(std::unique_ptr(message)); + } + return messages; } class ScopedRoutesTestBase : public TestBase { @@ -41,15 +52,13 @@ class ScopedRoutesTestBase : public TestBase { std::make_unique(factory_context_.admin_); const std::string rds_config_yaml = R"EOF( -config_source: - api_config_source: - api_type: REST - cluster_names: - - foo_rds_cluster - refresh_delay: { seconds: 1, nanos: 0 } -scoped_rds_template: true - )EOF"; - MessageUtil::loadFromYaml(rds_config_yaml, rds_config_); +api_config_source: + api_type: REST + cluster_names: + - foo_rds_cluster + refresh_delay: { seconds: 1, nanos: 0 } +)EOF"; + MessageUtil::loadFromYaml(rds_config_yaml, rds_config_source_); } ~ScopedRoutesTestBase() override { factory_context_.thread_local_.shutdownThread(); } @@ -59,7 +68,7 @@ scoped_rds_template: true NiceMock factory_context_; std::unique_ptr config_provider_manager_; Event::SimulatedTimeSystem time_system_; - envoy::config::filter::network::http_connection_manager::v2::Rds rds_config_; + envoy::api::v2::core::ConfigSource rds_config_source_; }; class ScopedRdsTest : public ScopedRoutesTestBase { @@ -81,19 +90,29 @@ class ScopedRdsTest : public ScopedRoutesTestBase { EXPECT_CALL(*cluster.info_, type()); const std::string config_yaml = R"EOF( -config_source: +scope_key_builder: + fragments: + - header_value_extractor: { name: X-Google-VIP } +rds_config_source: api_config_source: api_type: REST cluster_names: - foo_cluster refresh_delay: { seconds: 1, nanos: 0 } -scoped_routes_config_set_name: foo_scope_set - )EOF"; - envoy::config::filter::network::http_connection_manager::v2::ScopedRds scoped_rds_config; - MessageUtil::loadFromYaml(config_yaml, scoped_rds_config); +scoped_rds: + scoped_rds_config_source: + api_config_source: + api_type: REST + cluster_names: + - foo_cluster + refresh_delay: { seconds: 1, nanos: 0 } +)EOF"; + envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes scoped_routes_config; + MessageUtil::loadFromYaml(config_yaml, scoped_routes_config); provider_ = config_provider_manager_->createXdsConfigProvider( - scoped_rds_config, factory_context_, "foo.", - ScopedRoutesConfigProviderManagerOptArg(rds_config_)); + scoped_routes_config.scoped_rds(), factory_context_, "foo.", + ScopedRoutesConfigProviderManagerOptArg(scoped_routes_config.rds_config_source(), + scoped_routes_config.scope_key_builder())); subscription_ = &dynamic_cast(*provider_).subscription(); } @@ -112,45 +131,35 @@ TEST_F(ScopedRdsTest, ValidateFail) { // 'name' validation: value must be > 1 byte. const std::string config_yaml = R"EOF( name: -scope_key_builder: +route_configuration_name: foo_routes +key: fragments: - - header_value_extractor: { name: element } -scopes: - - route_configuration_name: foo_routes - key: - fragments: - - string_key: x-foo-key + - string_key: x-foo-key )EOF"; - Protobuf::RepeatedPtrField resources; - parseScopedRoutesConfigFromYaml(*resources.Add(), config_yaml); + Protobuf::RepeatedPtrField resources; + parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); EXPECT_THROW(subscription.onConfigUpdate(resources, "1"), ProtoValidationException); - // 'scope_key_builder.fragments' validation: must define at least 1 item in the repeated field. + // 'route_configuration_name' validation: value must be > 1 byte. const std::string config_yaml2 = R"EOF( name: foo_scope_set -scope_key_builder: +route_configuration_name: +key: fragments: -scopes: - - route_configuration_name: foo_routes - key: - fragments: - - string_key: x-foo-key + - string_key: x-foo-key )EOF"; - Protobuf::RepeatedPtrField resources2; - parseScopedRoutesConfigFromYaml(*resources2.Add(), config_yaml2); + Protobuf::RepeatedPtrField resources2; + parseScopedRouteConfigurationFromYaml(*resources2.Add(), config_yaml2); EXPECT_THROW(subscription.onConfigUpdate(resources2, "1"), ProtoValidationException); - // 'scopes.fragments' validation: must define at least 1 item in the repeated field. + // 'key' validation: must define at least 1 fragment. const std::string config_yaml3 = R"EOF( name: foo_scope_set -scope_key_builder: - fragments: -scopes: - - route_configuration_name: foo_routes - key: +route_configuration_name: foo_routes +key: )EOF"; - Protobuf::RepeatedPtrField resources3; - parseScopedRoutesConfigFromYaml(*resources3.Add(), config_yaml3); + Protobuf::RepeatedPtrField resources3; + parseScopedRouteConfigurationFromYaml(*resources3.Add(), config_yaml3); EXPECT_THROW(subscription.onConfigUpdate(resources3, "1"), ProtoValidationException); } @@ -158,68 +167,86 @@ name: foo_scope_set TEST_F(ScopedRdsTest, EmptyResource) { setup(); - Protobuf::RepeatedPtrField resources; + Protobuf::RepeatedPtrField resources; subscription().onConfigUpdate(resources, "1"); - EXPECT_EQ(1UL, - factory_context_.scope_.counter("foo.scoped_rds.foo_scope_set.update_empty").value()); + EXPECT_EQ(1UL, factory_context_.scope_.counter("foo.scoped_rds.update_empty").value()); } -// Tests that only one resource is provided during a config update. -TEST_F(ScopedRdsTest, TooManyResources) { +// Tests that multiple uniquely named resources are allowed in config updates. +TEST_F(ScopedRdsTest, MultipleResources) { setup(); - Protobuf::RepeatedPtrField resources; - resources.Add(); - resources.Add(); - EXPECT_THROW(subscription().onConfigUpdate(resources, "1"), EnvoyException); + const std::string config_yaml = R"EOF( +name: foo_scope +route_configuration_name: foo_routes +key: + fragments: + - string_key: x-foo-key +)EOF"; + Protobuf::RepeatedPtrField resources; + parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); + const std::string config_yaml2 = R"EOF( +name: foo_scope2 +route_configuration_name: foo_routes +key: + fragments: + - string_key: x-foo-key +)EOF"; + parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml2); + // EXPECT_NO_THROW(subscription().onConfigUpdate(resources, "1")); + subscription().onConfigUpdate(resources, "1"); + EXPECT_EQ(1UL, factory_context_.scope_.counter("foo.scoped_rds.config_reload").value()); } -TEST_F(ScopedRdsTest, ConfigUpdateSuccess) { +// Tests that only one resource is provided during a config update. +TEST_F(ScopedRdsTest, InvalidDuplicateResource) { setup(); - ScopedRdsConfigSubscription& subscription = - dynamic_cast(*provider_).subscription(); - - // 'name' validation const std::string config_yaml = R"EOF( -name: foo_scope_set -scope_key_builder: +name: foo_scope +route_configuration_name: foo_routes +key: fragments: - - header_value_extractor: { name: element } -scopes: - - route_configuration_name: foo_routes - key: - fragments: - - string_key: x-foo-key + - string_key: x-foo-key )EOF"; - Protobuf::RepeatedPtrField resources; - parseScopedRoutesConfigFromYaml(*resources.Add(), config_yaml); - subscription.onConfigUpdate(resources, "1"); - EXPECT_EQ(1UL, - factory_context_.scope_.counter("foo.scoped_rds.foo_scope_set.config_reload").value()); + Protobuf::RepeatedPtrField resources; + parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); + parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); + EXPECT_THROW_WITH_MESSAGE(subscription().onConfigUpdate(resources, "1"), EnvoyException, + "duplicate scoped route configuration foo_scope found"); } // Tests that defining an invalid cluster in the SRDS config results in an error. TEST_F(ScopedRdsTest, UnknownCluster) { const std::string config_yaml = R"EOF( -config_source: +scope_key_builder: + fragments: + - header_value_extractor: { name: X-Google-VIP } +rds_config_source: api_config_source: api_type: REST cluster_names: - foo_cluster refresh_delay: { seconds: 1, nanos: 0 } -scoped_routes_config_set_name: foo_scope_set - )EOF"; - envoy::config::filter::network::http_connection_manager::v2::ScopedRds scoped_rds_config; - MessageUtil::loadFromYaml(config_yaml, scoped_rds_config); +scoped_rds: + scoped_rds_config_source: + api_config_source: + api_type: REST + cluster_names: + - foo_cluster + refresh_delay: { seconds: 1, nanos: 0 } +)EOF"; + envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes scoped_routes_config; + MessageUtil::loadFromYaml(config_yaml, scoped_routes_config); Upstream::ClusterManager::ClusterInfoMap cluster_map; EXPECT_CALL(factory_context_.cluster_manager_, clusters()).WillOnce(Return(cluster_map)); EXPECT_THROW_WITH_MESSAGE( config_provider_manager_->createXdsConfigProvider( - scoped_rds_config, factory_context_, "foo.", - ScopedRoutesConfigProviderManagerOptArg(rds_config_)); - , EnvoyException, + scoped_routes_config.scoped_rds(), factory_context_, "foo.", + ScopedRoutesConfigProviderManagerOptArg(scoped_routes_config.rds_config_source(), + scoped_routes_config.scope_key_builder())), + EnvoyException, "envoy::api::v2::core::ConfigSource must have a statically defined non-EDS " "cluster: 'foo_cluster' does not exist, was added via api, or is an " "EDS cluster"); @@ -243,36 +270,49 @@ TEST_F(ScopedRoutesConfigProviderManagerTest, ConfigDump) { // No routes at all, no last_updated timestamp envoy::admin::v2alpha::ScopedRoutesConfigDump expected_config_dump; MessageUtil::loadFromYaml(R"EOF( -inline_scoped_routes_configs: -dynamic_scoped_routes_configs: +inline_scoped_route_configs: +dynamic_scoped_route_configs: )EOF", expected_config_dump); EXPECT_EQ(expected_config_dump.DebugString(), scoped_routes_config_dump.DebugString()); - std::string config_yaml = R"EOF( + const std::string config_yaml = R"EOF( name: foo -scope_key_builder: - fragments: - - header_value_extractor: { name: X-Google-VIP } -scopes: - - route_configuration_name: foo-route-config - key: - fragments: { string_key: "172.10.10.10" } +route_configuration_name: foo-route-config +key: + fragments: { string_key: "172.10.10.10" } +)EOF"; + const std::string config_yaml2 = R"EOF( +name: foo2 +route_configuration_name: foo-route-config2 +key: + fragments: { string_key: "172.10.10.20" } )EOF"; + std::vector> config_protos = + concreteProtosToMessageVec({parseScopedRouteConfigurationFromYaml(config_yaml), + parseScopedRouteConfigurationFromYaml(config_yaml2)}); timeSystem().setSystemTime(std::chrono::milliseconds(1234567891234)); + envoy::config::filter::network::http_connection_manager::v2 ::ScopedRoutes::ScopeKeyBuilder + scope_key_builder; + MessageUtil::loadFromYaml(R"EOF( +scope_key_builder: + fragments: + - header_value_extractor: { name: X-Google-VIP } +)EOF", + scope_key_builder); // Only load the inline scopes. Envoy::Config::ConfigProviderPtr inline_config = config_provider_manager_->createStaticConfigProvider( - parseScopedRoutesConfigFromYaml(config_yaml), factory_context_, - ScopedRoutesConfigProviderManagerOptArg(rds_config_)); + std::move(config_protos), factory_context_, + ScopedRoutesConfigProviderManagerOptArg(rds_config_source_, scope_key_builder)); message_ptr = factory_context_.admin_.config_tracker_.config_tracker_callbacks_["route_scopes"](); const auto& scoped_routes_config_dump2 = MessageUtil::downcastAndValidate( *message_ptr); MessageUtil::loadFromYaml(R"EOF( -inline_scoped_routes_configs: +inline_scoped_route_configs: - scoped_routes_config: name: foo scope_key_builder: diff --git a/test/extensions/filters/network/http_connection_manager/config_test.cc b/test/extensions/filters/network/http_connection_manager/config_test.cc index 4f87c8fc2dce2..362b26bc2c4a1 100644 --- a/test/extensions/filters/network/http_connection_manager/config_test.cc +++ b/test/extensions/filters/network/http_connection_manager/config_test.cc @@ -463,137 +463,6 @@ TEST_F(HttpConnectionManagerConfigTest, ReversedEncodeOrderConfig) { EXPECT_TRUE(config.reverseEncodeOrder()); } -TEST_F(HttpConnectionManagerConfigTest, ScopedRoutingValidationFailures) { - const std::string config_yaml = R"EOF( -route_config: - name: foo_route -scoped_routes_config: - name: foo_route_scopes -codec_type: auto -stat_prefix: foo -http_filters: - - name: envoy.router -)EOF"; - - EXPECT_THROW_WITH_MESSAGE( - HttpConnectionManagerConfig(parseHttpConnectionManagerFromV2Yaml(config_yaml), context_, - date_provider_, route_config_provider_manager_, - scoped_routes_config_provider_manager_), - EnvoyException, "Error: RDS must be used when scoped routing is enabled"); - - const std::string config_yaml2 = R"EOF( -route_config: - name: foo_route -scoped_rds: - config_source: { path: /path/to/file } - scoped_routes_config_set_name: foo_scope -codec_type: auto -stat_prefix: foo -http_filters: - - name: envoy.router -)EOF"; - - EXPECT_THROW_WITH_MESSAGE( - HttpConnectionManagerConfig(parseHttpConnectionManagerFromV2Yaml(config_yaml2), context_, - date_provider_, route_config_provider_manager_, - scoped_routes_config_provider_manager_), - EnvoyException, "Error: RDS must be used when scoped routing is enabled"); - - const std::string config_yaml3 = R"EOF( -rds: - config_source: { path: /path/to/file } - route_config_name: foo_routes -scoped_routes_config: - name: foo_route_scopes -codec_type: auto -stat_prefix: foo -http_filters: - - name: envoy.router -)EOF"; - - EXPECT_THROW_WITH_MESSAGE( - HttpConnectionManagerConfig(parseHttpConnectionManagerFromV2Yaml(config_yaml3), context_, - date_provider_, route_config_provider_manager_, - scoped_routes_config_provider_manager_), - EnvoyException, - "Error: the RDS subscription specifier must be set to scoped_rds_template=true " - "when scoped routing is enabled"); - - const std::string config_yaml4 = R"EOF( -rds: - config_source: { path: /path/to/file } - scoped_rds_template: true -codec_type: auto -stat_prefix: foo -http_filters: - - name: envoy.router -)EOF"; - - EXPECT_THROW_WITH_MESSAGE( - HttpConnectionManagerConfig(parseHttpConnectionManagerFromV2Yaml(config_yaml4), context_, - date_provider_, route_config_provider_manager_, - scoped_routes_config_provider_manager_), - EnvoyException, - "Error: RDS must specify a route_config_name when scoped routing is not enabled"); -} - -TEST_F(HttpConnectionManagerConfigTest, ScopedRoutingValidationSuccess) { - const std::string config_yaml = R"EOF( -rds: - config_source: { path: /path/to/file } - scoped_rds_template: true -scoped_routes_config: - name: foo_route_scopes -codec_type: auto -stat_prefix: foo -http_filters: - - name: envoy.router -)EOF"; - - EXPECT_NO_THROW(HttpConnectionManagerConfig( - parseHttpConnectionManagerFromV2Yaml(config_yaml), context_, date_provider_, - route_config_provider_manager_, scoped_routes_config_provider_manager_)); - - const std::string config_yaml2 = R"EOF( -rds: - config_source: { path: /path/to/file } - scoped_rds_template: true -scoped_rds: - config_source: { path: /path/to/file } - scoped_routes_config_set_name: foo_scope -codec_type: auto -stat_prefix: foo -http_filters: - - name: envoy.router -)EOF"; - - EXPECT_NO_THROW(HttpConnectionManagerConfig( - parseHttpConnectionManagerFromV2Yaml(config_yaml2), context_, date_provider_, - route_config_provider_manager_, scoped_routes_config_provider_manager_)); -} - -TEST_F(HttpConnectionManagerConfigTest, ScopedRdsAndStatic) { - const std::string config_yaml = R"EOF( -rds: - config_source: { path: /path/to/file } - scoped_rds_template: true -scoped_routes_config: - name: foo_route_scopes -scoped_rds: - config_source: { path: /path/to/file } - scoped_routes_config_set_name: foo_scope -codec_type: auto -stat_prefix: foo -http_filters: - - name: envoy.router -)EOF"; - - EXPECT_THROW(HttpConnectionManagerConfig(parseHttpConnectionManagerFromV2Yaml(config_yaml), - context_, date_provider_, route_config_provider_manager_, - scoped_routes_config_provider_manager_), - EnvoyException); -} - class FilterChainTest : public HttpConnectionManagerConfigTest { public: const std::string basic_config_ = R"EOF( diff --git a/test/mocks/config/mocks.h b/test/mocks/config/mocks.h index 713c1882b24ed..ceeee5fe8afb2 100644 --- a/test/mocks/config/mocks.h +++ b/test/mocks/config/mocks.h @@ -89,6 +89,11 @@ class MockConfigProviderManager : public ConfigProviderManager { ConfigProviderPtr(const Protobuf::Message& config_proto, Server::Configuration::FactoryContext& factory_context, const Envoy::Config::ConfigProviderManager::OptionalArg& optarg)); + MOCK_METHOD3( + createStaticConfigProvider, + ConfigProviderPtr(std::vector>&& config_protos, + Server::Configuration::FactoryContext& factory_context, + const Envoy::Config::ConfigProviderManager::OptionalArg& optarg)); }; } // namespace Config From 5aed783023b8d66113a009c11997c235e3471db6 Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Fri, 29 Mar 2019 15:58:26 -0400 Subject: [PATCH 15/37] Support delta SRDS proto in integration test. Also, cleanup. Signed-off-by: Andres Guedez --- include/envoy/config/config_provider.h | 2 +- source/common/config/resources.h | 4 +- source/common/router/scoped_rds.cc | 4 +- source/common/router/scoped_rds.h | 8 +- source/server/http/admin.h | 2 +- test/common/router/scoped_rds_test.cc | 40 ++++--- .../scoped_rds_integration_test.cc | 107 ++++++++++-------- 7 files changed, 88 insertions(+), 79 deletions(-) diff --git a/include/envoy/config/config_provider.h b/include/envoy/config/config_provider.h index 9e404dd9b7868..c5b1c2ab670f1 100644 --- a/include/envoy/config/config_provider.h +++ b/include/envoy/config/config_provider.h @@ -90,7 +90,7 @@ class ConfigProvider { } absl::optional configProtoInfoVec() const { - std::vector config_protos = getConfigProtos(); + const std::vector config_protos = getConfigProtos(); if (config_protos.empty()) { return absl::nullopt; } diff --git a/source/common/config/resources.h b/source/common/config/resources.h index d18ae493c6040..b3768a7d9206d 100644 --- a/source/common/config/resources.h +++ b/source/common/config/resources.h @@ -17,8 +17,8 @@ class TypeUrlValues { const std::string ClusterLoadAssignment{"type.googleapis.com/envoy.api.v2.ClusterLoadAssignment"}; const std::string Secret{"type.googleapis.com/envoy.api.v2.auth.Secret"}; const std::string RouteConfiguration{"type.googleapis.com/envoy.api.v2.RouteConfiguration"}; - const std::string ScopedRouteConfigurationsSet{ - "type.googleapis.com/envoy.api.v2.ScopedRouteConfigurationsSet"}; + const std::string ScopedRouteConfiguration{ + "type.googleapis.com/envoy.api.v2.ScopedRouteConfiguration"}; }; typedef ConstSingleton TypeUrl; diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 67397a710b615..5bccf01b49021 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -199,8 +199,10 @@ ProtobufTypes::MessagePtr ScopedRoutesConfigProviderManager::dumpConfigs() const immutableConfigProviders(Envoy::Config::ConfigProviderInstanceType::Inline)) { ASSERT(provider->configProtoInfoVec().has_value()); auto* inline_config = config_dump->mutable_inline_scoped_route_configs()->Add(); + const absl::optional protos_info = + provider->configProtoInfoVec(); const std::vector& scoped_route_configurations = - provider->configProtoInfoVec().value().config_protos_; + protos_info.value().config_protos_; for (auto it = scoped_route_configurations.begin(); it != scoped_route_configurations.end(); ++it) { inline_config->mutable_scoped_route_configs()->Add()->MergeFrom(**it); diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index fd659311c5323..4dfd67c62792d 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -44,12 +44,12 @@ class InlineScopedRoutesConfigProvider : public Envoy::Config::ImmutableConfigPr // Envoy::Config::ConfigProvider const Protobuf::Message* getConfigProto() const override { return nullptr; } const std::vector getConfigProtos() const override { - std::vector ret_protos(config_protos_.size()); + std::vector out_protos; std::for_each(config_protos_.begin(), config_protos_.end(), - [&ret_protos](const std::unique_ptr& message) { - ret_protos.push_back(message.get()); + [&out_protos](const std::unique_ptr& message) { + out_protos.push_back(message.get()); }); - return ret_protos; + return out_protos; } std::string getConfigVersion() const override { return ""; } diff --git a/source/server/http/admin.h b/source/server/http/admin.h index 3bb5fefb51d59..006f1e89587ae 100644 --- a/source/server/http/admin.h +++ b/source/server/http/admin.h @@ -179,7 +179,7 @@ class AdminImpl : public Admin, std::string getConfigVersion() const override { return ""; } ConfigConstSharedPtr getConfig() const override { return config_; } ApiType apiType() const override { return ApiType::Full; } - const std::vector getConfigProtos() const { return {}; } + const std::vector getConfigProtos() const override { return {}; } Router::ScopedConfigConstSharedPtr config_; TimeSource& time_source_; diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index 5ba004b1a045b..258868aa62383 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -34,7 +34,7 @@ parseScopedRouteConfigurationFromYaml(const std::string& yaml) { } std::vector> -concreteProtosToMessageVec(std::vector protos) { +protosToMessageVec(std::vector&& protos) { std::vector> messages; for (const auto& proto : protos) { Protobuf::Message* message = proto.New(); @@ -193,8 +193,7 @@ route_configuration_name: foo_routes - string_key: x-foo-key )EOF"; parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml2); - // EXPECT_NO_THROW(subscription().onConfigUpdate(resources, "1")); - subscription().onConfigUpdate(resources, "1"); + EXPECT_NO_THROW(subscription().onConfigUpdate(resources, "1")); EXPECT_EQ(1UL, factory_context_.scope_.counter("foo.scoped_rds.config_reload").value()); } @@ -289,17 +288,16 @@ route_configuration_name: foo-route-config2 fragments: { string_key: "172.10.10.20" } )EOF"; std::vector> config_protos = - concreteProtosToMessageVec({parseScopedRouteConfigurationFromYaml(config_yaml), - parseScopedRouteConfigurationFromYaml(config_yaml2)}); + protosToMessageVec({parseScopedRouteConfigurationFromYaml(config_yaml), + parseScopedRouteConfigurationFromYaml(config_yaml2)}); timeSystem().setSystemTime(std::chrono::milliseconds(1234567891234)); envoy::config::filter::network::http_connection_manager::v2 ::ScopedRoutes::ScopeKeyBuilder scope_key_builder; MessageUtil::loadFromYaml(R"EOF( -scope_key_builder: - fragments: - - header_value_extractor: { name: X-Google-VIP } +fragments: + - header_value_extractor: { name: X-Google-VIP } )EOF", scope_key_builder); // Only load the inline scopes. @@ -313,19 +311,19 @@ route_configuration_name: foo-route-config2 *message_ptr); MessageUtil::loadFromYaml(R"EOF( inline_scoped_route_configs: - - scoped_routes_config: - name: foo - scope_key_builder: - fragments: - - header_value_extractor: { name: X-Google-VIP } - scopes: - - route_configuration_name: foo-route-config - key: - fragments: { string_key: "172.10.10.10" } - last_updated: - seconds: 1234567891 - nanos: 234000000 -dynamic_scoped_routes_configs: + scoped_route_configs: + - name: foo + route_configuration_name: foo-route-config + key: + fragments: { string_key: "172.10.10.10" } + - name: foo2 + route_configuration_name: foo-route-config2 + key: + fragments: { string_key: "172.10.10.20" } + last_updated: + seconds: 1234567891 + nanos: 234000000 +dynamic_scoped_route_configs: )EOF", expected_config_dump); EXPECT_EQ(expected_config_dump.DebugString(), scoped_routes_config_dump2.DebugString()); diff --git a/test/integration/scoped_rds_integration_test.cc b/test/integration/scoped_rds_integration_test.cc index a7b35dcc82bf7..92fefb63a7301 100644 --- a/test/integration/scoped_rds_integration_test.cc +++ b/test/integration/scoped_rds_integration_test.cc @@ -46,21 +46,30 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, config_helper_.addConfigModifier( [this](envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& http_connection_manager) { - // RDS must be enabled along when SRDS is enabled. - auto* rds = http_connection_manager.mutable_rds(); - rds->set_scoped_rds_template(true); - auto* api_config_source = rds->mutable_config_source()->mutable_api_config_source(); - api_config_source->set_api_type(envoy::api::v2::core::ApiConfigSource::GRPC); - auto* grpc_service = api_config_source->add_grpc_services(); - setGrpcService(*grpc_service, "rds_cluster", fake_upstreams_[2]->localAddress()); - - // Configure SRDS. - auto* scoped_rds = http_connection_manager.mutable_scoped_rds(); - scoped_rds->set_scoped_routes_config_set_name("foo_scope_set"); - api_config_source = scoped_rds->mutable_config_source()->mutable_api_config_source(); - api_config_source->set_api_type(envoy::api::v2::core::ApiConfigSource::GRPC); - grpc_service = api_config_source->add_grpc_services(); - setGrpcService(*grpc_service, "srds_cluster", fake_upstreams_[1]->localAddress()); + const std::string& scope_key_builder_config_yaml = R"EOF( +fragments: + - header_value_extractor: { name: X-Google-VIP } +)EOF"; + envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder + scope_key_builder; + MessageUtil::loadFromYaml(scope_key_builder_config_yaml, scope_key_builder); + auto* scoped_routes = http_connection_manager.mutable_scoped_routes(); + *scoped_routes->mutable_scope_key_builder() = scope_key_builder; + + envoy::api::v2::core::ApiConfigSource* rds_api_config_source = + scoped_routes->mutable_rds_config_source()->mutable_api_config_source(); + rds_api_config_source->set_api_type(envoy::api::v2::core::ApiConfigSource::GRPC); + envoy::api::v2::core::GrpcService* grpc_service = + rds_api_config_source->add_grpc_services(); + setGrpcService(*grpc_service, "rds_cluster", getRdsFakeUpstream().localAddress()); + + envoy::api::v2::core::ApiConfigSource* srds_api_config_source = + scoped_routes->mutable_scoped_rds() + ->mutable_scoped_rds_config_source() + ->mutable_api_config_source(); + srds_api_config_source->set_api_type(envoy::api::v2::core::ApiConfigSource::GRPC); + grpc_service = srds_api_config_source->add_grpc_services(); + setGrpcService(*grpc_service, "srds_cluster", getScopedRdsFakeUpstream().localAddress()); }); HttpIntegrationTest::initialize(); @@ -114,16 +123,19 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, createStream(&scoped_rds_upstream_info_, getScopedRdsFakeUpstream()); } - void sendScopedRdsResponse(const std::string& response_yaml, const std::string& version) { + void sendScopedRdsResponse(const std::vector& resource_protos, + const std::string& version) { ASSERT(scoped_rds_upstream_info_.stream_ != nullptr); envoy::api::v2::DiscoveryResponse response; response.set_version_info(version); - response.set_type_url(Config::TypeUrl::get().ScopedRouteConfigurationsSet); + response.set_type_url(Config::TypeUrl::get().ScopedRouteConfiguration); - envoy::api::v2::ScopedRouteConfigurationsSet scoped_routes_proto; - MessageUtil::loadFromYaml(response_yaml, scoped_routes_proto); - response.add_resources()->PackFrom(scoped_routes_proto); + for (const auto& resource_proto : resource_protos) { + envoy::api::v2::ScopedRouteConfiguration scoped_route_proto; + MessageUtil::loadFromYaml(resource_proto, scoped_route_proto); + response.add_resources()->PackFrom(scoped_route_proto); + } scoped_rds_upstream_info_.stream_->sendGrpcMessage(response); } @@ -136,47 +148,44 @@ INSTANTIATE_TEST_CASE_P(IpVersionsAndGrpcTypes, ScopedRdsIntegrationTest, GRPC_CLIENT_INTEGRATION_PARAMS); TEST_P(ScopedRdsIntegrationTest, BasicSuccess) { - const std::string response_yaml = R"EOF( -name: foo_scope_set -scope_key_builder: + const std::string scope_route1 = R"EOF( +name: foo_scope1 +route_configuration_name: foo_route1 +key: + fragments: + - string_key: x-foo-key +)EOF"; + const std::string scope_route2 = R"EOF( +name: foo_scope2 +route_configuration_name: foo_route2 +key: fragments: - - header_value_extractor: { name: element } -scopes: - - route_configuration_name: foo_routes - key: - fragments: - - string_key: x-foo-key + - string_key: x-foo-key )EOF"; - pre_worker_start_test_steps_ = [this, &response_yaml]() { + pre_worker_start_test_steps_ = [this, &scope_route1, &scope_route2]() { createScopedRdsStream(); - sendScopedRdsResponse(response_yaml, "1"); + sendScopedRdsResponse({scope_route1, scope_route2}, "1"); }; initialize(); - test_server_->waitForCounterGe("http.config_test.scoped_rds.foo_scope_set.update_attempt", 1); - test_server_->waitForCounterGe("http.config_test.scoped_rds.foo_scope_set.update_success", 1); + test_server_->waitForCounterGe("http.config_test.scoped_rds.update_attempt", 1); + test_server_->waitForCounterGe("http.config_test.scoped_rds.update_success", 1); // The version gauge should be set to xxHash64("1"). - test_server_->waitForGaugeEq("http.config_test.scoped_rds.foo_scope_set.version", - 13237225503670494420UL); + test_server_->waitForGaugeEq("http.config_test.scoped_rds.version", 13237225503670494420UL); - const std::string response_yaml2 = R"EOF( -name: foo_scope_set -scope_key_builder: + const std::string scope_route3 = R"EOF( +name: foo_scope3 +route_configuration_name: foo_route3 +key: fragments: - - header_value_extractor: { name: element } -scopes: - - route_configuration_name: foo_routes2 - key: - fragments: - - string_key: x-baz-key + - string_key: x-baz-key )EOF"; - sendScopedRdsResponse(response_yaml, "2"); + sendScopedRdsResponse({scope_route3}, "2"); - test_server_->waitForCounterGe("http.config_test.scoped_rds.foo_scope_set.update_attempt", 2); - test_server_->waitForCounterGe("http.config_test.scoped_rds.foo_scope_set.update_success", 2); - test_server_->waitForGaugeEq("http.config_test.scoped_rds.foo_scope_set.version", - 6927017134761466251UL); + test_server_->waitForCounterGe("http.config_test.scoped_rds.update_attempt", 2); + test_server_->waitForCounterGe("http.config_test.scoped_rds.update_success", 2); + test_server_->waitForGaugeEq("http.config_test.scoped_rds.version", 6927017134761466251UL); // TODO(AndresGuedez): test actual scoped routing logic; only the config handling is implemented // at this point. From 15b1c7c540c359cc398e3a4af389415e92003ffe Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Fri, 5 Apr 2019 11:29:39 -0400 Subject: [PATCH 16/37] Cleanup. - Readability cleanup in various places. - Improve integration tests. - Add 'name' field to ScopedRoutes proto. Signed-off-by: Andres Guedez --- api/envoy/admin/v2alpha/config_dump.proto | 15 +- .../v2/http_connection_manager.proto | 10 +- include/envoy/config/config_provider.h | 13 +- source/common/router/scoped_config_impl.cc | 16 ++- source/common/router/scoped_rds.cc | 118 ++++++++------- source/common/router/scoped_rds.h | 43 +++--- .../router/thread_local_scoped_config.h | 12 +- test/common/http/conn_manager_impl_common.h | 4 + test/common/router/scoped_rds_test.cc | 134 ++++++++++++++---- .../scoped_rds_integration_test.cc | 15 +- 10 files changed, 255 insertions(+), 125 deletions(-) diff --git a/api/envoy/admin/v2alpha/config_dump.proto b/api/envoy/admin/v2alpha/config_dump.proto index 93b4c83f1bb9d..6b7f09deed5cb 100644 --- a/api/envoy/admin/v2alpha/config_dump.proto +++ b/api/envoy/admin/v2alpha/config_dump.proto @@ -186,24 +186,29 @@ message RoutesConfigDump { // dynamically obtained scopes via the SRDS API. message ScopedRoutesConfigDump { message InlineScopedRouteConfigs { + // TODO(AndresGuedez) + string name = 1; + // The scoped route configurations. - repeated envoy.api.v2.ScopedRouteConfiguration scoped_route_configs = 1; + repeated envoy.api.v2.ScopedRouteConfiguration scoped_route_configs = 2; // The timestamp when the scoped route config set was last updated. - google.protobuf.Timestamp last_updated = 2; + google.protobuf.Timestamp last_updated = 3; } message DynamicScopedRouteConfigs { + string name = 1; + // This is the per-resource version information. This version is currently taken from the // :ref:`version_info ` field at the time that // the scoped routes configuration was loaded. - string version_info = 1; + string version_info = 2; // The scoped route configurations. - repeated envoy.api.v2.ScopedRouteConfiguration scoped_route_configs = 2; + repeated envoy.api.v2.ScopedRouteConfiguration scoped_route_configs = 3; // The timestamp when the scoped route config set was last updated. - google.protobuf.Timestamp last_updated = 3; + google.protobuf.Timestamp last_updated = 4; } // The statically loaded scoped route configs. diff --git a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto index 5409cefb45a00..043e750121ebd 100644 --- a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto +++ b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto @@ -409,6 +409,8 @@ message ScopedRouteConfigurationsList { } message ScopedRoutes { + string name = 1 [(validate.rules).string.min_bytes = 1]; + // TODO(AndresGuedez): // Specifies the mechanism for constructing keys based on request attributes to match // :ref:`scopes ` against. @@ -473,12 +475,12 @@ message ScopedRoutes { repeated FragmentBuilder fragments = 1 [(validate.rules).repeated .min_items = 1]; } - ScopeKeyBuilder scope_key_builder = 1 [(validate.rules).message.required = true]; + ScopeKeyBuilder scope_key_builder = 2 [(validate.rules).message.required = true]; // Configuration source specifier for RDS. // This config source is used to subscribe to RouteConfiguration resources specified in // ScopedRouteConfiguration messages. - envoy.api.v2.core.ConfigSource rds_config_source = 2 + envoy.api.v2.core.ConfigSource rds_config_source = 3 [(validate.rules).message.required = true, (gogoproto.nullable) = false]; oneof config_specifier { @@ -487,12 +489,12 @@ message ScopedRoutes { // The set of routing scopes corresponding to the HCM. A scope is assigned to a request by // matching a key constructed from the request's attributes according to the algorithm specified // by the :ref:`ScopeKeyBuilder` in this message. - ScopedRouteConfigurationsList scoped_route_configurations_list = 3; + ScopedRouteConfigurationsList scoped_route_configurations_list = 4; // The connection manager's route table will be dynamically loaded via the SRDS API; the // :ref:`` received via // SRDS will specify a set of routing scopes, each containing . - ScopedRds scoped_rds = 4; + ScopedRds scoped_rds = 5; } } diff --git a/include/envoy/config/config_provider.h b/include/envoy/config/config_provider.h index c5b1c2ab670f1..89ecdfb2f4569 100644 --- a/include/envoy/config/config_provider.h +++ b/include/envoy/config/config_provider.h @@ -56,11 +56,12 @@ class ConfigProvider { std::string version_; }; + using ConfigProtoVector = std::vector; /** * */ - struct ConfigProtoInfoVec { - const std::vector config_protos_; + struct ConfigProtoInfoVector { + const ConfigProtoVector config_protos_; // Only populated by dynamic config providers. std::string version_; @@ -89,12 +90,12 @@ class ConfigProvider { return ConfigProtoInfo

{*config_proto, getConfigVersion()}; } - absl::optional configProtoInfoVec() const { - const std::vector config_protos = getConfigProtos(); + absl::optional configProtoInfoVector() const { + const ConfigProtoVector config_protos = getConfigProtos(); if (config_protos.empty()) { return absl::nullopt; } - return ConfigProtoInfoVec{std::move(config_protos), getConfigVersion()}; + return ConfigProtoInfoVector{std::move(config_protos), getConfigVersion()}; } /** @@ -125,7 +126,7 @@ class ConfigProvider { /** * */ - virtual const std::vector getConfigProtos() const PURE; + virtual const ConfigProtoVector getConfigProtos() const PURE; /** * Returns the config version associated with the provider. diff --git a/source/common/router/scoped_config_impl.cc b/source/common/router/scoped_config_impl.cc index ba5b1736e0b57..ae20ea6a37484 100644 --- a/source/common/router/scoped_config_impl.cc +++ b/source/common/router/scoped_config_impl.cc @@ -5,13 +5,19 @@ namespace Envoy { namespace Router { -ScopedRouteInfoConstSharedPtr -ScopedConfigManager::addOrUpdateRoutingScope(const envoy::api::v2::ScopedRouteConfiguration&, - const std::string&) { - return std::make_shared(); +ScopedRouteInfoConstSharedPtr ScopedConfigManager::addOrUpdateRoutingScope( + const envoy::api::v2::ScopedRouteConfiguration& config_proto, const std::string&) { + auto scoped_route_info = std::make_shared(config_proto); + scoped_route_map_[config_proto.name()] = scoped_route_info; + return scoped_route_info; } -bool ScopedConfigManager::removeRoutingScope(const std::string&) { return true; } +bool ScopedConfigManager::removeRoutingScope(const std::string& name) { + if (scoped_route_map_.erase(name) == 0) { + throw EnvoyException(fmt::format("could not find {} in scoped route map for removal", name)); + } + return true; +} void ThreadLocalScopedConfigImpl::addOrUpdateRoutingScope(ScopedRouteInfoConstSharedPtr) {} diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 5bccf01b49021..c03e7fb3a8310 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -8,14 +8,21 @@ #include "common/common/assert.h" #include "common/config/subscription_factory.h" +// Types are deeply nested under Envoy::Config::ConfigProvider; use 'using-directives' across all +// ConfigProvider related types for consistency. +using Envoy::Config::ConfigProvider; +using Envoy::Config::ConfigProviderInstanceType; +using Envoy::Config::ConfigProviderManager; +using Envoy::Config::ConfigProviderPtr; + namespace Envoy { namespace Router { -Envoy::Config::ConfigProviderPtr ScopedRoutesConfigProviderUtil::maybeCreate( +ConfigProviderPtr ScopedRoutesConfigProviderUtil::maybeCreate( const envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& config, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, - Envoy::Config::ConfigProviderManager& scoped_routes_config_provider_manager) { + ConfigProviderManager& scoped_routes_config_provider_manager) { if (config.route_specifier_case() != envoy::config::filter::network::http_connection_manager::v2:: HttpConnectionManager::kScopedRoutes) { return nullptr; @@ -38,14 +45,16 @@ Envoy::Config::ConfigProviderPtr ScopedRoutesConfigProviderUtil::maybeCreate( return scoped_routes_config_provider_manager.createStaticConfigProvider( std::move(config_protos), factory_context, - ScopedRoutesConfigProviderManagerOptArg(config.scoped_routes().rds_config_source(), + ScopedRoutesConfigProviderManagerOptArg(config.scoped_routes().name(), + config.scoped_routes().rds_config_source(), config.scoped_routes().scope_key_builder())); } case envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::kScopedRds: return scoped_routes_config_provider_manager.createXdsConfigProvider( config.scoped_routes().scoped_rds(), factory_context, stat_prefix, - ScopedRoutesConfigProviderManagerOptArg(config.scoped_routes().rds_config_source(), + ScopedRoutesConfigProviderManagerOptArg(config.scoped_routes().name(), + config.scoped_routes().rds_config_source(), config.scoped_routes().scope_key_builder())); case envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: @@ -58,29 +67,30 @@ Envoy::Config::ConfigProviderPtr ScopedRoutesConfigProviderUtil::maybeCreate( } InlineScopedRoutesConfigProvider::InlineScopedRoutesConfigProvider( - std::vector>&& config_protos, + std::vector>&& config_protos, const std::string& name, Server::Configuration::FactoryContext& factory_context, ScopedRoutesConfigProviderManager& config_provider_manager, const envoy::api::v2::core::ConfigSource& rds_config_source, const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: ScopeKeyBuilder& scope_key_builder) - : Envoy::Config::ImmutableConfigProviderImplBase( - factory_context, config_provider_manager, - Envoy::Config::ConfigProviderInstanceType::Inline, - Envoy::Config::ConfigProvider::ApiType::Delta), - config_(std::make_shared(scope_key_builder)), + : Envoy::Config::ImmutableConfigProviderImplBase(factory_context, config_provider_manager, + ConfigProviderInstanceType::Inline, + ConfigProvider::ApiType::Delta), + name_(name), config_(std::make_shared(scope_key_builder)), config_protos_(std::make_move_iterator(config_protos.begin()), std::make_move_iterator(config_protos.end())), rds_config_source_(rds_config_source) {} ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( const envoy::config::filter::network::http_connection_manager::v2::ScopedRds& scoped_rds, - const std::string& manager_identifier, Server::Configuration::FactoryContext& factory_context, - const std::string& stat_prefix, ScopedRoutesConfigProviderManager& config_provider_manager) + const std::string& manager_identifier, const std::string& name, + Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, + ScopedRoutesConfigProviderManager& config_provider_manager) : ConfigSubscriptionInstanceBase( "SRDS", manager_identifier, config_provider_manager, factory_context.timeSource(), factory_context.timeSource().systemTime(), factory_context.localInfo()), - scope_(factory_context.scope().createScope(stat_prefix + "scoped_rds.")), + name_(name), + scope_(factory_context.scope().createScope(stat_prefix + "scoped_rds." + name + ".")), stats_({ALL_SCOPED_RDS_STATS(POOL_COUNTER(*scope_))}) { subscription_ = Envoy::Config::SubscriptionFactory::subscriptionFromConfigSource< envoy::api::v2::ScopedRouteConfiguration>( @@ -110,6 +120,7 @@ void ScopedRdsConfigSubscription::onConfigUpdate(const ResourceVector& resources MessageUtil::validate(scoped_route); } + // TODO(AndresGuedez): refactor such that it can be shared with other delta APIs (e.g., CDS). std::vector exception_msgs; // We need to keep track of which scoped routes we might need to remove. ScopedConfigManager::ScopedRouteMap scoped_routes_to_remove = @@ -125,13 +136,12 @@ void ScopedRdsConfigSubscription::onConfigUpdate(const ResourceVector& resources fmt::format("failed to create/update global routing scope {}", scoped_route_name)); } ENVOY_LOG(debug, "srds: add/update scoped_route '{}'", scoped_route_name); - propagateDeltaConfigUpdate( - [scoped_route_info](Envoy::Config::ConfigProvider::ConfigConstSharedPtr config) { - ThreadLocalScopedConfigImpl* thread_local_scoped_config = - const_cast( - static_cast(config.get())); - thread_local_scoped_config->addOrUpdateRoutingScope(scoped_route_info); - }); + propagateDeltaConfigUpdate([scoped_route_info](ConfigProvider::ConfigConstSharedPtr config) { + ThreadLocalScopedConfigImpl* thread_local_scoped_config = + const_cast( + static_cast(config.get())); + thread_local_scoped_config->addOrUpdateRoutingScope(scoped_route_info); + }); } catch (const EnvoyException& ex) { exception_msgs.push_back(fmt::format("{}: {}", scoped_route_name, ex.what())); } @@ -140,13 +150,12 @@ void ScopedRdsConfigSubscription::onConfigUpdate(const ResourceVector& resources for (auto scoped_route : scoped_routes_to_remove) { const std::string scoped_route_name = scoped_route.first; ENVOY_LOG(debug, "srds: remove scoped route '{}'", scoped_route_name); - propagateDeltaConfigUpdate( - [scoped_route_name](Envoy::Config::ConfigProvider::ConfigConstSharedPtr config) { - ThreadLocalScopedConfigImpl* thread_local_scoped_config = - const_cast( - static_cast(config.get())); - thread_local_scoped_config->removeRoutingScope(scoped_route_name); - }); + propagateDeltaConfigUpdate([scoped_route_name](ConfigProvider::ConfigConstSharedPtr config) { + ThreadLocalScopedConfigImpl* thread_local_scoped_config = + const_cast( + static_cast(config.get())); + thread_local_scoped_config->removeRoutingScope(scoped_route_name); + }); } ConfigSubscriptionInstanceBase::onConfigUpdate(); @@ -165,7 +174,7 @@ ScopedRdsConfigProvider::ScopedRdsConfigProvider( const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: ScopeKeyBuilder& scope_key_builder) : MutableConfigProviderImplBase(std::move(subscription), factory_context, - Envoy::Config::ConfigProvider::ApiType::Delta), + ConfigProvider::ApiType::Delta), subscription_(static_cast( MutableConfigProviderImplBase::subscription().get())), rds_config_source_(rds_config_source) { @@ -183,28 +192,30 @@ ProtobufTypes::MessagePtr ScopedRoutesConfigProviderManager::dumpConfigs() const if (subscription->configInfo()) { auto* dynamic_config = config_dump->mutable_dynamic_scoped_route_configs()->Add(); dynamic_config->set_version_info(subscription->configInfo().value().last_config_version_); - const ScopedRdsConfigSubscription::ScopedRouteConfigurationMap& scoped_route_configurations = - static_cast(subscription.get()) - ->scopedRouteConfigurations(); - for (auto it = scoped_route_configurations.begin(); it != scoped_route_configurations.end(); - ++it) { - dynamic_config->mutable_scoped_route_configs()->Add()->MergeFrom(it->second); + const ScopedRdsConfigSubscription* typed_subscription = + static_cast(subscription.get()); + dynamic_config->set_name(typed_subscription->name()); + const ScopedConfigManager::ScopedRouteMap& scoped_route_map = + typed_subscription->scopedRouteMap(); + for (ScopedConfigManager::ScopedRouteMap::const_iterator it = scoped_route_map.begin(); + it != scoped_route_map.end(); ++it) { + dynamic_config->mutable_scoped_route_configs()->Add()->MergeFrom(it->second->config_proto_); } TimestampUtil::systemClockToTimestamp(subscription->lastUpdated(), *dynamic_config->mutable_last_updated()); } } - for (const auto& provider : - immutableConfigProviders(Envoy::Config::ConfigProviderInstanceType::Inline)) { - ASSERT(provider->configProtoInfoVec().has_value()); + for (const auto& provider : immutableConfigProviders(ConfigProviderInstanceType::Inline)) { + ASSERT(provider->configProtoInfoVector().has_value()); auto* inline_config = config_dump->mutable_inline_scoped_route_configs()->Add(); - const absl::optional protos_info = - provider->configProtoInfoVec(); - const std::vector& scoped_route_configurations = + inline_config->set_name(static_cast(provider)->name()); + const absl::optional protos_info = + provider->configProtoInfoVector(); + const ConfigProvider::ConfigProtoVector& scoped_route_configurations = protos_info.value().config_protos_; - for (auto it = scoped_route_configurations.begin(); it != scoped_route_configurations.end(); - ++it) { + for (ConfigProvider::ConfigProtoVector::const_iterator it = scoped_route_configurations.begin(); + it != scoped_route_configurations.end(); ++it) { inline_config->mutable_scoped_route_configs()->Add()->MergeFrom(**it); } TimestampUtil::systemClockToTimestamp(provider->lastUpdated(), @@ -214,22 +225,25 @@ ProtobufTypes::MessagePtr ScopedRoutesConfigProviderManager::dumpConfigs() const return config_dump; } -Envoy::Config::ConfigProviderPtr ScopedRoutesConfigProviderManager::createXdsConfigProvider( +ConfigProviderPtr ScopedRoutesConfigProviderManager::createXdsConfigProvider( const Protobuf::Message& config_source_proto, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, - const Envoy::Config::ConfigProviderManager::OptionalArg& optarg) { + const ConfigProviderManager::OptionalArg& optarg) { ScopedRdsConfigSubscriptionSharedPtr subscription = ConfigProviderManagerImplBase::getSubscription( config_source_proto, factory_context.initManager(), - [&config_source_proto, &factory_context, - &stat_prefix](const std::string& manager_identifier, - ConfigProviderManagerImplBase& config_provider_manager) + [&config_source_proto, &factory_context, &stat_prefix, + &optarg](const std::string& manager_identifier, + ConfigProviderManagerImplBase& config_provider_manager) -> Envoy::Config::ConfigSubscriptionInstanceBaseSharedPtr { const auto& scoped_rds_config_source = dynamic_cast< const envoy::config::filter::network::http_connection_manager::v2::ScopedRds&>( config_source_proto); return std::make_shared( - scoped_rds_config_source, manager_identifier, factory_context, stat_prefix, + scoped_rds_config_source, manager_identifier, + static_cast(optarg) + .scoped_routes_name_, + factory_context, stat_prefix, static_cast(config_provider_manager)); }); @@ -239,14 +253,14 @@ Envoy::Config::ConfigProviderPtr ScopedRoutesConfigProviderManager::createXdsCon typed_optarg.scope_key_builder_); } -Envoy::Config::ConfigProviderPtr ScopedRoutesConfigProviderManager::createStaticConfigProvider( +ConfigProviderPtr ScopedRoutesConfigProviderManager::createStaticConfigProvider( std::vector>&& config_protos, Server::Configuration::FactoryContext& factory_context, - const Envoy::Config::ConfigProviderManager::OptionalArg& optarg) { + const ConfigProviderManager::OptionalArg& optarg) { const auto& typed_optarg = static_cast(optarg); return absl::make_unique( - std::move(config_protos), factory_context, *this, typed_optarg.rds_config_source_, - typed_optarg.scope_key_builder_); + std::move(config_protos), typed_optarg.scoped_routes_name_, factory_context, *this, + typed_optarg.rds_config_source_, typed_optarg.scope_key_builder_); } } // namespace Router diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index 4dfd67c62792d..e695994558dd9 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -33,7 +33,7 @@ class InlineScopedRoutesConfigProvider : public Envoy::Config::ImmutableConfigPr public: InlineScopedRoutesConfigProvider( std::vector>&& config_protos, - Server::Configuration::FactoryContext& factory_context, + const std::string& name, Server::Configuration::FactoryContext& factory_context, ScopedRoutesConfigProviderManager& config_provider_manager, const envoy::api::v2::core::ConfigSource& rds_config_source, const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: @@ -41,10 +41,12 @@ class InlineScopedRoutesConfigProvider : public Envoy::Config::ImmutableConfigPr ~InlineScopedRoutesConfigProvider() override = default; + const std::string& name() const { return name_; } + // Envoy::Config::ConfigProvider const Protobuf::Message* getConfigProto() const override { return nullptr; } - const std::vector getConfigProtos() const override { - std::vector out_protos; + const Envoy::Config::ConfigProvider::ConfigProtoVector getConfigProtos() const override { + Envoy::Config::ConfigProvider::ConfigProtoVector out_protos; std::for_each(config_protos_.begin(), config_protos_.end(), [&out_protos](const std::unique_ptr& message) { out_protos.push_back(message.get()); @@ -56,6 +58,7 @@ class InlineScopedRoutesConfigProvider : public Envoy::Config::ImmutableConfigPr ConfigConstSharedPtr getConfig() const override { return config_; } private: + const std::string name_; ConfigConstSharedPtr config_; const std::vector> config_protos_; const envoy::api::v2::core::ConfigSource rds_config_source_; @@ -85,11 +88,14 @@ class ScopedRdsConfigSubscription ScopedRdsConfigSubscription( const envoy::config::filter::network::http_connection_manager::v2::ScopedRds& scoped_rds, - const std::string& manager_identifier, Server::Configuration::FactoryContext& factory_context, - const std::string& stat_prefix, ScopedRoutesConfigProviderManager& config_provider_manager); + const std::string& manager_identifier, const std::string& name, + Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, + ScopedRoutesConfigProviderManager& config_provider_manager); ~ScopedRdsConfigSubscription() override = default; + const std::string& name() const { return name_; } + // Envoy::Config::ConfigSubscriptionInstanceBase void start() override { subscription_->start({}, *this); } @@ -101,17 +107,16 @@ class ScopedRdsConfigSubscription std::string resourceName(const ProtobufWkt::Any& resource) override { return MessageUtil::anyConvert(resource).name(); } - - const ScopedRouteConfigurationMap& scopedRouteConfigurations() const { - return scoped_route_configurations_; + const ScopedConfigManager::ScopedRouteMap& scopedRouteMap() const { + return scoped_config_manager_.scopedRouteMap(); } private: + const std::string name_; std::unique_ptr> subscription_; Stats::ScopePtr scope_; ScopedRdsStats stats_; - ScopedRouteConfigurationMap scoped_route_configurations_; ScopedConfigManager scoped_config_manager_; }; @@ -137,17 +142,16 @@ class ScopedRdsConfigProvider : public Envoy::Config::MutableConfigProviderImplB // Envoy::Config::ConfigProvider const Protobuf::Message* getConfigProto() const override { return nullptr; } - const std::vector getConfigProtos() const override { - const ScopedRdsConfigSubscription::ScopedRouteConfigurationMap& scoped_route_configurations = - subscription_->scopedRouteConfigurations(); - if (scoped_route_configurations.empty()) { + const Envoy::Config::ConfigProvider::ConfigProtoVector getConfigProtos() const override { + const ScopedConfigManager::ScopedRouteMap& scoped_route_map = subscription_->scopedRouteMap(); + if (scoped_route_map.empty()) { return {}; } - std::vector config_protos(scoped_route_configurations.size()); - for (auto it = scoped_route_configurations.begin(); it != scoped_route_configurations.end(); - ++it) { - config_protos.push_back(&it->second); + Envoy::Config::ConfigProvider::ConfigProtoVector config_protos(scoped_route_map.size()); + for (ScopedConfigManager::ScopedRouteMap::const_iterator it = scoped_route_map.begin(); + it != scoped_route_map.end(); ++it) { + config_protos.push_back(&it->second->config_proto_); } return config_protos; } @@ -204,11 +208,14 @@ class ScopedRoutesConfigProviderManagerOptArg : public Envoy::Config::ConfigProviderManager::OptionalArg { public: ScopedRoutesConfigProviderManagerOptArg( + const std::string& scoped_routes_name, const envoy::api::v2::core::ConfigSource& rds_config_source, const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: ScopeKeyBuilder& scope_key_builder) - : rds_config_source_(rds_config_source), scope_key_builder_(scope_key_builder) {} + : scoped_routes_name_(scoped_routes_name), rds_config_source_(rds_config_source), + scope_key_builder_(scope_key_builder) {} + const std::string scoped_routes_name_; const envoy::api::v2::core::ConfigSource& rds_config_source_; const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder& scope_key_builder_; diff --git a/source/common/router/thread_local_scoped_config.h b/source/common/router/thread_local_scoped_config.h index 39a80f27068c8..62cc2da808287 100644 --- a/source/common/router/thread_local_scoped_config.h +++ b/source/common/router/thread_local_scoped_config.h @@ -8,13 +8,23 @@ namespace Envoy { namespace Router { -class ScopedRouteInfo {}; +// TODO(AndresGuedez): +class ScopedRouteInfo { +public: + ScopedRouteInfo(const envoy::api::v2::ScopedRouteConfiguration& config_proto) + : config_proto_(config_proto) {} + + const envoy::api::v2::ScopedRouteConfiguration config_proto_; +}; using ScopedRouteInfoConstSharedPtr = std::shared_ptr; +// TODO(AndresGuedez): class ThreadLocalScopedConfig : public ThreadLocal::ThreadLocalObject { public: virtual ~ThreadLocalScopedConfig() {} + // Adds/updates a routing scope specified via the Scoped RDS API. This scope will be added to the + // set of scopes matched against the scope keys built for each HTTP request. virtual void addOrUpdateRoutingScope(ScopedRouteInfoConstSharedPtr scoped_route_info) PURE; // Removes a routing scope from the set of scopes matched against each HTTP request. diff --git a/test/common/http/conn_manager_impl_common.h b/test/common/http/conn_manager_impl_common.h index a363d7d816fcf..54bd2415cefe1 100644 --- a/test/common/http/conn_manager_impl_common.h +++ b/test/common/http/conn_manager_impl_common.h @@ -39,8 +39,12 @@ struct ScopedRouteConfigProvider : public Config::ConfigProvider { // Config::ConfigProvider SystemTime lastUpdated() const override { return time_source_.systemTime(); } const Protobuf::Message* getConfigProto() const override { return nullptr; } + const Envoy::Config::ConfigProvider::ConfigProviderVector getConfigProtos() const override { + return {}; + } std::string getConfigVersion() const override { return ""; } ConfigConstSharedPtr getConfig() const override { return config_; } + ApiType apiType() const override { return ApiType::Delta; } std::shared_ptr config_; TimeSource& time_source_; diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index 258868aa62383..4e4ffc2356f43 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -63,9 +63,21 @@ class ScopedRoutesTestBase : public TestBase { ~ScopedRoutesTestBase() override { factory_context_.thread_local_.shutdownThread(); } + void setupMockClusterMap() { + InSequence s; + cluster_map_.emplace("foo_cluster", cluster_); + EXPECT_CALL(factory_context_.cluster_manager_, clusters()).WillOnce(Return(cluster_map_)); + EXPECT_CALL(cluster_, info()); + EXPECT_CALL(*cluster_.info_, addedViaApi()); + EXPECT_CALL(cluster_, info()); + EXPECT_CALL(*cluster_.info_, type()); + } + Event::SimulatedTimeSystem& timeSystem() { return time_system_; } NiceMock factory_context_; + Upstream::ClusterManager::ClusterInfoMap cluster_map_; + Upstream::MockCluster cluster_; std::unique_ptr config_provider_manager_; Event::SimulatedTimeSystem time_system_; envoy::api::v2::core::ConfigSource rds_config_source_; @@ -80,16 +92,9 @@ class ScopedRdsTest : public ScopedRoutesTestBase { void setup() { InSequence s; - Upstream::ClusterManager::ClusterInfoMap cluster_map; - Upstream::MockCluster cluster; - cluster_map.emplace("foo_cluster", cluster); - EXPECT_CALL(factory_context_.cluster_manager_, clusters()).WillOnce(Return(cluster_map)); - EXPECT_CALL(cluster, info()); - EXPECT_CALL(*cluster.info_, addedViaApi()); - EXPECT_CALL(cluster, info()); - EXPECT_CALL(*cluster.info_, type()); - + setupMockClusterMap(); const std::string config_yaml = R"EOF( +name: foo_scoped_routes scope_key_builder: fragments: - header_value_extractor: { name: X-Google-VIP } @@ -111,7 +116,8 @@ class ScopedRdsTest : public ScopedRoutesTestBase { MessageUtil::loadFromYaml(config_yaml, scoped_routes_config); provider_ = config_provider_manager_->createXdsConfigProvider( scoped_routes_config.scoped_rds(), factory_context_, "foo.", - ScopedRoutesConfigProviderManagerOptArg(scoped_routes_config.rds_config_source(), + ScopedRoutesConfigProviderManagerOptArg(scoped_routes_config.name(), + scoped_routes_config.rds_config_source(), scoped_routes_config.scope_key_builder())); subscription_ = &dynamic_cast(*provider_).subscription(); } @@ -142,7 +148,7 @@ route_configuration_name: foo_routes // 'route_configuration_name' validation: value must be > 1 byte. const std::string config_yaml2 = R"EOF( -name: foo_scope_set +name: foo_scope route_configuration_name: key: fragments: @@ -154,7 +160,7 @@ name: foo_scope_set // 'key' validation: must define at least 1 fragment. const std::string config_yaml3 = R"EOF( -name: foo_scope_set +name: foo_scope route_configuration_name: foo_routes key: )EOF"; @@ -169,7 +175,9 @@ TEST_F(ScopedRdsTest, EmptyResource) { Protobuf::RepeatedPtrField resources; subscription().onConfigUpdate(resources, "1"); - EXPECT_EQ(1UL, factory_context_.scope_.counter("foo.scoped_rds.update_empty").value()); + EXPECT_EQ( + 1UL, + factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.update_empty").value()); } // Tests that multiple uniquely named resources are allowed in config updates. @@ -194,7 +202,9 @@ route_configuration_name: foo_routes )EOF"; parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml2); EXPECT_NO_THROW(subscription().onConfigUpdate(resources, "1")); - EXPECT_EQ(1UL, factory_context_.scope_.counter("foo.scoped_rds.config_reload").value()); + EXPECT_EQ( + 1UL, + factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.config_reload").value()); } // Tests that only one resource is provided during a config update. @@ -218,6 +228,7 @@ route_configuration_name: foo_routes // Tests that defining an invalid cluster in the SRDS config results in an error. TEST_F(ScopedRdsTest, UnknownCluster) { const std::string config_yaml = R"EOF( +name: foo_scoped_routes scope_key_builder: fragments: - header_value_extractor: { name: X-Google-VIP } @@ -243,7 +254,8 @@ TEST_F(ScopedRdsTest, UnknownCluster) { EXPECT_THROW_WITH_MESSAGE( config_provider_manager_->createXdsConfigProvider( scoped_routes_config.scoped_rds(), factory_context_, "foo.", - ScopedRoutesConfigProviderManagerOptArg(scoped_routes_config.rds_config_source(), + ScopedRoutesConfigProviderManagerOptArg(scoped_routes_config.name(), + scoped_routes_config.rds_config_source(), scoped_routes_config.scope_key_builder())), EnvoyException, "envoy::api::v2::core::ConfigSource must have a statically defined non-EDS " @@ -304,29 +316,95 @@ route_configuration_name: foo-route-config2 Envoy::Config::ConfigProviderPtr inline_config = config_provider_manager_->createStaticConfigProvider( std::move(config_protos), factory_context_, - ScopedRoutesConfigProviderManagerOptArg(rds_config_source_, scope_key_builder)); + ScopedRoutesConfigProviderManagerOptArg("foo-scoped-routes", rds_config_source_, + scope_key_builder)); message_ptr = factory_context_.admin_.config_tracker_.config_tracker_callbacks_["route_scopes"](); const auto& scoped_routes_config_dump2 = MessageUtil::downcastAndValidate( *message_ptr); MessageUtil::loadFromYaml(R"EOF( inline_scoped_route_configs: - scoped_route_configs: - - name: foo - route_configuration_name: foo-route-config - key: - fragments: { string_key: "172.10.10.10" } - - name: foo2 - route_configuration_name: foo-route-config2 - key: - fragments: { string_key: "172.10.10.20" } - last_updated: - seconds: 1234567891 - nanos: 234000000 + - name: foo-scoped-routes + scoped_route_configs: + - name: foo + route_configuration_name: foo-route-config + key: + fragments: { string_key: "172.10.10.10" } + - name: foo2 + route_configuration_name: foo-route-config2 + key: + fragments: { string_key: "172.10.10.20" } + last_updated: + seconds: 1234567891 + nanos: 234000000 dynamic_scoped_route_configs: )EOF", expected_config_dump); EXPECT_EQ(expected_config_dump.DebugString(), scoped_routes_config_dump2.DebugString()); + + setupMockClusterMap(); + envoy::config::filter::network::http_connection_manager::v2::ScopedRds scoped_rds_config; + const std::string config_source_yaml = R"EOF( +scoped_rds_config_source: + api_config_source: + api_type: REST + cluster_names: + - foo_cluster + refresh_delay: { seconds: 1, nanos: 0 } +)EOF"; + MessageUtil::loadFromYaml(config_source_yaml, scoped_rds_config); + Envoy::Config::ConfigProviderPtr dynamic_provider = + config_provider_manager_->createXdsConfigProvider( + scoped_rds_config, factory_context_, "foo.", + ScopedRoutesConfigProviderManagerOptArg("foo-dynamic-scoped-routes", rds_config_source_, + scope_key_builder)); + + Protobuf::RepeatedPtrField resources; + resources.Add()->MergeFrom(parseScopedRouteConfigurationFromYaml(R"EOF( +name: dynamic-foo +route_configuration_name: dynamic-foo-route-config +key: + fragments: { string_key: "172.30.30.10" } +)EOF")); + + timeSystem().setSystemTime(std::chrono::milliseconds(1234567891567)); + ScopedRdsConfigSubscription& subscription = + dynamic_cast(*dynamic_provider).subscription(); + subscription.onConfigUpdate(resources, "1"); + + MessageUtil::loadFromYaml(R"EOF( +inline_scoped_route_configs: + - name: foo-scoped-routes + scoped_route_configs: + - name: foo + route_configuration_name: foo-route-config + key: + fragments: { string_key: "172.10.10.10" } + - name: foo2 + route_configuration_name: foo-route-config2 + key: + fragments: { string_key: "172.10.10.20" } + last_updated: + seconds: 1234567891 + nanos: 234000000 +dynamic_scoped_route_configs: + - name: foo-dynamic-scoped-routes + scoped_route_configs: + - name: dynamic-foo + route_configuration_name: dynamic-foo-route-config + key: + fragments: { string_key: "172.30.30.10" } + last_updated: + seconds: 1234567891 + nanos: 567000000 + version_info: "1" +)EOF", + expected_config_dump); + message_ptr = factory_context_.admin_.config_tracker_.config_tracker_callbacks_["route_scopes"](); + const auto& scoped_routes_config_dump3 = + MessageUtil::downcastAndValidate( + *message_ptr); + EXPECT_EQ(expected_config_dump.DebugString(), scoped_routes_config_dump3.DebugString()); } } // namespace diff --git a/test/integration/scoped_rds_integration_test.cc b/test/integration/scoped_rds_integration_test.cc index 92fefb63a7301..0baeae2a6d340 100644 --- a/test/integration/scoped_rds_integration_test.cc +++ b/test/integration/scoped_rds_integration_test.cc @@ -54,6 +54,7 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, scope_key_builder; MessageUtil::loadFromYaml(scope_key_builder_config_yaml, scope_key_builder); auto* scoped_routes = http_connection_manager.mutable_scoped_routes(); + scoped_routes->set_name("foo-scoped-routes"); *scoped_routes->mutable_scope_key_builder() = scope_key_builder; envoy::api::v2::core::ApiConfigSource* rds_api_config_source = @@ -169,10 +170,11 @@ route_configuration_name: foo_route2 }; initialize(); - test_server_->waitForCounterGe("http.config_test.scoped_rds.update_attempt", 1); - test_server_->waitForCounterGe("http.config_test.scoped_rds.update_success", 1); + test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_attempt", 1); + test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_success", 1); // The version gauge should be set to xxHash64("1"). - test_server_->waitForGaugeEq("http.config_test.scoped_rds.version", 13237225503670494420UL); + test_server_->waitForGaugeEq("http.config_test.scoped_rds.foo-scoped-routes.version", + 13237225503670494420UL); const std::string scope_route3 = R"EOF( name: foo_scope3 @@ -183,9 +185,10 @@ route_configuration_name: foo_route3 )EOF"; sendScopedRdsResponse({scope_route3}, "2"); - test_server_->waitForCounterGe("http.config_test.scoped_rds.update_attempt", 2); - test_server_->waitForCounterGe("http.config_test.scoped_rds.update_success", 2); - test_server_->waitForGaugeEq("http.config_test.scoped_rds.version", 6927017134761466251UL); + test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_attempt", 2); + test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_success", 2); + test_server_->waitForGaugeEq("http.config_test.scoped_rds.foo-scoped-routes.version", + 6927017134761466251UL); // TODO(AndresGuedez): test actual scoped routing logic; only the config handling is implemented // at this point. From b2160347bb2b232d4fc502738058366ea7a956ba Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Wed, 10 Apr 2019 09:20:39 -0400 Subject: [PATCH 17/37] Add comments. Signed-off-by: Andres Guedez --- api/envoy/api/v2/srds.proto | 92 +++++++++---------- .../v2/http_connection_manager.proto | 37 ++++---- include/envoy/config/config_provider.h | 42 +++++++-- source/common/router/scoped_rds.h | 2 - .../config/config_provider_impl_test.cc | 2 - 5 files changed, 97 insertions(+), 78 deletions(-) diff --git a/api/envoy/api/v2/srds.proto b/api/envoy/api/v2/srds.proto index 8dcd086b3fcae..53d41245dfb3b 100644 --- a/api/envoy/api/v2/srds.proto +++ b/api/envoy/api/v2/srds.proto @@ -27,10 +27,11 @@ option (gogoproto.equal_all) = true; // TODO(AndresGuedez): Update :ref:`arch_overview_http_routing` with scoped routing overview and // configuration details. -// The `ref:resource_names` field in -// DiscoveryRequest specifies a set of route configuration scopes. Each scope points to a -// route_configuration_name which will be used to request a -// :ref:`RouteConfiguration` via the RDS API. +// The Scoped Routes Discovery Service (SRDS) API distributes +// :ref:`ScopedRouteConfiguration` resources. Each +// ScopedRouteConfiguration resource represents a "routing scope" containing a mapping that allows +// the HTTP connection manager to dynamically assign a routing table (specified via +// a :ref:`RouteConfiguration` message) to each HTTP request. service ScopedRoutesDiscoveryService { rpc StreamScopedRoutes(stream DiscoveryRequest) returns (stream DiscoveryResponse) { } @@ -47,33 +48,22 @@ service ScopedRoutesDiscoveryService { } } -// This configuration represents a set of routing 'scopes', each containing independent routing -// configuration (see :ref:`routing architecture overview `). -// -// A ScopedRouteConfigurationsSet is associated with an -// :ref:`HttpConnectionManager` -// via the -// :ref:`scoped_rds` -// or -// :ref:`scoped_routes_config` -// fields. When scoped routing is enabled, the Envoy router will first attempt to determine the -// routing scope assigned to a request by building a -// :ref:`Scope.Key` based on request header -// attributes (such as the value of a header field) defined by the -// :ref:`ScopeKeyBuilder` -// this message. -// -// The key will be looked up in a table maintained by the router, containing the -// :ref:`Scope.Key` to -// :ref:`RouteConfiguration` mappings specified by the -// :ref:`scopes` field in this message. The -// routing rules specified by the resulting RouteConfiguration will be used to determine the route -// for the request. -// -// For example, with the following YAML configuration: +// Specifies a routing scope, which associates a :ref:`envoy_api_msg_RouteConfiguration` (identified +// by its resource name) to a :ref:`Key`. +// +// The HTTP connection manager builds up a table consisting of these Key to RouteConfiguration +// mappings, and looks up the RouteConfiguration to use per request according to the algorithm +// specified in the +// :ref:`scope_key_builder` +// assigned to the HttpConnectionManager. +// +// For example, with the following configurations (in YAML): +// +// HttpConnectionManager config: // // .. code:: // +// ... // scope_key_builder: // fragments: // - header_element: @@ -82,15 +72,26 @@ service ScopedRoutesDiscoveryService { // element: // separator: = // key: vip -// scopes: -// - route_configuration_name: route-config1 -// key: -// fragments: -// - string_key: 172.10.10.20 -// - route_configuration_name: route-config2 -// key: -// fragments: -// - string_key: 172.20.20.30 +// +// ScopedRouteConfiguration resources (specified statically via +// HttpConnectionManager.ScopedRoutes.scoped_route_configurations_list or obtained dynamically via +// SRDS): +// +// .. code:: +// +// (1) +// name: route-scope1 +// route_configuration_name: route-config1 +// key: +// fragments: +// - string_key: 172.10.10.20 +// +// (2) +// name: route-scope2 +// route_configuration_name: route-config2 +// key: +// fragments: +// - string_key: 172.20.20.30 // // A request from a client such as: // @@ -100,25 +101,18 @@ service ScopedRoutesDiscoveryService { // Host: foo.com // X-Route-Selector: vip=172.10.10.20 // -// Would result in the `route-config1` RouteConfiguration being assigned to the request/stream, -// which Envoy would fetch via RDS. +// Would result in the routing table defined by the `route-config1` RouteConfiguration being +// assigned to the HTTP request/stream. // // [#comment:next free field: 4] - -// TODO(AndresGuedez): -// Specifies a routing scope, which associates a :ref:`envoy_api_msg_RouteConfiguration` -// (identified by its resource name) to a -// :ref:`Key` which is matched against each -// HTTP request. message ScopedRouteConfiguration { - // TODO(AndresGuedez): // The name assigned to the routing scope. string name = 1 [(validate.rules).string.min_bytes = 1]; // Specifies a key which is matched against the output of the - // :ref:`scope_key_builder` to - // obtain the route_configuration_name/RouteConfiguration associated with the Scope. The - // matching is dependent on the order of the fragments contained in the Key. + // :ref:`scope_key_builder` + // specified in the HttpConnectionManager. The matching is done per HTTP request and is dependent + // on the order of the fragments contained in the Key. message Key { message Fragment { oneof type { diff --git a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto index 043e750121ebd..e91cbefab5892 100644 --- a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto +++ b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto @@ -64,7 +64,7 @@ message HttpConnectionManager { envoy.api.v2.RouteConfiguration route_config = 4; // A route table will be dynamically assigned to each request based on request attributes - // (e.g., the value of a header). The "routing scopes" (i.e., route tables) and scope keys are + // (e.g., the value of a header). The "routing scopes" (i.e., route tables) and "scope keys" are // specified in this message. ScopedRoutes scoped_routes = 30; } @@ -409,18 +409,20 @@ message ScopedRouteConfigurationsList { } message ScopedRoutes { + // The name assigned to the scoped routing configuration. string name = 1 [(validate.rules).string.min_bytes = 1]; - // TODO(AndresGuedez): - // Specifies the mechanism for constructing keys based on request attributes to match - // :ref:`scopes ` against. + // Specifies the mechanism for constructing "scope keys" based on HTTP request attributes. These + // keys are matched against a set of :ref:`Key` + // objects assembled from :ref:`ScopedRouteConfiguration` + // messages distributed via SRDS (the Scoped Route Discovery Service) or assigned statically via + // :ref:`scoped_route_configurations_list`. // // Upon receiving a request's headers, the Router will build a key using the algorithm specified - // by this message. This key will be used to look up a corresponding - // :ref:`Scope ` in a table containing the set - // of :ref:`scopes ` in this message. + // by this message. This key will be used to look up the routing table (i.e., the + // :ref:`RouteConfiguration`) to use for the request. message ScopeKeyBuilder { - // Specifies the mechanism for constructing fragments which are composed into keys. + // Specifies the mechanism for constructing key fragments which are composed into scope keys. message FragmentBuilder { // Specifies how the value of a header should be extracted. // The following example maps the structure of a header to the fields in this message. @@ -471,10 +473,11 @@ message ScopedRoutes { } } - // The constructed key consists of the ordered union of these fragments. + // The final scope key consists of the ordered union of these fragments. repeated FragmentBuilder fragments = 1 [(validate.rules).repeated .min_items = 1]; } + // The algorithm to use for constructing a scope key for each request. ScopeKeyBuilder scope_key_builder = 2 [(validate.rules).message.required = true]; // Configuration source specifier for RDS. @@ -488,12 +491,16 @@ message ScopedRoutes { // The set of routing scopes corresponding to the HCM. A scope is assigned to a request by // matching a key constructed from the request's attributes according to the algorithm specified - // by the :ref:`ScopeKeyBuilder` in this message. + // by the + // :ref:`ScopeKeyBuilder` + // in this message. ScopedRouteConfigurationsList scoped_route_configurations_list = 4; - // The connection manager's route table will be dynamically loaded via the SRDS API; the - // :ref:`` received via - // SRDS will specify a set of routing scopes, each containing . + // The set of routing scopes associated with the HCM will be dynamically loaded via the SRDS + // API. A scope is assigned to a request by matching a key constructed from the request's + // attributes according to the algorithm specified by the + // :ref:`ScopeKeyBuilder` + // in this message. ScopedRds scoped_rds = 5; } } @@ -502,10 +509,6 @@ message ScopedRds { // Configuration source specifier for scoped RDS. envoy.api.v2.core.ConfigSource scoped_rds_config_source = 1 [(validate.rules).message.required = true, (gogoproto.nullable) = false]; - - // This is a hint to the xDS server containing the list of resources that Envoy is interested in - // tracking updates for. - // repeated string scoped_route_config_names = 3; } message HttpFilter { diff --git a/include/envoy/config/config_provider.h b/include/envoy/config/config_provider.h index 89ecdfb2f4569..0b500b9cf9180 100644 --- a/include/envoy/config/config_provider.h +++ b/include/envoy/config/config_provider.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include "envoy/common/time.h" @@ -42,9 +43,21 @@ class ConfigProvider { using ConfigConstSharedPtr = std::shared_ptr; /** - * + * The type of API represented by a ConfigProvider. */ - enum class ApiType { Full, Delta }; + enum class ApiType { + /** + * A "Full" API delivers a complete configuration as part of each resource (top level + * config proto); i.e., each resource contains the whole representation of the config intent. An + * example of this type of API is RDS. + */ + Full, + /** + * A "Delta" API delivers a subset of the config intent as part of each resource (top level + * config proto). Examples of this type of API are CDS, LDS and SRDS. + */ + Delta + }; /** * Stores the config proto as well as the associated version. @@ -58,7 +71,7 @@ class ConfigProvider { using ConfigProtoVector = std::vector; /** - * + * Stores the config protos associated with a "Delta" API. */ struct ConfigProtoInfoVector { const ConfigProtoVector config_protos_; @@ -70,12 +83,12 @@ class ConfigProvider { virtual ~ConfigProvider() = default; /** - * + * The type of API. */ virtual ApiType apiType() const PURE; /** - * Returns a ConfigProtoInfo associated with the provider. + * Returns a ConfigProtoInfo associated with a ApiType::Full provider. * @return absl::optional> an optional ConfigProtoInfo; the value is set when a * config is available. */ @@ -90,6 +103,11 @@ class ConfigProvider { return ConfigProtoInfo

{*config_proto, getConfigVersion()}; } + /** + * Returns a ConfigProtoInfoVector associated with a ApiType::Delta provider. + * @return absl::optional an optional ConfigProtoInfoVector; the value is + * set when a config is available. + */ absl::optional configProtoInfoVector() const { const ConfigProtoVector config_protos = getConfigProtos(); if (config_protos.empty()) { @@ -121,12 +139,20 @@ class ConfigProvider { * @return Protobuf::Message* the config proto corresponding to the Config instantiated by the * provider. */ - virtual const Protobuf::Message* getConfigProto() const PURE; + virtual const Protobuf::Message* getConfigProto() const { + assert("not implemented" && false); + return nullptr; + } /** - * + * Returns the config protos associated with the provider. + * @return const ConfigProtoVector the config protos corresponding to the Config instantiated by + * the provider. */ - virtual const ConfigProtoVector getConfigProtos() const PURE; + virtual const ConfigProtoVector getConfigProtos() const { + assert("not implemented" && false); + return {}; + } /** * Returns the config version associated with the provider. diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index e695994558dd9..26f0ab8a495f2 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -44,7 +44,6 @@ class InlineScopedRoutesConfigProvider : public Envoy::Config::ImmutableConfigPr const std::string& name() const { return name_; } // Envoy::Config::ConfigProvider - const Protobuf::Message* getConfigProto() const override { return nullptr; } const Envoy::Config::ConfigProvider::ConfigProtoVector getConfigProtos() const override { Envoy::Config::ConfigProvider::ConfigProtoVector out_protos; std::for_each(config_protos_.begin(), config_protos_.end(), @@ -141,7 +140,6 @@ class ScopedRdsConfigProvider : public Envoy::Config::MutableConfigProviderImplB } // Envoy::Config::ConfigProvider - const Protobuf::Message* getConfigProto() const override { return nullptr; } const Envoy::Config::ConfigProvider::ConfigProtoVector getConfigProtos() const override { const ScopedConfigManager::ScopedRouteMap& scoped_route_map = subscription_->scopedRouteMap(); if (scoped_route_map.empty()) { diff --git a/test/common/config/config_provider_impl_test.cc b/test/common/config/config_provider_impl_test.cc index 30c111e52413b..debd2a01f3c40 100644 --- a/test/common/config/config_provider_impl_test.cc +++ b/test/common/config/config_provider_impl_test.cc @@ -26,7 +26,6 @@ class StaticDummyConfigProvider : public ImmutableConfigProviderImplBase { // Envoy::Config::ConfigProvider const Protobuf::Message* getConfigProto() const override { return &config_proto_; } - const std::vector getConfigProtos() const override { return {}; } // Envoy::Config::ConfigProvider std::string getConfigVersion() const override { return ""; } @@ -112,7 +111,6 @@ class DummyDynamicConfigProvider : public MutableConfigProviderImplBase { } return &subscription_->config_proto().value(); } - const std::vector getConfigProtos() const override { return {}; } // Envoy::Config::ConfigProvider std::string getConfigVersion() const override { return ""; } From 39309acce148338dcc46833c84ec452f9b711eed Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Fri, 12 Apr 2019 12:07:32 -0400 Subject: [PATCH 18/37] Fix build failures after master merge. Signed-off-by: Andres Guedez --- api/envoy/api/v2/srds.proto | 4 +-- source/common/router/scoped_rds.cc | 28 ++++++++++------ source/common/router/scoped_rds.h | 17 ++++++---- test/common/http/conn_manager_impl_common.h | 2 +- test/common/router/scoped_rds_test.cc | 33 +++++++++---------- .../http_connection_manager/config_test.cc | 19 +++++++---- .../scoped_rds_integration_test.cc | 2 +- 7 files changed, 60 insertions(+), 45 deletions(-) diff --git a/api/envoy/api/v2/srds.proto b/api/envoy/api/v2/srds.proto index 53d41245dfb3b..e85768601d82b 100644 --- a/api/envoy/api/v2/srds.proto +++ b/api/envoy/api/v2/srds.proto @@ -36,8 +36,8 @@ service ScopedRoutesDiscoveryService { rpc StreamScopedRoutes(stream DiscoveryRequest) returns (stream DiscoveryResponse) { } - rpc IncrementalScopedRoutes(stream IncrementalDiscoveryRequest) - returns (stream IncrementalDiscoveryResponse) { + rpc DeltaScopedRoutes(stream DeltaDiscoveryRequest) + returns (stream DeltaDiscoveryResponse) { } rpc FetchScopedRoutes(DiscoveryRequest) returns (DiscoveryResponse) { diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index c03e7fb3a8310..1b9f2dffadea3 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -83,7 +83,7 @@ InlineScopedRoutesConfigProvider::InlineScopedRoutesConfigProvider( ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( const envoy::config::filter::network::http_connection_manager::v2::ScopedRds& scoped_rds, - const std::string& manager_identifier, const std::string& name, + const uint64_t manager_identifier, const std::string& name, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, ScopedRoutesConfigProviderManager& config_provider_manager) : ConfigSubscriptionInstanceBase( @@ -92,31 +92,39 @@ ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( name_(name), scope_(factory_context.scope().createScope(stat_prefix + "scoped_rds." + name + ".")), stats_({ALL_SCOPED_RDS_STATS(POOL_COUNTER(*scope_))}) { - subscription_ = Envoy::Config::SubscriptionFactory::subscriptionFromConfigSource< - envoy::api::v2::ScopedRouteConfiguration>( + subscription_ = Envoy::Config::SubscriptionFactory::subscriptionFromConfigSource( scoped_rds.scoped_rds_config_source(), factory_context.localInfo(), factory_context.dispatcher(), factory_context.clusterManager(), factory_context.random(), *scope_, "envoy.api.v2.ScopedRoutesDiscoveryService.FetchScopedRoutes", - "envoy.api.v2.ScopedRoutesDiscoveryService.StreamScopedRoutes", factory_context.api()); + "envoy.api.v2.ScopedRoutesDiscoveryService.StreamScopedRoutes", + Grpc::Common::typeUrl( + envoy::api::v2::ScopedRouteConfiguration().GetDescriptor()->full_name()), + factory_context.api()); } -void ScopedRdsConfigSubscription::onConfigUpdate(const ResourceVector& resources, - const std::string& version_info) { +void ScopedRdsConfigSubscription::onConfigUpdate( + const Protobuf::RepeatedPtrField& resources, + const std::string& version_info) { if (resources.empty()) { ENVOY_LOG(debug, "Empty resources in scoped RDS onConfigUpdate()"); stats_.update_empty_.inc(); ConfigSubscriptionInstanceBase::onConfigUpdateFailed(); return; } + std::vector scoped_routes; + for (const auto& resource_any : resources) { + scoped_routes.emplace_back( + MessageUtil::anyConvert(resource_any)); + } std::unordered_set resource_names; - for (const auto& scoped_route : resources) { + for (const auto& scoped_route : scoped_routes) { if (!resource_names.insert(scoped_route.name()).second) { throw EnvoyException( fmt::format("duplicate scoped route configuration {} found", scoped_route.name())); } } - for (const auto& scoped_route : resources) { + for (const auto& scoped_route : scoped_routes) { MessageUtil::validate(scoped_route); } @@ -125,7 +133,7 @@ void ScopedRdsConfigSubscription::onConfigUpdate(const ResourceVector& resources // We need to keep track of which scoped routes we might need to remove. ScopedConfigManager::ScopedRouteMap scoped_routes_to_remove = scoped_config_manager_.scopedRouteMap(); - for (auto& scoped_route : resources) { + for (auto& scoped_route : scoped_routes) { const std::string scoped_route_name = scoped_route.name(); try { scoped_routes_to_remove.erase(scoped_route_name); @@ -233,7 +241,7 @@ ConfigProviderPtr ScopedRoutesConfigProviderManager::createXdsConfigProvider( ConfigProviderManagerImplBase::getSubscription( config_source_proto, factory_context.initManager(), [&config_source_proto, &factory_context, &stat_prefix, - &optarg](const std::string& manager_identifier, + &optarg](const uint64_t manager_identifier, ConfigProviderManagerImplBase& config_provider_manager) -> Envoy::Config::ConfigSubscriptionInstanceBaseSharedPtr { const auto& scoped_rds_config_source = dynamic_cast< diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index 26f0ab8a495f2..8257b425cf57c 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -78,16 +78,15 @@ struct ScopedRdsStats { }; // A scoped RDS subscription to be used with the dynamic scoped RDS ConfigProvider. -class ScopedRdsConfigSubscription - : public Envoy::Config::ConfigSubscriptionInstanceBase, - Envoy::Config::SubscriptionCallbacks { +class ScopedRdsConfigSubscription : public Envoy::Config::ConfigSubscriptionInstanceBase, + Envoy::Config::SubscriptionCallbacks { public: using ScopedRouteConfigurationMap = std::map; ScopedRdsConfigSubscription( const envoy::config::filter::network::http_connection_manager::v2::ScopedRds& scoped_rds, - const std::string& manager_identifier, const std::string& name, + const uint64_t manager_identifier, const std::string& name, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, ScopedRoutesConfigProviderManager& config_provider_manager); @@ -99,7 +98,12 @@ class ScopedRdsConfigSubscription void start() override { subscription_->start({}, *this); } // Envoy::Config::SubscriptionCallbacks - void onConfigUpdate(const ResourceVector& resources, const std::string& version_info) override; + void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, + const std::string& version_info) override; + void onConfigUpdate(const Protobuf::RepeatedPtrField&, + const Protobuf::RepeatedPtrField&, const std::string&) override { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; + } void onConfigUpdateFailed(const EnvoyException*) override { ConfigSubscriptionInstanceBase::onConfigUpdateFailed(); } @@ -112,8 +116,7 @@ class ScopedRdsConfigSubscription private: const std::string name_; - std::unique_ptr> - subscription_; + std::unique_ptr subscription_; Stats::ScopePtr scope_; ScopedRdsStats stats_; ScopedConfigManager scoped_config_manager_; diff --git a/test/common/http/conn_manager_impl_common.h b/test/common/http/conn_manager_impl_common.h index 54bd2415cefe1..1bf4cf01dc6ba 100644 --- a/test/common/http/conn_manager_impl_common.h +++ b/test/common/http/conn_manager_impl_common.h @@ -39,7 +39,7 @@ struct ScopedRouteConfigProvider : public Config::ConfigProvider { // Config::ConfigProvider SystemTime lastUpdated() const override { return time_source_.systemTime(); } const Protobuf::Message* getConfigProto() const override { return nullptr; } - const Envoy::Config::ConfigProvider::ConfigProviderVector getConfigProtos() const override { + const Envoy::Config::ConfigProvider::ConfigProtoVector getConfigProtos() const override { return {}; } std::string getConfigVersion() const override { return ""; } diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index 4e4ffc2356f43..e254437e8c83f 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -8,7 +8,6 @@ #include "test/mocks/server/mocks.h" #include "test/test_common/simulated_time_system.h" -#include "test/test_common/test_base.h" #include "absl/strings/string_view.h" #include "gmock/gmock.h" @@ -21,18 +20,18 @@ namespace Envoy { namespace Router { namespace { -void parseScopedRouteConfigurationFromYaml( - envoy::api::v2::ScopedRouteConfiguration& scoped_route_config, const std::string& yaml) { - MessageUtil::loadFromYaml(yaml, scoped_route_config); -} - envoy::api::v2::ScopedRouteConfiguration parseScopedRouteConfigurationFromYaml(const std::string& yaml) { envoy::api::v2::ScopedRouteConfiguration scoped_route_config; - parseScopedRouteConfigurationFromYaml(scoped_route_config, yaml); + MessageUtil::loadFromYaml(yaml, scoped_route_config); return scoped_route_config; } +void parseScopedRouteConfigurationFromYaml(ProtobufWkt::Any& scoped_route_config, + const std::string& yaml) { + scoped_route_config.PackFrom(parseScopedRouteConfigurationFromYaml(yaml)); +} + std::vector> protosToMessageVec(std::vector&& protos) { std::vector> messages; @@ -44,7 +43,7 @@ protosToMessageVec(std::vector&& proto return messages; } -class ScopedRoutesTestBase : public TestBase { +class ScopedRoutesTestBase : public testing::Test { protected: ScopedRoutesTestBase() { EXPECT_CALL(factory_context_.admin_.config_tracker_, add_("route_scopes", _)); @@ -77,7 +76,7 @@ class ScopedRoutesTestBase : public TestBase { NiceMock factory_context_; Upstream::ClusterManager::ClusterInfoMap cluster_map_; - Upstream::MockCluster cluster_; + Upstream::MockClusterMockPrioritySet cluster_; std::unique_ptr config_provider_manager_; Event::SimulatedTimeSystem time_system_; envoy::api::v2::core::ConfigSource rds_config_source_; @@ -142,7 +141,7 @@ route_configuration_name: foo_routes fragments: - string_key: x-foo-key )EOF"; - Protobuf::RepeatedPtrField resources; + Protobuf::RepeatedPtrField resources; parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); EXPECT_THROW(subscription.onConfigUpdate(resources, "1"), ProtoValidationException); @@ -154,7 +153,7 @@ name: foo_scope fragments: - string_key: x-foo-key )EOF"; - Protobuf::RepeatedPtrField resources2; + Protobuf::RepeatedPtrField resources2; parseScopedRouteConfigurationFromYaml(*resources2.Add(), config_yaml2); EXPECT_THROW(subscription.onConfigUpdate(resources2, "1"), ProtoValidationException); @@ -164,7 +163,7 @@ name: foo_scope route_configuration_name: foo_routes key: )EOF"; - Protobuf::RepeatedPtrField resources3; + Protobuf::RepeatedPtrField resources3; parseScopedRouteConfigurationFromYaml(*resources3.Add(), config_yaml3); EXPECT_THROW(subscription.onConfigUpdate(resources3, "1"), ProtoValidationException); } @@ -173,7 +172,7 @@ route_configuration_name: foo_routes TEST_F(ScopedRdsTest, EmptyResource) { setup(); - Protobuf::RepeatedPtrField resources; + Protobuf::RepeatedPtrField resources; subscription().onConfigUpdate(resources, "1"); EXPECT_EQ( 1UL, @@ -191,7 +190,7 @@ route_configuration_name: foo_routes fragments: - string_key: x-foo-key )EOF"; - Protobuf::RepeatedPtrField resources; + Protobuf::RepeatedPtrField resources; parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); const std::string config_yaml2 = R"EOF( name: foo_scope2 @@ -218,7 +217,7 @@ route_configuration_name: foo_routes fragments: - string_key: x-foo-key )EOF"; - Protobuf::RepeatedPtrField resources; + Protobuf::RepeatedPtrField resources; parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); parseScopedRouteConfigurationFromYaml(*resources.Add(), config_yaml); EXPECT_THROW_WITH_MESSAGE(subscription().onConfigUpdate(resources, "1"), EnvoyException, @@ -359,8 +358,8 @@ route_configuration_name: foo-route-config2 ScopedRoutesConfigProviderManagerOptArg("foo-dynamic-scoped-routes", rds_config_source_, scope_key_builder)); - Protobuf::RepeatedPtrField resources; - resources.Add()->MergeFrom(parseScopedRouteConfigurationFromYaml(R"EOF( + Protobuf::RepeatedPtrField resources; + resources.Add()->PackFrom(parseScopedRouteConfigurationFromYaml(R"EOF( name: dynamic-foo route_configuration_name: dynamic-foo-route-config key: diff --git a/test/extensions/filters/network/http_connection_manager/config_test.cc b/test/extensions/filters/network/http_connection_manager/config_test.cc index 43bc075a64ecf..9668b52cb22d9 100644 --- a/test/extensions/filters/network/http_connection_manager/config_test.cc +++ b/test/extensions/filters/network/http_connection_manager/config_test.cc @@ -68,7 +68,7 @@ stat_prefix: router )EOF"; EXPECT_THROW_WITH_MESSAGE( - HttpConnectionManagerConfig(parseHttpConnectionManagerFromJson(yaml_string), context_, + HttpConnectionManagerConfig(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, date_provider_, route_config_provider_manager_, scoped_routes_config_provider_manager_), EnvoyException, "Didn't find a registered implementation for name: 'foo'"); @@ -98,7 +98,7 @@ stat_prefix: router config: {} )EOF"; - HttpConnectionManagerConfig config(parseHttpConnectionManagerFromJson(yaml_string), context_, + HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, date_provider_, route_config_provider_manager_, scoped_routes_config_provider_manager_); @@ -173,7 +173,8 @@ TEST_F(HttpConnectionManagerConfigTest, MaxRequestHeadersKbMaxConfigurable) { )EOF"; HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, - date_provider_, route_config_provider_manager_); + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); EXPECT_EQ(96, config.maxRequestHeadersKb()); } @@ -205,7 +206,8 @@ TEST_F(HttpConnectionManagerConfigTest, NormalizePathDefault) { )EOF"; HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, - date_provider_, route_config_provider_manager_); + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); EXPECT_FALSE(config.shouldNormalizePath()); } @@ -223,7 +225,8 @@ TEST_F(HttpConnectionManagerConfigTest, NormalizePathRuntime) { featureEnabled("http_connection_manager.normalize_path", 0)) .WillOnce(Return(true)); HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, - date_provider_, route_config_provider_manager_); + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); EXPECT_TRUE(config.shouldNormalizePath()); } @@ -242,7 +245,8 @@ TEST_F(HttpConnectionManagerConfigTest, NormalizePathTrue) { featureEnabled("http_connection_manager.normalize_path", 0)) .Times(0); HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, - date_provider_, route_config_provider_manager_); + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); EXPECT_TRUE(config.shouldNormalizePath()); } @@ -261,7 +265,8 @@ TEST_F(HttpConnectionManagerConfigTest, NormalizePathFalse) { featureEnabled("http_connection_manager.normalize_path", 0)) .Times(0); HttpConnectionManagerConfig config(parseHttpConnectionManagerFromV2Yaml(yaml_string), context_, - date_provider_, route_config_provider_manager_); + date_provider_, route_config_provider_manager_, + scoped_routes_config_provider_manager_); EXPECT_FALSE(config.shouldNormalizePath()); } diff --git a/test/integration/scoped_rds_integration_test.cc b/test/integration/scoped_rds_integration_test.cc index 0baeae2a6d340..10e3494b803ee 100644 --- a/test/integration/scoped_rds_integration_test.cc +++ b/test/integration/scoped_rds_integration_test.cc @@ -164,7 +164,7 @@ route_configuration_name: foo_route2 - string_key: x-foo-key )EOF"; - pre_worker_start_test_steps_ = [this, &scope_route1, &scope_route2]() { + on_server_init_function_ = [this, &scope_route1, &scope_route2]() { createScopedRdsStream(); sendScopedRdsResponse({scope_route1, scope_route2}, "1"); }; From 8fe33ad8cc2cedfa464916d7c799cbd9020fb2aa Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Fri, 12 Apr 2019 15:37:34 -0400 Subject: [PATCH 19/37] Cleanup and comments. Signed-off-by: Andres Guedez --- source/common/config/config_provider_impl.h | 8 ++- source/common/router/BUILD | 10 ++-- source/common/router/scoped_config_impl.cc | 16 ------ source/common/router/scoped_config_impl.h | 44 ++++++----------- source/common/router/scoped_config_manager.cc | 24 +++++++++ source/common/router/scoped_config_manager.h | 49 +++++++++++++++++++ .../router/thread_local_scoped_config.h | 35 ------------- 7 files changed, 96 insertions(+), 90 deletions(-) create mode 100644 source/common/router/scoped_config_manager.cc create mode 100644 source/common/router/scoped_config_manager.h delete mode 100644 source/common/router/thread_local_scoped_config.h diff --git a/source/common/config/config_provider_impl.h b/source/common/config/config_provider_impl.h index a7c3dec72ed2f..c830c10648c07 100644 --- a/source/common/config/config_provider_impl.h +++ b/source/common/config/config_provider_impl.h @@ -213,7 +213,7 @@ class ConfigSubscriptionInstanceBase : protected Logger::Loggable updateFn); @@ -290,9 +290,6 @@ class MutableConfigProviderImplBase : public ConfigProvider { }); } - /** - * TODO(AndresGuedez): - */ virtual void initialize(ThreadLocal::Slot::InitializeCb initializeCb) { subscription_->bindConfigProvider(this); tls_->set(initializeCb); @@ -311,7 +308,8 @@ class MutableConfigProviderImplBase : public ConfigProvider { } /** - * TODO(AndresGuedez): + * Propagates a delta config update to all workers. + * @param updateCb the callback to run on each worker. */ virtual void onDeltaConfigUpdate(Envoy::Event::PostCb updateCb) { tls_->runOnAllThreads(updateCb); diff --git a/source/common/router/BUILD b/source/common/router/BUILD index 2e75d9c54fd1c..bfb3351d232f7 100644 --- a/source/common/router/BUILD +++ b/source/common/router/BUILD @@ -94,10 +94,11 @@ envoy_cc_library( ) envoy_cc_library( - name = "thread_local_scoped_config_lib", - hdrs = ["thread_local_scoped_config.h"], + name = "scoped_config_manager_lib", + srcs = ["scoped_config_manager.cc"], + hdrs = ["scoped_config_manager.h"], deps = [ - "//include/envoy/thread_local:thread_local_interface", + "@envoy_api//envoy/api/v2:srds_cc", ], ) @@ -107,8 +108,9 @@ envoy_cc_library( hdrs = ["scoped_config_impl.h"], deps = [ ":config_lib", - ":thread_local_scoped_config_lib", + ":scoped_config_manager_lib", "//include/envoy/router:scopes_interface", + "//include/envoy/thread_local:thread_local_interface", "@envoy_api//envoy/api/v2:srds_cc", "@envoy_api//envoy/config/filter/network/http_connection_manager/v2:http_connection_manager_cc", ], diff --git a/source/common/router/scoped_config_impl.cc b/source/common/router/scoped_config_impl.cc index ae20ea6a37484..2a715811bd1b7 100644 --- a/source/common/router/scoped_config_impl.cc +++ b/source/common/router/scoped_config_impl.cc @@ -1,24 +1,8 @@ #include "common/router/scoped_config_impl.h" -#include "common/router/config_impl.h" - namespace Envoy { namespace Router { -ScopedRouteInfoConstSharedPtr ScopedConfigManager::addOrUpdateRoutingScope( - const envoy::api::v2::ScopedRouteConfiguration& config_proto, const std::string&) { - auto scoped_route_info = std::make_shared(config_proto); - scoped_route_map_[config_proto.name()] = scoped_route_info; - return scoped_route_info; -} - -bool ScopedConfigManager::removeRoutingScope(const std::string& name) { - if (scoped_route_map_.erase(name) == 0) { - throw EnvoyException(fmt::format("could not find {} in scoped route map for removal", name)); - } - return true; -} - void ThreadLocalScopedConfigImpl::addOrUpdateRoutingScope(ScopedRouteInfoConstSharedPtr) {} void ThreadLocalScopedConfigImpl::removeRoutingScope(const std::string&) {} diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h index e003e501172a0..8623c42939aea 100644 --- a/source/common/router/scoped_config_impl.h +++ b/source/common/router/scoped_config_impl.h @@ -4,49 +4,33 @@ #include "envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.pb.h" #include "envoy/router/router.h" #include "envoy/router/scopes.h" +#include "envoy/thread_local/thread_local.h" #include "common/router/config_impl.h" -#include "common/router/thread_local_scoped_config.h" +#include "common/router/scoped_config_manager.h" namespace Envoy { namespace Router { -// TODO(AndresGuedez): -class ScopedConfigManager { -public: - using ScopedRouteMap = std::map; - - // Adds/updates a routing scope specified via the Scoped RDS API. This scope will be added to the - // set of scopes matched against the scope keys built for each HTTP request. - ScopedRouteInfoConstSharedPtr - addOrUpdateRoutingScope(const envoy::api::v2::ScopedRouteConfiguration& scoped_route_config, - const std::string& version_info); - - // Removes a routing scope from the set of scopes matched against each HTTP request. - bool removeRoutingScope(const std::string& scope_name); - - const ScopedRouteMap& scopedRouteMap() const { return scoped_route_map_; } - -private: - ScopedRouteMap scoped_route_map_; -}; - /** - * The implementation of scoped routing configuration logic. - * NOTE: This is not yet implemented and simply mimics the behavior of the - * NullScopedConfigImpl. - * * TODO(AndresGuedez): implement scoped routing logic. + * + * Each Envoy worker is assigned an instance of this type. When config updates are received, + * addOrUpdateRoutingScope() and removeRoutingScope() are called to update the set of scoped routes. + * + * ConnectionManagerImpl::refreshCachedRoute() will call getRouterConfig() to obtain the + * Router::ConfigConstSharedPtr to use for route selection. */ -class ThreadLocalScopedConfigImpl : public ScopedConfig, public ThreadLocalScopedConfig { +class ThreadLocalScopedConfigImpl : public ScopedConfig, public ThreadLocal::ThreadLocalObject { public: ThreadLocalScopedConfigImpl(const envoy::config::filter::network::http_connection_manager::v2:: ScopedRoutes::ScopeKeyBuilder& scope_key_builder) : scope_key_builder_(scope_key_builder) {} - // Envoy::Router::ThreadLocalScopedConfig - void addOrUpdateRoutingScope(ScopedRouteInfoConstSharedPtr scoped_route_info) override; - void removeRoutingScope(const std::string& scope_name) override; + virtual ~ThreadLocalScopedConfigImpl() = default; + + void addOrUpdateRoutingScope(ScopedRouteInfoConstSharedPtr scoped_route_info); + void removeRoutingScope(const std::string& scope_name); // Envoy::Router::ScopedConfig Router::ConfigConstSharedPtr getRouterConfig(const Http::HeaderMap& headers) const override; @@ -57,7 +41,7 @@ class ThreadLocalScopedConfigImpl : public ScopedConfig, public ThreadLocalScope }; /** - * An empty implementation of the scoped routing configuration. + * A NULL implementation of the scoped routing configuration. */ class NullScopedConfigImpl : public ScopedConfig { public: diff --git a/source/common/router/scoped_config_manager.cc b/source/common/router/scoped_config_manager.cc new file mode 100644 index 0000000000000..b29b533c105c2 --- /dev/null +++ b/source/common/router/scoped_config_manager.cc @@ -0,0 +1,24 @@ +#include "envoy/common/exception.h" + +#include "common/common/fmt.h" +#include "common/router/scoped_config_manager.h" + +namespace Envoy { +namespace Router { + +ScopedRouteInfoConstSharedPtr ScopedConfigManager::addOrUpdateRoutingScope( + const envoy::api::v2::ScopedRouteConfiguration& config_proto, const std::string&) { + auto scoped_route_info = std::make_shared(config_proto); + scoped_route_map_[config_proto.name()] = scoped_route_info; + return scoped_route_info; +} + +bool ScopedConfigManager::removeRoutingScope(const std::string& name) { + if (scoped_route_map_.erase(name) == 0) { + throw EnvoyException(fmt::format("could not find {} in scoped route map for removal", name)); + } + return true; +} + +} // namespace Router +} // namespace Envoy diff --git a/source/common/router/scoped_config_manager.h b/source/common/router/scoped_config_manager.h new file mode 100644 index 0000000000000..c13bc81ee8106 --- /dev/null +++ b/source/common/router/scoped_config_manager.h @@ -0,0 +1,49 @@ +#pragma once + +#include +#include + +#include "envoy/api/v2/srds.pb.h" + +namespace Envoy { +namespace Router { + +// The internal representation of the configuration distributed via the ScopedRouteConfiguration +// proto. +class ScopedRouteInfo { +public: + ScopedRouteInfo(const envoy::api::v2::ScopedRouteConfiguration& config_proto) + : config_proto_(config_proto) {} + + // TODO(AndresGuedez): Add the necessary APIs required for the scoped routing logic. + + const envoy::api::v2::ScopedRouteConfiguration config_proto_; +}; +using ScopedRouteInfoConstSharedPtr = std::shared_ptr; + +// A manager for routing configuration scopes. +// An instance of the manager is owned by each ScopedRdsConfigSubscription. When config updates are +// received (on the main thread), the manager is called to track changes to the set of scoped route +// configurations and build ScopedRouteInfo's as needed. +class ScopedConfigManager { +public: + // Ordered map for consistent config dumping. + using ScopedRouteMap = std::map; + + // Adds/updates a routing scope specified via the Scoped RDS API. This scope will be added to the + // set of scopes matched against the scope keys built for each HTTP request. + ScopedRouteInfoConstSharedPtr + addOrUpdateRoutingScope(const envoy::api::v2::ScopedRouteConfiguration& scoped_route_config, + const std::string& version_info); + + // Removes a routing scope from the set of scopes matched against each HTTP request. + bool removeRoutingScope(const std::string& scope_name); + + const ScopedRouteMap& scopedRouteMap() const { return scoped_route_map_; } + +private: + ScopedRouteMap scoped_route_map_; +}; + +} // namespace Router +} // namespace Envoy diff --git a/source/common/router/thread_local_scoped_config.h b/source/common/router/thread_local_scoped_config.h deleted file mode 100644 index 62cc2da808287..0000000000000 --- a/source/common/router/thread_local_scoped_config.h +++ /dev/null @@ -1,35 +0,0 @@ -#pragma once - -#include -#include - -#include "envoy/thread_local/thread_local.h" - -namespace Envoy { -namespace Router { - -// TODO(AndresGuedez): -class ScopedRouteInfo { -public: - ScopedRouteInfo(const envoy::api::v2::ScopedRouteConfiguration& config_proto) - : config_proto_(config_proto) {} - - const envoy::api::v2::ScopedRouteConfiguration config_proto_; -}; -using ScopedRouteInfoConstSharedPtr = std::shared_ptr; - -// TODO(AndresGuedez): -class ThreadLocalScopedConfig : public ThreadLocal::ThreadLocalObject { -public: - virtual ~ThreadLocalScopedConfig() {} - - // Adds/updates a routing scope specified via the Scoped RDS API. This scope will be added to the - // set of scopes matched against the scope keys built for each HTTP request. - virtual void addOrUpdateRoutingScope(ScopedRouteInfoConstSharedPtr scoped_route_info) PURE; - - // Removes a routing scope from the set of scopes matched against each HTTP request. - virtual void removeRoutingScope(const std::string& scope_name) PURE; -}; - -} // namespace Router -} // namespace Envoy From 8c3852682c6d55417fab4cd96cddd5cbd451b7d6 Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Fri, 12 Apr 2019 15:44:49 -0400 Subject: [PATCH 20/37] Fix format. Signed-off-by: Andres Guedez --- api/envoy/api/v2/srds.proto | 3 +- .../v2/http_connection_manager.proto | 10 +++--- include/envoy/config/config_provider.h | 1 + source/common/http/conn_manager_impl.cc | 34 +++++++++---------- source/common/router/scoped_config_manager.cc | 3 +- 5 files changed, 26 insertions(+), 25 deletions(-) diff --git a/api/envoy/api/v2/srds.proto b/api/envoy/api/v2/srds.proto index e85768601d82b..4574ff98e2547 100644 --- a/api/envoy/api/v2/srds.proto +++ b/api/envoy/api/v2/srds.proto @@ -36,8 +36,7 @@ service ScopedRoutesDiscoveryService { rpc StreamScopedRoutes(stream DiscoveryRequest) returns (stream DiscoveryResponse) { } - rpc DeltaScopedRoutes(stream DeltaDiscoveryRequest) - returns (stream DeltaDiscoveryResponse) { + rpc DeltaScopedRoutes(stream DeltaDiscoveryRequest) returns (stream DeltaDiscoveryResponse) { } rpc FetchScopedRoutes(DiscoveryRequest) returns (DiscoveryResponse) { diff --git a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto index 1bc0b9f373f98..56dee2cc630aa 100644 --- a/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto +++ b/api/envoy/config/filter/network/http_connection_manager/v2/http_connection_manager.proto @@ -405,11 +405,11 @@ message Rds { envoy.api.v2.core.ConfigSource config_source = 1 [(validate.rules).message.required = true, (gogoproto.nullable) = false]; - // The name of the route configuration. This name will be passed to the RDS - // API. This allows an Envoy configuration with multiple HTTP listeners (and - // associated HTTP connection manager filters) to use different route - // configurations. - string route_config_name = 2 [(validate.rules).string.min_bytes = 1]; + // The name of the route configuration. This name will be passed to the RDS + // API. This allows an Envoy configuration with multiple HTTP listeners (and + // associated HTTP connection manager filters) to use different route + // configurations. + string route_config_name = 2 [(validate.rules).string.min_bytes = 1]; } // This message is used to work around the limitations with 'oneof' and repeated fields. diff --git a/include/envoy/config/config_provider.h b/include/envoy/config/config_provider.h index 0b500b9cf9180..441d2711fdafd 100644 --- a/include/envoy/config/config_provider.h +++ b/include/envoy/config/config_provider.h @@ -1,6 +1,7 @@ #pragma once #include + #include #include "envoy/common/time.h" diff --git a/source/common/http/conn_manager_impl.cc b/source/common/http/conn_manager_impl.cc index 5b73766289f5a..e9f40b4351043 100644 --- a/source/common/http/conn_manager_impl.cc +++ b/source/common/http/conn_manager_impl.cc @@ -1124,23 +1124,23 @@ void ConnectionManagerImpl::ActiveStream::sendLocalReply( if (!state_.created_filter_chain_) { createFilterChain(); } - Utility::sendLocalReply( - is_grpc_request, - [this, modify_headers](HeaderMapPtr&& headers, bool end_stream) -> void { - if (modify_headers != nullptr) { - modify_headers(*headers); - } - response_headers_ = std::move(headers); - // TODO: Start encoding from the last decoder filter that saw the - // request instead. - encodeHeaders(nullptr, *response_headers_, end_stream); - }, - [this](Buffer::Instance& data, bool end_stream) -> void { - // TODO: Start encoding from the last decoder filter that saw the - // request instead. - encodeData(nullptr, data, end_stream, FilterIterationStartState::CanStartFromCurrent); - }, - state_.destroyed_, code, body, grpc_status, is_head_request); + Utility::sendLocalReply(is_grpc_request, + [this, modify_headers](HeaderMapPtr&& headers, bool end_stream) -> void { + if (modify_headers != nullptr) { + modify_headers(*headers); + } + response_headers_ = std::move(headers); + // TODO: Start encoding from the last decoder filter that saw the + // request instead. + encodeHeaders(nullptr, *response_headers_, end_stream); + }, + [this](Buffer::Instance& data, bool end_stream) -> void { + // TODO: Start encoding from the last decoder filter that saw the + // request instead. + encodeData(nullptr, data, end_stream, + FilterIterationStartState::CanStartFromCurrent); + }, + state_.destroyed_, code, body, grpc_status, is_head_request); } void ConnectionManagerImpl::ActiveStream::encode100ContinueHeaders( diff --git a/source/common/router/scoped_config_manager.cc b/source/common/router/scoped_config_manager.cc index b29b533c105c2..ee92a58dc4778 100644 --- a/source/common/router/scoped_config_manager.cc +++ b/source/common/router/scoped_config_manager.cc @@ -1,7 +1,8 @@ +#include "common/router/scoped_config_manager.h" + #include "envoy/common/exception.h" #include "common/common/fmt.h" -#include "common/router/scoped_config_manager.h" namespace Envoy { namespace Router { From dd2261c1b7981cb3b9652e9521a0a1d62ef1bce6 Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Fri, 12 Apr 2019 22:10:41 -0400 Subject: [PATCH 21/37] Comments. Signed-off-by: Andres Guedez --- api/envoy/admin/v2alpha/config_dump.proto | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/envoy/admin/v2alpha/config_dump.proto b/api/envoy/admin/v2alpha/config_dump.proto index 6b7f09deed5cb..ba5b36df36c7c 100644 --- a/api/envoy/admin/v2alpha/config_dump.proto +++ b/api/envoy/admin/v2alpha/config_dump.proto @@ -186,7 +186,7 @@ message RoutesConfigDump { // dynamically obtained scopes via the SRDS API. message ScopedRoutesConfigDump { message InlineScopedRouteConfigs { - // TODO(AndresGuedez) + // The name assigned to the scoped route configurations. string name = 1; // The scoped route configurations. @@ -197,6 +197,7 @@ message ScopedRoutesConfigDump { } message DynamicScopedRouteConfigs { + // The name assigned to the scoped route configurations. string name = 1; // This is the per-resource version information. This version is currently taken from the From 090bbd89bb61c82860fdb90f3381396956289801 Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Tue, 16 Apr 2019 16:56:20 -0400 Subject: [PATCH 22/37] Cleanup and minor refactor of ConfigProvider framework. - Refactor the framework to create a cleaner, more cohesive set of abstractions based on the ConfigProvider::ApiType of the DS API. - Rename base classes for consistency and clarity. Signed-off-by: Andres Guedez --- include/envoy/config/config_provider.h | 15 +- source/common/config/config_provider_impl.cc | 61 ++-- source/common/config/config_provider_impl.h | 317 +++++++++++------- source/common/router/scoped_rds.cc | 24 +- source/common/router/scoped_rds.h | 42 +-- .../config/config_provider_impl_test.cc | 32 +- 6 files changed, 272 insertions(+), 219 deletions(-) diff --git a/include/envoy/config/config_provider.h b/include/envoy/config/config_provider.h index 441d2711fdafd..0b22f74618dec 100644 --- a/include/envoy/config/config_provider.h +++ b/include/envoy/config/config_provider.h @@ -1,11 +1,10 @@ #pragma once -#include - #include #include "envoy/common/time.h" +#include "common/common/assert.h" #include "common/protobuf/protobuf.h" #include "absl/types/optional.h" @@ -140,26 +139,20 @@ class ConfigProvider { * @return Protobuf::Message* the config proto corresponding to the Config instantiated by the * provider. */ - virtual const Protobuf::Message* getConfigProto() const { - assert("not implemented" && false); - return nullptr; - } + virtual const Protobuf::Message* getConfigProto() const { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } /** * Returns the config protos associated with the provider. * @return const ConfigProtoVector the config protos corresponding to the Config instantiated by * the provider. */ - virtual const ConfigProtoVector getConfigProtos() const { - assert("not implemented" && false); - return {}; - } + virtual const ConfigProtoVector getConfigProtos() const { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } /** * Returns the config version associated with the provider. * @return std::string the config version. */ - virtual std::string getConfigVersion() const PURE; + virtual std::string getConfigVersion() const { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } /** * Returns the config implementation associated with the provider. diff --git a/source/common/config/config_provider_impl.cc b/source/common/config/config_provider_impl.cc index 3e290dde48e30..7d8f01719b07a 100644 --- a/source/common/config/config_provider_impl.cc +++ b/source/common/config/config_provider_impl.cc @@ -3,7 +3,7 @@ namespace Envoy { namespace Config { -ImmutableConfigProviderImplBase::ImmutableConfigProviderImplBase( +ImmutableConfigProviderBase::ImmutableConfigProviderBase( Server::Configuration::FactoryContext& factory_context, ConfigProviderManagerImplBase& config_provider_manager, ConfigProviderInstanceType instance_type, ApiType api_type) @@ -13,18 +13,35 @@ ImmutableConfigProviderImplBase::ImmutableConfigProviderImplBase( config_provider_manager_.bindImmutableConfigProvider(this); } -ImmutableConfigProviderImplBase::~ImmutableConfigProviderImplBase() { +ImmutableConfigProviderBase::~ImmutableConfigProviderBase() { config_provider_manager_.unbindImmutableConfigProvider(this); } -ConfigSubscriptionInstanceBase::~ConfigSubscriptionInstanceBase() { +ConfigSubscriptionCommonBase::~ConfigSubscriptionCommonBase() { init_target_.ready(); config_provider_manager_.unbindSubscription(manager_identifier_); } -bool ConfigSubscriptionInstanceBase::checkAndApplyConfig(const Protobuf::Message& config_proto, - const std::string& config_name, - const std::string& version_info) { +void ConfigSubscriptionCommonBase::bindConfigProvider(MutableConfigProviderCommonBase* provider) { + // All config providers bound to a ConfigSubscriptionCommonBase must be of the same concrete + // type; this is assumed by ConfigSubscriptionInstance::checkAndApplyConfigUpdate() and is + // verified by the assertion below. NOTE: an inlined statement ASSERT() triggers a potentially + // evaluated expression warning from clang due to `typeid(**mutable_config_providers_.begin())`. + // To avoid this, we use a lambda to separate the first mutable provider dereference from the + // typeid() statement. + ASSERT([&]() { + if (!mutable_config_providers_.empty()) { + const auto& first_provider = **mutable_config_providers_.begin(); + return typeid(*provider) == typeid(first_provider); + } + return true; + }()); + mutable_config_providers_.insert(provider); +} + +bool ConfigSubscriptionInstance::checkAndApplyConfigUpdate(const Protobuf::Message& config_proto, + const std::string& config_name, + const std::string& version_info) { const uint64_t new_hash = MessageUtil::hash(config_proto); if (config_info_) { ASSERT(config_info_.value().last_config_hash_.has_value()); @@ -44,41 +61,27 @@ bool ConfigSubscriptionInstanceBase::checkAndApplyConfig(const Protobuf::Message // bindConfigProvider()). // This makes it safe to call any of the provider's onConfigProtoUpdate() to get a new config // impl, which can then be passed to all providers. + auto* typed_provider = static_cast(provider); if (new_config == nullptr) { - if ((new_config = provider->onConfigProtoUpdate(config_proto)) == nullptr) { + if ((new_config = typed_provider->onConfigProtoUpdate(config_proto)) == nullptr) { return false; } } - provider->onConfigUpdate(new_config); + typed_provider->onConfigUpdate(new_config); } return true; } -void ConfigSubscriptionInstanceBase::propagateDeltaConfigUpdate( +void DeltaConfigSubscriptionInstance::applyConfigUpdate( std::function updateFn) { for (auto* provider : mutable_config_providers_) { - ConfigProvider::ConfigConstSharedPtr config = provider->getConfig(); - provider->onDeltaConfigUpdate([config, updateFn]() { updateFn(config); }); + auto* typed_provider = static_cast(provider); + ConfigProvider::ConfigConstSharedPtr config = typed_provider->getConfig(); + typed_provider->onConfigUpdate([config, updateFn]() { updateFn(config); }); } } -void ConfigSubscriptionInstanceBase::bindConfigProvider(MutableConfigProviderImplBase* provider) { - // All config providers bound to a ConfigSubscriptionInstanceBase must be of the same concrete - // type; this is assumed by checkAndApplyConfig() and is verified by the assertion below. - // NOTE: an inlined statement ASSERT() triggers a potentially evaluated expression warning from - // clang due to `typeid(**mutable_config_providers_.begin())`. To avoid this, we use a lambda to - // separate the first mutable provider dereference from the typeid() statement. - ASSERT([&]() { - if (!mutable_config_providers_.empty()) { - const auto& first_provider = **mutable_config_providers_.begin(); - return typeid(*provider) == typeid(first_provider); - } - return true; - }()); - mutable_config_providers_.insert(provider); -} - ConfigProviderManagerImplBase::ConfigProviderManagerImplBase(Server::Admin& admin, const std::string& config_name) { config_tracker_entry_ = @@ -100,7 +103,7 @@ ConfigProviderManagerImplBase::immutableConfigProviders(ConfigProviderInstanceTy } void ConfigProviderManagerImplBase::bindImmutableConfigProvider( - ImmutableConfigProviderImplBase* provider) { + ImmutableConfigProviderBase* provider) { ASSERT(provider->instanceType() == ConfigProviderInstanceType::Static || provider->instanceType() == ConfigProviderInstanceType::Inline); ConfigProviderMap::iterator it; @@ -115,7 +118,7 @@ void ConfigProviderManagerImplBase::bindImmutableConfigProvider( } void ConfigProviderManagerImplBase::unbindImmutableConfigProvider( - ImmutableConfigProviderImplBase* provider) { + ImmutableConfigProviderBase* provider) { ASSERT(provider->instanceType() == ConfigProviderInstanceType::Static || provider->instanceType() == ConfigProviderInstanceType::Inline); auto it = immutable_config_providers_map_.find(provider->instanceType()); diff --git a/source/common/config/config_provider_impl.h b/source/common/config/config_provider_impl.h index c830c10648c07..ea494d31e9b88 100644 --- a/source/common/config/config_provider_impl.h +++ b/source/common/config/config_provider_impl.h @@ -19,14 +19,15 @@ namespace Envoy { namespace Config { -// This file provides a set of base classes, (ImmutableConfigProviderImplBase, -// MutableConfigProviderImplBase, ConfigProviderManagerImplBase, ConfigSubscriptionInstanceBase), -// conforming to the ConfigProvider/ConfigProviderManager interfaces, which in tandem provide a -// framework for implementing statically defined (i.e., immutable) and dynamic (mutable via -// subscriptions) configuration for Envoy. +// This file provides a set of base classes, (ImmutableConfigProviderBase, +// MutableConfigProviderCommonBase, MutableConfigProviderBase, DeltaMutableConfigProviderBase, +// ConfigProviderManagerImplBase, ConfigSubscriptionCommonBase, ConfigSubscriptionInstance, +// DeltaConfigSubscriptionInstance), conforming to the ConfigProvider/ConfigProviderManager +// interfaces, which in tandem provide a framework for implementing statically defined (i.e., +// immutable) and dynamic (mutable via subscriptions) configuration for Envoy. // // The mutability property applies to the ConfigProvider itself and _not_ the underlying config -// proto, which is always immutable. MutableConfigProviderImplBase objects receive config proto +// proto, which is always immutable. MutableConfigProviderCommonBase objects receive config proto // updates via xDS subscriptions, resulting in new ConfigProvider::Config objects being instantiated // with the corresponding change in behavior corresponding to updated config. ConfigProvider::Config // objects must be latched/associated with the appropriate objects in the connection and request @@ -49,27 +50,36 @@ namespace Config { // 1) Create a class derived from ConfigProviderManagerImplBase and implement the required // interface. // When implementing createXdsConfigProvider(), it is expected that getSubscription() will -// be called to fetch either an existing ConfigSubscriptionInstanceBase if the config source -// configuration matches, or a newly instantiated subscription otherwise. +// be called to fetch either an existing ConfigSubscriptionCommonBase if the config +// source configuration matches, or a newly instantiated subscription otherwise. // // For immutable providers: -// 1) Create a class derived from ImmutableConfigProviderImplBase and implement the required +// 1) Create a class derived from ImmutableConfigProviderBase and implement the required // interface. // // For mutable (xDS) providers: -// 1) Create a class derived from MutableConfigProviderImplBase and implement the required -// interface. -// 2) Create a class derived from ConfigSubscriptionInstanceBase; this is the entity -// responsible for owning and managing the Envoy::Config::Subscription that provides -// the underlying config subscription. +// 1) According to the API type, create a class derived from MutableConfigProviderBase or +// DeltaMutableConfigProviderBase and implement the required interface. +// 2) According to the API type, create a class derived from ConfigSubscriptionInstance or +// DeltaConfigSubscriptionInstance; this is the entity responsible for owning and managing the +// Envoy::Config::Subscription that provides the underlying config subscription. +// a) For a ConfigProvider::ApiType::Full subscription instance (i.e., a +// ConfigSubscriptionInstance child): // - When subscription callbacks (onConfigUpdate, onConfigUpdateFailed) are issued by the -// underlying subscription, the corresponding ConfigSubscriptionInstanceBase functions must be -// called as well. -// - On a successful config update, checkAndApplyConfig() should be called to instantiate the -// new config implementation and propagate it to the shared config providers and all -// worker threads. -// - On a successful return from checkAndApplyConfig(), the config proto must be latched into -// this class and returned via the getConfigProto() override. +// underlying subscription, the corresponding ConfigSubscriptionInstance functions +// must be called as well. +// - On a successful config update, checkAndApplyConfigUpdate() should be called to instantiate +// the new config implementation and propagate it to the shared config providers and all worker +// threads. +// - On a successful return from checkAndApplyConfigUpdate(), the config proto must be latched +// into this class and returned via the getConfigProto() override. +// b) For a ConfigProvider::ApiType::Delta subscription instance (i.e., a +// DeltaConfigSubscriptionInstance child): +// - When subscription callbacks (onConfigUpdate, onConfigUpdateFailed) are issued by the +// underlying subscription, the corresponding ConfigSubscriptionInstance functions must be called +// as well. +// - On a successful config update, applyConfigUpdate() should be called to propagate the config +// updates to all bound config providers and worker threads. class ConfigProviderManagerImplBase; @@ -90,14 +100,14 @@ enum class ConfigProviderInstanceType { * ConfigProvider implementation for immutable configuration. * * TODO(AndresGuedez): support sharing of config protos and config impls, as is - * done with the MutableConfigProviderImplBase. + * done with the MutableConfigProviderCommonBase. * * This class can not be instantiated directly; instead, it provides the foundation for * immutable config provider implementations which derive from it. */ -class ImmutableConfigProviderImplBase : public ConfigProvider { +class ImmutableConfigProviderBase : public ConfigProvider { public: - ~ImmutableConfigProviderImplBase() override; + ~ImmutableConfigProviderBase() override; // Envoy::Config::ConfigProvider SystemTime lastUpdated() const override { return last_updated_; } @@ -106,9 +116,9 @@ class ImmutableConfigProviderImplBase : public ConfigProvider { ConfigProviderInstanceType instanceType() const { return instance_type_; } protected: - ImmutableConfigProviderImplBase(Server::Configuration::FactoryContext& factory_context, - ConfigProviderManagerImplBase& config_provider_manager, - ConfigProviderInstanceType instance_type, ApiType api_type); + ImmutableConfigProviderBase(Server::Configuration::FactoryContext& factory_context, + ConfigProviderManagerImplBase& config_provider_manager, + ConfigProviderInstanceType instance_type, ApiType api_type); private: SystemTime last_updated_; @@ -117,13 +127,13 @@ class ImmutableConfigProviderImplBase : public ConfigProvider { ApiType api_type_; }; -class MutableConfigProviderImplBase; +class MutableConfigProviderCommonBase; /** - * Provides generic functionality required by all xDS ConfigProvider subscriptions, including - * shared lifetime management via shared_ptr. + * Provides common DS API subscription functionality required by the ConfigProvider::ApiType + * specific base classes (see ConfigSubscriptionInstance and DeltaConfigSubscriptionInstance). * - * To do so, this class keeps track of a set of MutableConfigProviderImplBase instances associated + * To do so, this class keeps track of a set of MutableConfigProviderCommonBase instances associated * with an underlying subscription; providers are bound/unbound as needed as they are created and * destroyed. * @@ -136,14 +146,14 @@ class MutableConfigProviderImplBase; * This class can not be instantiated directly; instead, it provides the foundation for * config subscription implementations which derive from it. */ -class ConfigSubscriptionInstanceBase : protected Logger::Loggable { +class ConfigSubscriptionCommonBase : protected Logger::Loggable { public: struct LastConfigInfo { absl::optional last_config_hash_; std::string last_config_version_; }; - virtual ~ConfigSubscriptionInstanceBase(); + virtual ~ConfigSubscriptionCommonBase(); /** * Starts the subscription corresponding to a config source. @@ -174,33 +184,22 @@ class ConfigSubscriptionInstanceBase : protected Logger::Loggable updateFn); + const std::string name_; + std::unordered_set mutable_config_providers_; + absl::optional config_info_; private: - void bindConfigProvider(MutableConfigProviderImplBase* provider); + void bindConfigProvider(MutableConfigProviderCommonBase* provider); - void unbindConfigProvider(MutableConfigProviderImplBase* provider) { + void unbindConfigProvider(MutableConfigProviderCommonBase* provider) { mutable_config_providers_.erase(provider); } - const std::string name_; Init::TargetImpl init_target_; - std::unordered_set mutable_config_providers_; const uint64_t manager_identifier_; ConfigProviderManagerImplBase& config_provider_manager_; TimeSource& time_source_; SystemTime last_updated_; - absl::optional config_info_; - // ConfigSubscriptionInstanceBase, MutableConfigProviderImplBase and ConfigProviderManagerImplBase - // are tightly coupled with the current shared ownership model; use friend classes to explicitly - // denote the binding between them. + // ConfigSubscriptionCommonBase, MutableConfigProviderCommonBase and + // ConfigProviderManagerImplBase are tightly coupled with the current shared ownership model; use + // friend classes to explicitly denote the binding between them. // // TODO(AndresGuedez): Investigate whether a shared ownership model avoiding the s and // instead centralizing lifetime management in the ConfigProviderManagerImplBase with explicit // reference counting would be more maintainable. - friend class MutableConfigProviderImplBase; + friend class MutableConfigProviderCommonBase; + friend class MutableConfigProviderBase; + friend class DeltaMutableConfigProviderBase; friend class ConfigProviderManagerImplBase; }; -using ConfigSubscriptionInstanceBaseSharedPtr = std::shared_ptr; +using ConfigSubscriptionCommonBaseSharedPtr = std::shared_ptr; /** - * Provides generic functionality required by all dynamic config providers, including distribution - * of config updates to all workers. + * Provides common subscription functionality required by ConfigProvider::ApiType::Full DS APIs. + */ +class ConfigSubscriptionInstance : public ConfigSubscriptionCommonBase { +protected: + ConfigSubscriptionInstance(const std::string& name, const uint64_t manager_identifier, + ConfigProviderManagerImplBase& config_provider_manager, + TimeSource& time_source, const SystemTime& last_updated, + const LocalInfo::LocalInfo& local_info) + : ConfigSubscriptionCommonBase(name, manager_identifier, config_provider_manager, time_source, + last_updated, local_info) {} + + ~ConfigSubscriptionInstance() override = default; + + /** + * Determines whether a configuration proto is a new update, and if so, propagates it to all + * config providers associated with this subscription. + * @param config_proto supplies the newly received config proto. + * @param config_name supplies the name associated with the config. + * @param version_info supplies the version associated with the config. + * @return bool false when the config proto has no delta from the previous config, true otherwise. + */ + bool checkAndApplyConfigUpdate(const Protobuf::Message& config_proto, + const std::string& config_name, const std::string& version_info); +}; + +/** + * Provides common subscription functionality required by ConfigProvider::ApiType::Delta DS APIs. + */ +class DeltaConfigSubscriptionInstance : public ConfigSubscriptionCommonBase { +protected: + DeltaConfigSubscriptionInstance(const std::string& name, const uint64_t manager_identifier, + ConfigProviderManagerImplBase& config_provider_manager, + TimeSource& time_source, const SystemTime& last_updated, + const LocalInfo::LocalInfo& local_info) + : ConfigSubscriptionCommonBase(name, manager_identifier, config_provider_manager, time_source, + last_updated, local_info) {} + + ~DeltaConfigSubscriptionInstance() override = default; + + /** + * Propagates a config update to all config providers and worker threads associated with the + * subscription. + * + * @param updateFn the callback to run on each provider and worker thread. + */ + void applyConfigUpdate(std::function updateFn); +}; + +/** + * Provides generic functionality required by the ConfigProvider::ApiType specific dynamic config + * providers (see MutableConfigProviderBase and DeltaMutableConfigProviderBase). * * This class can not be instantiated directly; instead, it provides the foundation for * dynamic config provider implementations which derive from it. */ -class MutableConfigProviderImplBase : public ConfigProvider { +class MutableConfigProviderCommonBase : public ConfigProvider { public: - ~MutableConfigProviderImplBase() override { subscription_->unbindConfigProvider(this); } + ~MutableConfigProviderCommonBase() override { subscription_->unbindConfigProvider(this); } // Envoy::Config::ConfigProvider SystemTime lastUpdated() const override { return subscription_->lastUpdated(); } + ApiType apiType() const override { return api_type_; } + +protected: + MutableConfigProviderCommonBase(ConfigSubscriptionCommonBaseSharedPtr&& subscription, + Server::Configuration::FactoryContext& factory_context, + ApiType api_type) + : tls_(factory_context.threadLocal().allocateSlot()), subscription_(subscription), + api_type_(api_type) {} + + ThreadLocal::SlotPtr tls_; + ConfigSubscriptionCommonBaseSharedPtr subscription_; + +private: + ApiType api_type_; +}; + +/** + * Provides common mutable (dynamic) config provider functionality required by + * ConfigProvider::ApiType::Full DS APIs. + */ +class MutableConfigProviderBase : public MutableConfigProviderCommonBase { +public: + // Envoy::Config::ConfigProvider + // NOTE: This is being promoted to public for internal uses to avoid an unnecessary dynamic_cast + // in the public API (ConfigProvider::config()). ConfigConstSharedPtr getConfig() const override { return tls_->getTyped().config_; } - ApiType apiType() const override { return api_type_; } /** * Called when a new config proto is received via an xDS subscription. @@ -273,7 +342,6 @@ class MutableConfigProviderImplBase : public ConfigProvider { * Note that this function is called _once_ across all shared config providers per xDS * subscription config update. * @param config_proto supplies the configuration proto. - * @param version_info supplies the version associated with the config. * @return ConfigConstSharedPtr the ConfigProvider::Config to share with other providers. */ virtual ConfigConstSharedPtr onConfigProtoUpdate(const Protobuf::Message& config_proto) PURE; @@ -283,23 +351,18 @@ class MutableConfigProviderImplBase : public ConfigProvider { * @param initial_config supplies an initial Envoy::Config::ConfigProvider::Config associated with * the underlying subscription. */ - virtual void initialize(const ConfigConstSharedPtr& initial_config) { + void initialize(const ConfigConstSharedPtr& initial_config) { subscription_->bindConfigProvider(this); tls_->set([initial_config](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { return std::make_shared(initial_config); }); } - virtual void initialize(ThreadLocal::Slot::InitializeCb initializeCb) { - subscription_->bindConfigProvider(this); - tls_->set(initializeCb); - } - /** * Propagates a newly instantiated Envoy::Config::ConfigProvider::Config to all workers. * @param config supplies the newly instantiated config. */ - virtual void onConfigUpdate(const ConfigConstSharedPtr& config) { + void onConfigUpdate(const ConfigConstSharedPtr& config) { if (getConfig() == config) { return; } @@ -307,23 +370,13 @@ class MutableConfigProviderImplBase : public ConfigProvider { [this, config]() -> void { tls_->getTyped().config_ = config; }); } - /** - * Propagates a delta config update to all workers. - * @param updateCb the callback to run on each worker. - */ - virtual void onDeltaConfigUpdate(Envoy::Event::PostCb updateCb) { - tls_->runOnAllThreads(updateCb); - } - protected: - MutableConfigProviderImplBase(ConfigSubscriptionInstanceBaseSharedPtr&& subscription, - Server::Configuration::FactoryContext& factory_context, - ApiType api_type) - : subscription_(subscription), tls_(factory_context.threadLocal().allocateSlot()), - api_type_(api_type) {} + MutableConfigProviderBase(ConfigSubscriptionCommonBaseSharedPtr&& subscription, + Server::Configuration::FactoryContext& factory_context, + ApiType api_type) + : MutableConfigProviderCommonBase(std::move(subscription), factory_context, api_type) {} - const ConfigSubscriptionInstanceBaseSharedPtr& subscription() const { return subscription_; } - ThreadLocal::Slot* tls() const { return tls_.get(); } + ~MutableConfigProviderBase() override = default; private: struct ThreadLocalConfig : public ThreadLocal::ThreadLocalObject { @@ -332,10 +385,42 @@ class MutableConfigProviderImplBase : public ConfigProvider { ConfigProvider::ConfigConstSharedPtr config_; }; +}; - ConfigSubscriptionInstanceBaseSharedPtr subscription_; - ThreadLocal::SlotPtr tls_; - ApiType api_type_; +/** + * Provides common mutable (dynamic) config provider functionality required by + * ConfigProvider::ApiType::Delta DS APIs. + */ +class DeltaMutableConfigProviderBase : public MutableConfigProviderCommonBase { +public: + // Envoy::Config::ConfigProvider + // This promotes getConfig() to public so that internal uses can avoid an unnecessary dynamic_cast + // in the public API (ConfigProvider::config()). + ConfigConstSharedPtr getConfig() const override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } + + /** + * Propagates a delta config update to all workers. + * @param updateCb the callback to run on each worker. + */ + void onConfigUpdate(Envoy::Event::PostCb updateCb) { tls_->runOnAllThreads(updateCb); } + +protected: + DeltaMutableConfigProviderBase(ConfigSubscriptionCommonBaseSharedPtr&& subscription, + Server::Configuration::FactoryContext& factory_context, + ApiType api_type) + : MutableConfigProviderCommonBase(std::move(subscription), factory_context, api_type) {} + + ~DeltaMutableConfigProviderBase() override = default; + + /** + * Must be called by the derived class' constructor. + * @param initializeCb supplies the initialization callback to be issued for each worker + * thread. + */ + void initialize(ThreadLocal::Slot::InitializeCb initializeCb) { + subscription_->bindConfigProvider(this); + tls_->set(initializeCb); + } }; /** @@ -343,9 +428,9 @@ class MutableConfigProviderImplBase : public ConfigProvider { * lifetime of subscriptions and dynamic config providers, along with determining which * subscriptions should be associated with newly instantiated providers. * - * The implementation of this class is not thread safe. Note that ImmutableConfigProviderImplBase - * and ConfigSubscriptionInstanceBase call the corresponding {bind,unbind}* functions exposed by - * this class. + * The implementation of this class is not thread safe. Note that ImmutableConfigProviderBase + * and ConfigSubscriptionCommonBase call the corresponding {bind,unbind}* functions exposed + * by this class. * * All config processing is done on the main thread, so instantiation of *ConfigProvider* objects * via createStaticConfigProvider() and createXdsConfigProvider() is naturally thread safe. Care @@ -372,14 +457,14 @@ class ConfigProviderManagerImplBase : public ConfigProviderManager, public Singl using ConfigProviderMap = std::unordered_map, EnumClassHash>; using ConfigSubscriptionMap = - std::unordered_map>; + std::unordered_map>; ConfigProviderManagerImplBase(Server::Admin& admin, const std::string& config_name); const ConfigSubscriptionMap& configSubscriptions() const { return config_subscriptions_; } /** - * Returns the set of bound ImmutableConfigProviderImplBase-derived providers of a given type. + * Returns the set of bound ImmutableConfigProviderBase-derived providers of a given type. * @param type supplies the type of config providers to return. * @return const ConfigProviderSet* the set of config providers corresponding to the type. */ @@ -397,12 +482,12 @@ class ConfigProviderManagerImplBase : public ConfigProviderManager, public Singl template std::shared_ptr getSubscription(const Protobuf::Message& config_source_proto, Init::Manager& init_manager, - const std::function& subscription_factory_fn) { - static_assert(std::is_base_of::value, - "T must be a subclass of ConfigSubscriptionInstanceBase"); + static_assert(std::is_base_of::value, + "T must be a subclass of ConfigSubscriptionCommonBase"); - ConfigSubscriptionInstanceBaseSharedPtr subscription; + ConfigSubscriptionCommonBaseSharedPtr subscription; const uint64_t manager_identifier = MessageUtil::hash(config_source_proto); auto it = config_subscriptions_.find(manager_identifier); @@ -416,7 +501,7 @@ class ConfigProviderManagerImplBase : public ConfigProviderManager, public Singl bindSubscription(manager_identifier, subscription); } else { // Because the ConfigProviderManagerImplBase's weak_ptrs only get cleaned up - // in the ConfigSubscriptionInstanceBase destructor, and the single threaded nature + // in the ConfigSubscriptionCommonBase destructor, and the single threaded nature // of this code, locking the weak_ptr will not fail. subscription = it->second.lock(); } @@ -427,7 +512,7 @@ class ConfigProviderManagerImplBase : public ConfigProviderManager, public Singl private: void bindSubscription(const uint64_t manager_identifier, - ConfigSubscriptionInstanceBaseSharedPtr& subscription) { + ConfigSubscriptionCommonBaseSharedPtr& subscription) { config_subscriptions_.insert({manager_identifier, subscription}); } @@ -435,8 +520,8 @@ class ConfigProviderManagerImplBase : public ConfigProviderManager, public Singl config_subscriptions_.erase(manager_identifier); } - void bindImmutableConfigProvider(ImmutableConfigProviderImplBase* provider); - void unbindImmutableConfigProvider(ImmutableConfigProviderImplBase* provider); + void bindImmutableConfigProvider(ImmutableConfigProviderBase* provider); + void unbindImmutableConfigProvider(ImmutableConfigProviderBase* provider); // TODO(jsedgwick) These two members are prime candidates for the owned-entry list/map // as in ConfigTracker. I.e. the ProviderImpls would have an EntryOwner for these lists @@ -446,10 +531,10 @@ class ConfigProviderManagerImplBase : public ConfigProviderManager, public Singl Server::ConfigTracker::EntryOwnerPtr config_tracker_entry_; - // See comment for friend classes in the ConfigSubscriptionInstanceBase for more details on the - // use of friends. - friend class ConfigSubscriptionInstanceBase; - friend class ImmutableConfigProviderImplBase; + // See comment for friend classes in the ConfigSubscriptionCommonBase for more details on + // the use of friends. + friend class ConfigSubscriptionCommonBase; + friend class ImmutableConfigProviderBase; }; } // namespace Config diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 1b9f2dffadea3..a229851672eca 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -73,9 +73,9 @@ InlineScopedRoutesConfigProvider::InlineScopedRoutesConfigProvider( const envoy::api::v2::core::ConfigSource& rds_config_source, const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: ScopeKeyBuilder& scope_key_builder) - : Envoy::Config::ImmutableConfigProviderImplBase(factory_context, config_provider_manager, - ConfigProviderInstanceType::Inline, - ConfigProvider::ApiType::Delta), + : Envoy::Config::ImmutableConfigProviderBase(factory_context, config_provider_manager, + ConfigProviderInstanceType::Inline, + ConfigProvider::ApiType::Delta), name_(name), config_(std::make_shared(scope_key_builder)), config_protos_(std::make_move_iterator(config_protos.begin()), std::make_move_iterator(config_protos.end())), @@ -86,7 +86,7 @@ ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( const uint64_t manager_identifier, const std::string& name, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, ScopedRoutesConfigProviderManager& config_provider_manager) - : ConfigSubscriptionInstanceBase( + : DeltaConfigSubscriptionInstance( "SRDS", manager_identifier, config_provider_manager, factory_context.timeSource(), factory_context.timeSource().systemTime(), factory_context.localInfo()), name_(name), @@ -108,7 +108,7 @@ void ScopedRdsConfigSubscription::onConfigUpdate( if (resources.empty()) { ENVOY_LOG(debug, "Empty resources in scoped RDS onConfigUpdate()"); stats_.update_empty_.inc(); - ConfigSubscriptionInstanceBase::onConfigUpdateFailed(); + ConfigSubscriptionCommonBase::onConfigUpdateFailed(); return; } std::vector scoped_routes; @@ -144,7 +144,7 @@ void ScopedRdsConfigSubscription::onConfigUpdate( fmt::format("failed to create/update global routing scope {}", scoped_route_name)); } ENVOY_LOG(debug, "srds: add/update scoped_route '{}'", scoped_route_name); - propagateDeltaConfigUpdate([scoped_route_info](ConfigProvider::ConfigConstSharedPtr config) { + applyConfigUpdate([scoped_route_info](ConfigProvider::ConfigConstSharedPtr config) { ThreadLocalScopedConfigImpl* thread_local_scoped_config = const_cast( static_cast(config.get())); @@ -158,7 +158,7 @@ void ScopedRdsConfigSubscription::onConfigUpdate( for (auto scoped_route : scoped_routes_to_remove) { const std::string scoped_route_name = scoped_route.first; ENVOY_LOG(debug, "srds: remove scoped route '{}'", scoped_route_name); - propagateDeltaConfigUpdate([scoped_route_name](ConfigProvider::ConfigConstSharedPtr config) { + applyConfigUpdate([scoped_route_name](ConfigProvider::ConfigConstSharedPtr config) { ThreadLocalScopedConfigImpl* thread_local_scoped_config = const_cast( static_cast(config.get())); @@ -166,7 +166,7 @@ void ScopedRdsConfigSubscription::onConfigUpdate( }); } - ConfigSubscriptionInstanceBase::onConfigUpdate(); + ConfigSubscriptionCommonBase::onConfigUpdate(); setLastConfigInfo(absl::optional({absl::nullopt, version_info})); if (!exception_msgs.empty()) { throw EnvoyException(fmt::format("Error adding/updating scoped route(s) {}", @@ -181,10 +181,10 @@ ScopedRdsConfigProvider::ScopedRdsConfigProvider( const envoy::api::v2::core::ConfigSource& rds_config_source, const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: ScopeKeyBuilder& scope_key_builder) - : MutableConfigProviderImplBase(std::move(subscription), factory_context, - ConfigProvider::ApiType::Delta), + : DeltaMutableConfigProviderBase(std::move(subscription), factory_context, + ConfigProvider::ApiType::Delta), subscription_(static_cast( - MutableConfigProviderImplBase::subscription().get())), + MutableConfigProviderCommonBase::subscription_.get())), rds_config_source_(rds_config_source) { initialize([scope_key_builder](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { return std::make_shared(scope_key_builder); @@ -243,7 +243,7 @@ ConfigProviderPtr ScopedRoutesConfigProviderManager::createXdsConfigProvider( [&config_source_proto, &factory_context, &stat_prefix, &optarg](const uint64_t manager_identifier, ConfigProviderManagerImplBase& config_provider_manager) - -> Envoy::Config::ConfigSubscriptionInstanceBaseSharedPtr { + -> Envoy::Config::ConfigSubscriptionCommonBaseSharedPtr { const auto& scoped_rds_config_source = dynamic_cast< const envoy::config::filter::network::http_connection_manager::v2::ScopedRds&>( config_source_proto); diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index 8257b425cf57c..2840b31b02adf 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -29,7 +29,7 @@ class ScopedRoutesConfigProviderUtil { class ScopedRoutesConfigProviderManager; // A ConfigProvider for inline scoped routing configuration. -class InlineScopedRoutesConfigProvider : public Envoy::Config::ImmutableConfigProviderImplBase { +class InlineScopedRoutesConfigProvider : public Envoy::Config::ImmutableConfigProviderBase { public: InlineScopedRoutesConfigProvider( std::vector>&& config_protos, @@ -78,7 +78,7 @@ struct ScopedRdsStats { }; // A scoped RDS subscription to be used with the dynamic scoped RDS ConfigProvider. -class ScopedRdsConfigSubscription : public Envoy::Config::ConfigSubscriptionInstanceBase, +class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptionInstance, Envoy::Config::SubscriptionCallbacks { public: using ScopedRouteConfigurationMap = @@ -94,7 +94,7 @@ class ScopedRdsConfigSubscription : public Envoy::Config::ConfigSubscriptionInst const std::string& name() const { return name_; } - // Envoy::Config::ConfigSubscriptionInstanceBase + // Envoy::Config::ConfigSubscriptionCommonBase void start() override { subscription_->start({}, *this); } // Envoy::Config::SubscriptionCallbacks @@ -105,7 +105,7 @@ class ScopedRdsConfigSubscription : public Envoy::Config::ConfigSubscriptionInst NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } void onConfigUpdateFailed(const EnvoyException*) override { - ConfigSubscriptionInstanceBase::onConfigUpdateFailed(); + ConfigSubscriptionCommonBase::onConfigUpdateFailed(); } std::string resourceName(const ProtobufWkt::Any& resource) override { return MessageUtil::anyConvert(resource).name(); @@ -126,7 +126,7 @@ using ScopedRdsConfigSubscriptionSharedPtr = std::shared_ptrscopedRouteMap(); - if (scoped_route_map.empty()) { - return {}; - } - - Envoy::Config::ConfigProvider::ConfigProtoVector config_protos(scoped_route_map.size()); - for (ScopedConfigManager::ScopedRouteMap::const_iterator it = scoped_route_map.begin(); - it != scoped_route_map.end(); ++it) { - config_protos.push_back(&it->second->config_proto_); - } - return config_protos; - } - std::string getConfigVersion() const override { - if (subscription_->configInfo().has_value()) { - return subscription_->configInfo().value().last_config_version_; - } - - return ""; - } ConfigConstSharedPtr getConfig() const override { - return std::dynamic_pointer_cast(tls()->get()); + return std::dynamic_pointer_cast(tls_->get()); } private: @@ -193,10 +167,10 @@ class ScopedRoutesConfigProviderManager : public Envoy::Config::ConfigProviderMa Envoy::Config::ConfigProviderPtr createStaticConfigProvider(const Protobuf::Message&, Server::Configuration::FactoryContext&, const Envoy::Config::ConfigProviderManager::OptionalArg&) override { - ASSERT(false || + ASSERT(false, "SRDS supports delta updates and requires the use of the createStaticConfigProvider() " "overload that accepts a config proto set as an argument."); - return nullptr; + NOT_REACHED_GCOVR_EXCL_LINE; } Envoy::Config::ConfigProviderPtr createStaticConfigProvider( std::vector>&& config_protos, diff --git a/test/common/config/config_provider_impl_test.cc b/test/common/config/config_provider_impl_test.cc index 2e163ad2c534c..4bb85bd5a8afb 100644 --- a/test/common/config/config_provider_impl_test.cc +++ b/test/common/config/config_provider_impl_test.cc @@ -16,7 +16,7 @@ namespace { class DummyConfigProviderManager; -class StaticDummyConfigProvider : public ImmutableConfigProviderImplBase { +class StaticDummyConfigProvider : public ImmutableConfigProviderBase { public: StaticDummyConfigProvider(const test::common::config::DummyConfig& config_proto, Server::Configuration::FactoryContext& factory_context, @@ -38,7 +38,7 @@ class StaticDummyConfigProvider : public ImmutableConfigProviderImplBase { test::common::config::DummyConfig config_proto_; }; -class DummyConfigSubscription : public ConfigSubscriptionInstanceBase, +class DummyConfigSubscription : public ConfigSubscriptionInstance, Envoy::Config::SubscriptionCallbacks { public: DummyConfigSubscription(const uint64_t manager_identifier, @@ -47,7 +47,7 @@ class DummyConfigSubscription : public ConfigSubscriptionInstanceBase, ~DummyConfigSubscription() override = default; - // Envoy::Config::ConfigSubscriptionInstanceBase + // Envoy::Config::ConfigSubscriptionCommonBase void start() override {} // Envoy::Config::SubscriptionCallbacks @@ -55,11 +55,11 @@ class DummyConfigSubscription : public ConfigSubscriptionInstanceBase, void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, const std::string& version_info) override { auto config = MessageUtil::anyConvert(resources[0]); - if (checkAndApplyConfig(config, "dummy_config", version_info)) { + if (checkAndApplyConfigUpdate(config, "dummy_config", version_info)) { config_proto_ = config; } - ConfigSubscriptionInstanceBase::onConfigUpdate(); + ConfigSubscriptionCommonBase::onConfigUpdate(); } void onConfigUpdate(const Protobuf::RepeatedPtrField&, const Protobuf::RepeatedPtrField&, const std::string&) override { @@ -87,14 +87,14 @@ class DummyConfig : public ConfigProvider::Config { DummyConfig(const test::common::config::DummyConfig&) {} }; -class DummyDynamicConfigProvider : public MutableConfigProviderImplBase { +class DummyDynamicConfigProvider : public MutableConfigProviderBase { public: DummyDynamicConfigProvider(DummyConfigSubscriptionSharedPtr&& subscription, ConfigConstSharedPtr initial_config, Server::Configuration::FactoryContext& factory_context) - : MutableConfigProviderImplBase(std::move(subscription), factory_context, ApiType::Full), + : MutableConfigProviderBase(std::move(subscription), factory_context, ApiType::Full), subscription_(static_cast( - MutableConfigProviderImplBase::subscription().get())) { + MutableConfigProviderCommonBase::subscription_.get())) { initialize(initial_config); } @@ -102,7 +102,7 @@ class DummyDynamicConfigProvider : public MutableConfigProviderImplBase { DummyConfigSubscription& subscription() { return *subscription_; } - // Envoy::Config::MutableConfigProviderImplBase + // Envoy::Config::MutableConfigProviderBase ConfigProvider::ConfigConstSharedPtr onConfigProtoUpdate(const Protobuf::Message& config) override { return std::make_shared( @@ -116,8 +116,6 @@ class DummyDynamicConfigProvider : public MutableConfigProviderImplBase { } return &subscription_->config_proto().value(); } - - // Envoy::Config::ConfigProvider std::string getConfigVersion() const override { return ""; } private: @@ -171,15 +169,15 @@ class DummyConfigProviderManager : public ConfigProviderManagerImplBase { config_source_proto, factory_context.initManager(), [&factory_context](const uint64_t manager_identifier, ConfigProviderManagerImplBase& config_provider_manager) - -> ConfigSubscriptionInstanceBaseSharedPtr { + -> ConfigSubscriptionCommonBaseSharedPtr { return std::make_shared( manager_identifier, factory_context, static_cast(config_provider_manager)); }); ConfigProvider::ConfigConstSharedPtr initial_config; - const MutableConfigProviderImplBase* provider = - subscription->getAnyBoundMutableConfigProvider(); + const auto* provider = static_cast( + subscription->getAnyBoundMutableConfigProvider()); if (provider) { initial_config = provider->getConfig(); } @@ -208,14 +206,14 @@ StaticDummyConfigProvider::StaticDummyConfigProvider( const test::common::config::DummyConfig& config_proto, Server::Configuration::FactoryContext& factory_context, DummyConfigProviderManager& config_provider_manager) - : ImmutableConfigProviderImplBase(factory_context, config_provider_manager, - ConfigProviderInstanceType::Static, ApiType::Full), + : ImmutableConfigProviderBase(factory_context, config_provider_manager, + ConfigProviderInstanceType::Static, ApiType::Full), config_(std::make_shared(config_proto)), config_proto_(config_proto) {} DummyConfigSubscription::DummyConfigSubscription( const uint64_t manager_identifier, Server::Configuration::FactoryContext& factory_context, DummyConfigProviderManager& config_provider_manager) - : ConfigSubscriptionInstanceBase( + : ConfigSubscriptionInstance( "DummyDS", manager_identifier, config_provider_manager, factory_context.timeSource(), factory_context.timeSource().systemTime(), factory_context.localInfo()) {} From 8b8a5e7d14a830d52e52e6ab46928d3a69249049 Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Wed, 17 Apr 2019 09:02:43 -0400 Subject: [PATCH 23/37] clang-tidy cleanup. Signed-off-by: Andres Guedez --- include/envoy/router/scopes.h | 2 +- source/common/config/config_provider_impl.cc | 2 +- source/common/config/config_provider_impl.h | 5 +- source/common/router/scoped_config_impl.h | 9 ++-- source/common/router/scoped_config_manager.h | 4 +- source/common/router/scoped_rds.cc | 52 +++++++++----------- source/common/router/scoped_rds.h | 17 +++---- 7 files changed, 44 insertions(+), 47 deletions(-) diff --git a/include/envoy/router/scopes.h b/include/envoy/router/scopes.h index fcdee3c38472d..d3207036f5f1c 100644 --- a/include/envoy/router/scopes.h +++ b/include/envoy/router/scopes.h @@ -13,7 +13,7 @@ namespace Router { */ class ScopedConfig : public Envoy::Config::ConfigProvider::Config { public: - virtual ~ScopedConfig() {} + ~ScopedConfig() override = default; /** * Based on the incoming HTTP request headers, returns the configuration to use for selecting a diff --git a/source/common/config/config_provider_impl.cc b/source/common/config/config_provider_impl.cc index 7d8f01719b07a..4ee6e2289f346 100644 --- a/source/common/config/config_provider_impl.cc +++ b/source/common/config/config_provider_impl.cc @@ -74,7 +74,7 @@ bool ConfigSubscriptionInstance::checkAndApplyConfigUpdate(const Protobuf::Messa } void DeltaConfigSubscriptionInstance::applyConfigUpdate( - std::function updateFn) { + const std::function& updateFn) { for (auto* provider : mutable_config_providers_) { auto* typed_provider = static_cast(provider); ConfigProvider::ConfigConstSharedPtr config = typed_provider->getConfig(); diff --git a/source/common/config/config_provider_impl.h b/source/common/config/config_provider_impl.h index ea494d31e9b88..b3eb24fbc6bda 100644 --- a/source/common/config/config_provider_impl.h +++ b/source/common/config/config_provider_impl.h @@ -289,7 +289,8 @@ class DeltaConfigSubscriptionInstance : public ConfigSubscriptionCommonBase { * * @param updateFn the callback to run on each provider and worker thread. */ - void applyConfigUpdate(std::function updateFn); + void applyConfigUpdate( + const std::function& updateFn); }; /** @@ -419,7 +420,7 @@ class DeltaMutableConfigProviderBase : public MutableConfigProviderCommonBase { */ void initialize(ThreadLocal::Slot::InitializeCb initializeCb) { subscription_->bindConfigProvider(this); - tls_->set(initializeCb); + tls_->set(std::move(initializeCb)); } }; diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h index 8623c42939aea..c8f2de6096dab 100644 --- a/source/common/router/scoped_config_impl.h +++ b/source/common/router/scoped_config_impl.h @@ -23,11 +23,12 @@ namespace Router { */ class ThreadLocalScopedConfigImpl : public ScopedConfig, public ThreadLocal::ThreadLocalObject { public: - ThreadLocalScopedConfigImpl(const envoy::config::filter::network::http_connection_manager::v2:: - ScopedRoutes::ScopeKeyBuilder& scope_key_builder) - : scope_key_builder_(scope_key_builder) {} + ThreadLocalScopedConfigImpl( + envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder + scope_key_builder) + : scope_key_builder_(std::move(scope_key_builder)) {} - virtual ~ThreadLocalScopedConfigImpl() = default; + ~ThreadLocalScopedConfigImpl() override = default; void addOrUpdateRoutingScope(ScopedRouteInfoConstSharedPtr scoped_route_info); void removeRoutingScope(const std::string& scope_name); diff --git a/source/common/router/scoped_config_manager.h b/source/common/router/scoped_config_manager.h index c13bc81ee8106..aa63ebf9f17ae 100644 --- a/source/common/router/scoped_config_manager.h +++ b/source/common/router/scoped_config_manager.h @@ -12,8 +12,8 @@ namespace Router { // proto. class ScopedRouteInfo { public: - ScopedRouteInfo(const envoy::api::v2::ScopedRouteConfiguration& config_proto) - : config_proto_(config_proto) {} + ScopedRouteInfo(envoy::api::v2::ScopedRouteConfiguration config_proto) + : config_proto_(std::move(config_proto)) {} // TODO(AndresGuedez): Add the necessary APIs required for the scoped routing logic. diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index a229851672eca..befc15b8112e7 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -36,10 +36,9 @@ ConfigProviderPtr ScopedRoutesConfigProviderUtil::maybeCreate( config.scoped_routes().scoped_route_configurations_list(); std::vector> config_protos( scoped_route_list.scoped_route_configurations().size()); - for (auto it = scoped_route_list.scoped_route_configurations().begin(); - it != scoped_route_list.scoped_route_configurations().end(); ++it) { - Protobuf::Message* clone = (*it).New(); - clone->CopyFrom(*it); + for (const auto& it : scoped_route_list.scoped_route_configurations()) { + Protobuf::Message* clone = it.New(); + clone->CopyFrom(it); config_protos.push_back(std::unique_ptr(clone)); } @@ -67,19 +66,20 @@ ConfigProviderPtr ScopedRoutesConfigProviderUtil::maybeCreate( } InlineScopedRoutesConfigProvider::InlineScopedRoutesConfigProvider( - std::vector>&& config_protos, const std::string& name, + std::vector>&& config_protos, std::string name, Server::Configuration::FactoryContext& factory_context, ScopedRoutesConfigProviderManager& config_provider_manager, - const envoy::api::v2::core::ConfigSource& rds_config_source, - const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: - ScopeKeyBuilder& scope_key_builder) + envoy::api::v2::core::ConfigSource rds_config_source, + envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder + scope_key_builder) : Envoy::Config::ImmutableConfigProviderBase(factory_context, config_provider_manager, ConfigProviderInstanceType::Inline, ConfigProvider::ApiType::Delta), - name_(name), config_(std::make_shared(scope_key_builder)), + name_(std::move(name)), + config_(std::make_shared(std::move(scope_key_builder))), config_protos_(std::make_move_iterator(config_protos.begin()), std::make_move_iterator(config_protos.end())), - rds_config_source_(rds_config_source) {} + rds_config_source_(std::move(rds_config_source)) {} ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( const envoy::config::filter::network::http_connection_manager::v2::ScopedRds& scoped_rds, @@ -134,7 +134,7 @@ void ScopedRdsConfigSubscription::onConfigUpdate( ScopedConfigManager::ScopedRouteMap scoped_routes_to_remove = scoped_config_manager_.scopedRouteMap(); for (auto& scoped_route : scoped_routes) { - const std::string scoped_route_name = scoped_route.name(); + const std::string& scoped_route_name = scoped_route.name(); try { scoped_routes_to_remove.erase(scoped_route_name); ScopedRouteInfoConstSharedPtr scoped_route_info = @@ -144,10 +144,9 @@ void ScopedRdsConfigSubscription::onConfigUpdate( fmt::format("failed to create/update global routing scope {}", scoped_route_name)); } ENVOY_LOG(debug, "srds: add/update scoped_route '{}'", scoped_route_name); - applyConfigUpdate([scoped_route_info](ConfigProvider::ConfigConstSharedPtr config) { - ThreadLocalScopedConfigImpl* thread_local_scoped_config = - const_cast( - static_cast(config.get())); + applyConfigUpdate([scoped_route_info](const ConfigProvider::ConfigConstSharedPtr& config) { + auto* thread_local_scoped_config = const_cast( + static_cast(config.get())); thread_local_scoped_config->addOrUpdateRoutingScope(scoped_route_info); }); } catch (const EnvoyException& ex) { @@ -155,13 +154,12 @@ void ScopedRdsConfigSubscription::onConfigUpdate( } } - for (auto scoped_route : scoped_routes_to_remove) { + for (const auto& scoped_route : scoped_routes_to_remove) { const std::string scoped_route_name = scoped_route.first; ENVOY_LOG(debug, "srds: remove scoped route '{}'", scoped_route_name); - applyConfigUpdate([scoped_route_name](ConfigProvider::ConfigConstSharedPtr config) { - ThreadLocalScopedConfigImpl* thread_local_scoped_config = - const_cast( - static_cast(config.get())); + applyConfigUpdate([scoped_route_name](const ConfigProvider::ConfigConstSharedPtr& config) { + auto* thread_local_scoped_config = const_cast( + static_cast(config.get())); thread_local_scoped_config->removeRoutingScope(scoped_route_name); }); } @@ -178,14 +176,14 @@ void ScopedRdsConfigSubscription::onConfigUpdate( ScopedRdsConfigProvider::ScopedRdsConfigProvider( ScopedRdsConfigSubscriptionSharedPtr&& subscription, Server::Configuration::FactoryContext& factory_context, - const envoy::api::v2::core::ConfigSource& rds_config_source, + envoy::api::v2::core::ConfigSource rds_config_source, const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: ScopeKeyBuilder& scope_key_builder) : DeltaMutableConfigProviderBase(std::move(subscription), factory_context, ConfigProvider::ApiType::Delta), subscription_(static_cast( MutableConfigProviderCommonBase::subscription_.get())), - rds_config_source_(rds_config_source) { + rds_config_source_(std::move(rds_config_source)) { initialize([scope_key_builder](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { return std::make_shared(scope_key_builder); }); @@ -205,9 +203,8 @@ ProtobufTypes::MessagePtr ScopedRoutesConfigProviderManager::dumpConfigs() const dynamic_config->set_name(typed_subscription->name()); const ScopedConfigManager::ScopedRouteMap& scoped_route_map = typed_subscription->scopedRouteMap(); - for (ScopedConfigManager::ScopedRouteMap::const_iterator it = scoped_route_map.begin(); - it != scoped_route_map.end(); ++it) { - dynamic_config->mutable_scoped_route_configs()->Add()->MergeFrom(it->second->config_proto_); + for (const auto& it : scoped_route_map) { + dynamic_config->mutable_scoped_route_configs()->Add()->MergeFrom(it.second->config_proto_); } TimestampUtil::systemClockToTimestamp(subscription->lastUpdated(), *dynamic_config->mutable_last_updated()); @@ -222,9 +219,8 @@ ProtobufTypes::MessagePtr ScopedRoutesConfigProviderManager::dumpConfigs() const provider->configProtoInfoVector(); const ConfigProvider::ConfigProtoVector& scoped_route_configurations = protos_info.value().config_protos_; - for (ConfigProvider::ConfigProtoVector::const_iterator it = scoped_route_configurations.begin(); - it != scoped_route_configurations.end(); ++it) { - inline_config->mutable_scoped_route_configs()->Add()->MergeFrom(**it); + for (const auto& it : scoped_route_configurations) { + inline_config->mutable_scoped_route_configs()->Add()->MergeFrom(*it); } TimestampUtil::systemClockToTimestamp(provider->lastUpdated(), *inline_config->mutable_last_updated()); diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index 2840b31b02adf..bbd0c6175bebf 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -32,12 +32,12 @@ class ScopedRoutesConfigProviderManager; class InlineScopedRoutesConfigProvider : public Envoy::Config::ImmutableConfigProviderBase { public: InlineScopedRoutesConfigProvider( - std::vector>&& config_protos, - const std::string& name, Server::Configuration::FactoryContext& factory_context, + std::vector>&& config_protos, std::string name, + Server::Configuration::FactoryContext& factory_context, ScopedRoutesConfigProviderManager& config_provider_manager, - const envoy::api::v2::core::ConfigSource& rds_config_source, - const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: - ScopeKeyBuilder& scope_key_builder); + envoy::api::v2::core::ConfigSource rds_config_source, + envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder + scope_key_builder); ~InlineScopedRoutesConfigProvider() override = default; @@ -130,7 +130,7 @@ class ScopedRdsConfigProvider : public Envoy::Config::DeltaMutableConfigProvider public: ScopedRdsConfigProvider(ScopedRdsConfigSubscriptionSharedPtr&& subscription, Server::Configuration::FactoryContext& factory_context, - const envoy::api::v2::core::ConfigSource& rds_config_source, + envoy::api::v2::core::ConfigSource rds_config_source, const envoy::config::filter::network::http_connection_manager::v2:: ScopedRoutes::ScopeKeyBuilder& scope_key_builder); @@ -183,11 +183,10 @@ class ScopedRoutesConfigProviderManagerOptArg : public Envoy::Config::ConfigProviderManager::OptionalArg { public: ScopedRoutesConfigProviderManagerOptArg( - const std::string& scoped_routes_name, - const envoy::api::v2::core::ConfigSource& rds_config_source, + std::string scoped_routes_name, const envoy::api::v2::core::ConfigSource& rds_config_source, const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: ScopeKeyBuilder& scope_key_builder) - : scoped_routes_name_(scoped_routes_name), rds_config_source_(rds_config_source), + : scoped_routes_name_(std::move(scoped_routes_name)), rds_config_source_(rds_config_source), scope_key_builder_(scope_key_builder) {} const std::string scoped_routes_name_; From 43c95e8bc81750fd6be48137038bb7dbb32a41b7 Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Wed, 17 Apr 2019 10:20:15 -0400 Subject: [PATCH 24/37] More clang-tidy cleanup. Signed-off-by: Andres Guedez --- include/envoy/config/BUILD | 1 + source/common/config/config_provider_impl.h | 2 +- source/common/router/scoped_config_impl.cc | 2 +- source/common/router/scoped_config_impl.h | 2 +- test/common/config/config_provider_impl_test.cc | 2 +- test/integration/scoped_rds_integration_test.cc | 2 +- 6 files changed, 6 insertions(+), 5 deletions(-) diff --git a/include/envoy/config/BUILD b/include/envoy/config/BUILD index 8c16805bf754f..777cf7a32fb25 100644 --- a/include/envoy/config/BUILD +++ b/include/envoy/config/BUILD @@ -14,6 +14,7 @@ envoy_cc_library( external_deps = ["abseil_optional"], deps = [ "//include/envoy/common:time_interface", + "//source/common/common:assert_lib", "//source/common/protobuf", ], ) diff --git a/source/common/config/config_provider_impl.h b/source/common/config/config_provider_impl.h index b3eb24fbc6bda..0ef64d0c2b8e1 100644 --- a/source/common/config/config_provider_impl.h +++ b/source/common/config/config_provider_impl.h @@ -403,7 +403,7 @@ class DeltaMutableConfigProviderBase : public MutableConfigProviderCommonBase { * Propagates a delta config update to all workers. * @param updateCb the callback to run on each worker. */ - void onConfigUpdate(Envoy::Event::PostCb updateCb) { tls_->runOnAllThreads(updateCb); } + void onConfigUpdate(Envoy::Event::PostCb updateCb) { tls_->runOnAllThreads(std::move(updateCb)); } protected: DeltaMutableConfigProviderBase(ConfigSubscriptionCommonBaseSharedPtr&& subscription, diff --git a/source/common/router/scoped_config_impl.cc b/source/common/router/scoped_config_impl.cc index 2a715811bd1b7..0c59432103ccb 100644 --- a/source/common/router/scoped_config_impl.cc +++ b/source/common/router/scoped_config_impl.cc @@ -3,7 +3,7 @@ namespace Envoy { namespace Router { -void ThreadLocalScopedConfigImpl::addOrUpdateRoutingScope(ScopedRouteInfoConstSharedPtr) {} +void ThreadLocalScopedConfigImpl::addOrUpdateRoutingScope(ScopedRouteInfoConstSharedPtr&) {} void ThreadLocalScopedConfigImpl::removeRoutingScope(const std::string&) {} diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h index c8f2de6096dab..6fb2460e1ba08 100644 --- a/source/common/router/scoped_config_impl.h +++ b/source/common/router/scoped_config_impl.h @@ -30,7 +30,7 @@ class ThreadLocalScopedConfigImpl : public ScopedConfig, public ThreadLocal::Thr ~ThreadLocalScopedConfigImpl() override = default; - void addOrUpdateRoutingScope(ScopedRouteInfoConstSharedPtr scoped_route_info); + void addOrUpdateRoutingScope(ScopedRouteInfoConstSharedPtr& scoped_route_info); void removeRoutingScope(const std::string& scope_name); // Envoy::Router::ScopedConfig diff --git a/test/common/config/config_provider_impl_test.cc b/test/common/config/config_provider_impl_test.cc index 4bb85bd5a8afb..e7d0646230a56 100644 --- a/test/common/config/config_provider_impl_test.cc +++ b/test/common/config/config_provider_impl_test.cc @@ -90,7 +90,7 @@ class DummyConfig : public ConfigProvider::Config { class DummyDynamicConfigProvider : public MutableConfigProviderBase { public: DummyDynamicConfigProvider(DummyConfigSubscriptionSharedPtr&& subscription, - ConfigConstSharedPtr initial_config, + const ConfigConstSharedPtr& initial_config, Server::Configuration::FactoryContext& factory_context) : MutableConfigProviderBase(std::move(subscription), factory_context, ApiType::Full), subscription_(static_cast( diff --git a/test/integration/scoped_rds_integration_test.cc b/test/integration/scoped_rds_integration_test.cc index 10e3494b803ee..afc0897e0b84f 100644 --- a/test/integration/scoped_rds_integration_test.cc +++ b/test/integration/scoped_rds_integration_test.cc @@ -23,7 +23,7 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, ScopedRdsIntegrationTest() : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, ipVersion(), realTime()) {} - ~ScopedRdsIntegrationTest() { + ~ScopedRdsIntegrationTest() override { resetConnections(); cleanupUpstreamAndDownstream(); } From 6adaa22c5e6470abec922ea4c9bed8d4f6f7560b Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Wed, 17 Apr 2019 13:08:10 -0400 Subject: [PATCH 25/37] Fix build break. Signed-off-by: Andres Guedez --- source/common/router/scoped_config_impl.cc | 2 +- source/common/router/scoped_config_impl.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/router/scoped_config_impl.cc b/source/common/router/scoped_config_impl.cc index 0c59432103ccb..04b153176b041 100644 --- a/source/common/router/scoped_config_impl.cc +++ b/source/common/router/scoped_config_impl.cc @@ -3,7 +3,7 @@ namespace Envoy { namespace Router { -void ThreadLocalScopedConfigImpl::addOrUpdateRoutingScope(ScopedRouteInfoConstSharedPtr&) {} +void ThreadLocalScopedConfigImpl::addOrUpdateRoutingScope(const ScopedRouteInfoConstSharedPtr&) {} void ThreadLocalScopedConfigImpl::removeRoutingScope(const std::string&) {} diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h index 6fb2460e1ba08..8a812bc71ada9 100644 --- a/source/common/router/scoped_config_impl.h +++ b/source/common/router/scoped_config_impl.h @@ -30,7 +30,7 @@ class ThreadLocalScopedConfigImpl : public ScopedConfig, public ThreadLocal::Thr ~ThreadLocalScopedConfigImpl() override = default; - void addOrUpdateRoutingScope(ScopedRouteInfoConstSharedPtr& scoped_route_info); + void addOrUpdateRoutingScope(const ScopedRouteInfoConstSharedPtr& scoped_route_info); void removeRoutingScope(const std::string& scope_name); // Envoy::Router::ScopedConfig From d401e7cc4294a694caa189c41ebb7311942aa755 Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Tue, 14 May 2019 18:02:57 -0400 Subject: [PATCH 26/37] Fix GCC compiler warning related to hidden overloads. Signed-off-by: Andres Guedez --- source/common/router/scoped_rds.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index a14c335cf8651..9c20e221ec9ea 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -136,6 +136,10 @@ class ScopedRdsConfigProvider : public Envoy::Config::DeltaMutableConfigProvider ScopedRdsConfigSubscription& subscription() { return *subscription_; } + // getConfig() is overloaded (const/non-const only). Make all base getConfig()s visible to avoid + // compiler warnings. + using DeltaMutableConfigProviderBase::getConfig; + // Envoy::Config::DeltaMutableConfigProviderBase Envoy::Config::ConfigSharedPtr getConfig() override { return std::dynamic_pointer_cast(tls_->get()); From 52d32b5420f3e29029a3e0428e2f02125a16107b Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Mon, 20 May 2019 07:02:15 -0400 Subject: [PATCH 27/37] Improve test coverage. Signed-off-by: Andres Guedez --- test/common/router/scoped_rds_test.cc | 32 +++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index e254437e8c83f..1e9a2724fea9f 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -262,6 +262,22 @@ name: foo_scoped_routes "EDS cluster"); } +// Tests a config update failure. +TEST_F(ScopedRdsTest, ConfigUpdateFailure) { + setup(); + + ScopedRdsConfigSubscription& subscription = + dynamic_cast(*provider_).subscription(); + const auto time = std::chrono::milliseconds(1234567891234); + timeSystem().setSystemTime(time); + const EnvoyException ex(fmt::format("config failure")); + // Verify the failure updates the lastUpdated() timestamp. + subscription.onConfigUpdateFailed(&ex); + EXPECT_EQ(std::chrono::time_point_cast(provider_->lastUpdated()) + .time_since_epoch(), + time); +} + class ScopedRoutesConfigProviderManagerTest : public ScopedRoutesTestBase { public: ScopedRoutesConfigProviderManagerTest() = default; @@ -406,6 +422,22 @@ route_configuration_name: dynamic-foo-route-config EXPECT_EQ(expected_config_dump.DebugString(), scoped_routes_config_dump3.DebugString()); } +using ScopedRoutesConfigProviderManagerDeathTest = ScopedRoutesConfigProviderManagerTest; + +// Tests that SRDS only allows creation of delta static config providers. +TEST_F(ScopedRoutesConfigProviderManagerDeathTest, DeltaStaticConfigProviderOnly) { + EXPECT_DEBUG_DEATH( + config_provider_manager_->createStaticConfigProvider( + parseScopedRouteConfigurationFromYaml(R"EOF( +name: dynamic-foo +route_configuration_name: static-foo-route-config +key: + fragments: { string_key: "172.30.30.10" } +)EOF"), + factory_context_, Envoy::Config::ConfigProviderManager::NullOptionalArg()), + ".*SRDS supports delta updates and requires the use of the createStaticConfigProvider().*"); +} + } // namespace } // namespace Router } // namespace Envoy From c611a46501ec85780c7f33b87c66408373ae45c2 Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Mon, 20 May 2019 15:26:08 -0400 Subject: [PATCH 28/37] Improve test coverage. Also, fixes a bug removing scoped routes. Signed-off-by: Andres Guedez --- source/common/router/scoped_config_manager.cc | 5 +- source/common/router/scoped_rds.cc | 7 +-- test/common/router/scoped_rds_test.cc | 46 +++++++++++++------ 3 files changed, 34 insertions(+), 24 deletions(-) diff --git a/source/common/router/scoped_config_manager.cc b/source/common/router/scoped_config_manager.cc index ee92a58dc4778..2a5b75f3b29ce 100644 --- a/source/common/router/scoped_config_manager.cc +++ b/source/common/router/scoped_config_manager.cc @@ -15,10 +15,7 @@ ScopedRouteInfoConstSharedPtr ScopedConfigManager::addOrUpdateRoutingScope( } bool ScopedConfigManager::removeRoutingScope(const std::string& name) { - if (scoped_route_map_.erase(name) == 0) { - throw EnvoyException(fmt::format("could not find {} in scoped route map for removal", name)); - } - return true; + return scoped_route_map_.erase(name) == 0; } } // namespace Router diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 9a46bebb829b6..6c975c574b3fa 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -105,12 +105,6 @@ ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( void ScopedRdsConfigSubscription::onConfigUpdate( const Protobuf::RepeatedPtrField& resources, const std::string& version_info) { - if (resources.empty()) { - ENVOY_LOG(debug, "Empty resources in scoped RDS onConfigUpdate()"); - stats_.update_empty_.inc(); - ConfigSubscriptionCommonBase::onConfigUpdateFailed(); - return; - } std::vector scoped_routes; for (const auto& resource_any : resources) { scoped_routes.emplace_back( @@ -158,6 +152,7 @@ void ScopedRdsConfigSubscription::onConfigUpdate( for (const auto& scoped_route : scoped_routes_to_remove) { const std::string scoped_route_name = scoped_route.first; ENVOY_LOG(debug, "srds: remove scoped route '{}'", scoped_route_name); + scoped_config_manager_.removeRoutingScope(scoped_route_name); applyDeltaConfigUpdate([scoped_route_name](const ConfigProvider::ConfigConstSharedPtr& config) { auto* thread_local_scoped_config = const_cast( static_cast(config.get())); diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index 1e9a2724fea9f..309cfa6190c5a 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -168,17 +168,6 @@ route_configuration_name: foo_routes EXPECT_THROW(subscription.onConfigUpdate(resources3, "1"), ProtoValidationException); } -// Tests that an empty config update will update the corresponding stat. -TEST_F(ScopedRdsTest, EmptyResource) { - setup(); - - Protobuf::RepeatedPtrField resources; - subscription().onConfigUpdate(resources, "1"); - EXPECT_EQ( - 1UL, - factory_context_.scope_.counter("foo.scoped_rds.foo_scoped_routes.update_empty").value()); -} - // Tests that multiple uniquely named resources are allowed in config updates. TEST_F(ScopedRdsTest, MultipleResources) { setup(); @@ -266,13 +255,11 @@ name: foo_scoped_routes TEST_F(ScopedRdsTest, ConfigUpdateFailure) { setup(); - ScopedRdsConfigSubscription& subscription = - dynamic_cast(*provider_).subscription(); const auto time = std::chrono::milliseconds(1234567891234); timeSystem().setSystemTime(time); const EnvoyException ex(fmt::format("config failure")); // Verify the failure updates the lastUpdated() timestamp. - subscription.onConfigUpdateFailed(&ex); + subscription().onConfigUpdateFailed(&ex); EXPECT_EQ(std::chrono::time_point_cast(provider_->lastUpdated()) .time_since_epoch(), time); @@ -420,6 +407,37 @@ route_configuration_name: dynamic-foo-route-config MessageUtil::downcastAndValidate( *message_ptr); EXPECT_EQ(expected_config_dump.DebugString(), scoped_routes_config_dump3.DebugString()); + + resources.Clear(); + subscription.onConfigUpdate(resources, "2"); + MessageUtil::loadFromYaml(R"EOF( +inline_scoped_route_configs: + - name: foo-scoped-routes + scoped_route_configs: + - name: foo + route_configuration_name: foo-route-config + key: + fragments: { string_key: "172.10.10.10" } + - name: foo2 + route_configuration_name: foo-route-config2 + key: + fragments: { string_key: "172.10.10.20" } + last_updated: + seconds: 1234567891 + nanos: 234000000 +dynamic_scoped_route_configs: + - name: foo-dynamic-scoped-routes + last_updated: + seconds: 1234567891 + nanos: 567000000 + version_info: "2" +)EOF", + expected_config_dump); + message_ptr = factory_context_.admin_.config_tracker_.config_tracker_callbacks_["route_scopes"](); + const auto& scoped_routes_config_dump4 = + MessageUtil::downcastAndValidate( + *message_ptr); + EXPECT_EQ(expected_config_dump.DebugString(), scoped_routes_config_dump4.DebugString()); } using ScopedRoutesConfigProviderManagerDeathTest = ScopedRoutesConfigProviderManagerTest; From e507e55fc9968e9eac90c0274b7ceac390d67bcd Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Mon, 20 May 2019 15:56:20 -0400 Subject: [PATCH 29/37] Cleanup. Signed-off-by: Andres Guedez --- source/common/router/scoped_config_manager.cc | 2 +- source/common/router/scoped_config_manager.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/router/scoped_config_manager.cc b/source/common/router/scoped_config_manager.cc index 2a5b75f3b29ce..e8cdc959792f1 100644 --- a/source/common/router/scoped_config_manager.cc +++ b/source/common/router/scoped_config_manager.cc @@ -9,7 +9,7 @@ namespace Router { ScopedRouteInfoConstSharedPtr ScopedConfigManager::addOrUpdateRoutingScope( const envoy::api::v2::ScopedRouteConfiguration& config_proto, const std::string&) { - auto scoped_route_info = std::make_shared(config_proto); + auto scoped_route_info = std::make_shared(std::move(config_proto)); scoped_route_map_[config_proto.name()] = scoped_route_info; return scoped_route_info; } diff --git a/source/common/router/scoped_config_manager.h b/source/common/router/scoped_config_manager.h index 89aad8d6710ea..c6c3b3570f582 100644 --- a/source/common/router/scoped_config_manager.h +++ b/source/common/router/scoped_config_manager.h @@ -12,7 +12,7 @@ namespace Router { // proto. class ScopedRouteInfo { public: - ScopedRouteInfo(envoy::api::v2::ScopedRouteConfiguration config_proto) + ScopedRouteInfo(const envoy::api::v2::ScopedRouteConfiguration&& config_proto) : config_proto_(std::move(config_proto)) {} // TODO(AndresGuedez): Add the necessary APIs required for the scoped routing logic. From 22645d1a4dab7608ed28dea9b434631f6ac8b41a Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Mon, 20 May 2019 16:50:52 -0400 Subject: [PATCH 30/37] Cleanup. Signed-off-by: Andres Guedez --- include/envoy/config/config_provider_manager.h | 2 +- source/common/protobuf/protobuf.h | 1 + source/common/router/scoped_rds.cc | 12 +++++------- source/common/router/scoped_rds.h | 2 +- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/include/envoy/config/config_provider_manager.h b/include/envoy/config/config_provider_manager.h index 81beb1ecb5fbb..9c9dce88d1071 100644 --- a/include/envoy/config/config_provider_manager.h +++ b/include/envoy/config/config_provider_manager.h @@ -81,7 +81,7 @@ class ConfigProviderManager { * @return ConfigProviderPtr a newly allocated static config provider. */ virtual ConfigProviderPtr - createStaticConfigProvider(std::vector>&& config_protos, + createStaticConfigProvider(ProtobufTypes::ConstMessagePtrVector&& config_protos, Server::Configuration::FactoryContext& factory_context, const OptionalArg& optarg) { UNREFERENCED_PARAMETER(config_protos); diff --git a/source/common/protobuf/protobuf.h b/source/common/protobuf/protobuf.h index fbeffd9dcb4b4..4f5e7de88c2a1 100644 --- a/source/common/protobuf/protobuf.h +++ b/source/common/protobuf/protobuf.h @@ -45,6 +45,7 @@ namespace ProtobufWkt = google::protobuf; namespace ProtobufTypes { typedef std::unique_ptr MessagePtr; +typedef std::vector> ConstMessagePtrVector; typedef int64_t Int64; diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 6c975c574b3fa..e1dbe9d3b8fae 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -23,10 +23,8 @@ ConfigProviderPtr ScopedRoutesConfigProviderUtil::maybeCreate( config, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, ConfigProviderManager& scoped_routes_config_provider_manager) { - if (config.route_specifier_case() != envoy::config::filter::network::http_connection_manager::v2:: - HttpConnectionManager::kScopedRoutes) { - return nullptr; - } + ASSERT(config.route_specifier_case() == envoy::config::filter::network::http_connection_manager:: + v2::HttpConnectionManager::kScopedRoutes); switch (config.scoped_routes().config_specifier_case()) { case envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: @@ -34,7 +32,7 @@ ConfigProviderPtr ScopedRoutesConfigProviderUtil::maybeCreate( const envoy::config::filter::network::http_connection_manager::v2:: ScopedRouteConfigurationsList& scoped_route_list = config.scoped_routes().scoped_route_configurations_list(); - std::vector> config_protos( + ProtobufTypes::ConstMessagePtrVector config_protos( scoped_route_list.scoped_route_configurations().size()); for (const auto& it : scoped_route_list.scoped_route_configurations()) { Protobuf::Message* clone = it.New(); @@ -66,7 +64,7 @@ ConfigProviderPtr ScopedRoutesConfigProviderUtil::maybeCreate( } InlineScopedRoutesConfigProvider::InlineScopedRoutesConfigProvider( - std::vector>&& config_protos, std::string name, + ProtobufTypes::ConstMessagePtrVector&& config_protos, std::string name, Server::Configuration::FactoryContext& factory_context, ScopedRoutesConfigProviderManager& config_provider_manager, envoy::api::v2::core::ConfigSource rds_config_source, @@ -254,7 +252,7 @@ ConfigProviderPtr ScopedRoutesConfigProviderManager::createXdsConfigProvider( } ConfigProviderPtr ScopedRoutesConfigProviderManager::createStaticConfigProvider( - std::vector>&& config_protos, + ProtobufTypes::ConstMessagePtrVector&& config_protos, Server::Configuration::FactoryContext& factory_context, const ConfigProviderManager::OptionalArg& optarg) { const auto& typed_optarg = static_cast(optarg); diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index 9c20e221ec9ea..2627a2461b198 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -32,7 +32,7 @@ class ScopedRoutesConfigProviderManager; class InlineScopedRoutesConfigProvider : public Envoy::Config::ImmutableConfigProviderBase { public: InlineScopedRoutesConfigProvider( - std::vector>&& config_protos, std::string name, + ProtobufTypes::ConstMessagePtrVector&& config_protos, std::string name, Server::Configuration::FactoryContext& factory_context, ScopedRoutesConfigProviderManager& config_provider_manager, envoy::api::v2::core::ConfigSource rds_config_source, From f8421321b4df9c7cf398d63010ba778562760c8e Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Mon, 20 May 2019 16:57:03 -0400 Subject: [PATCH 31/37] Use a namespace instead of a class with a static function. Signed-off-by: Andres Guedez --- source/common/router/scoped_rds.cc | 14 +++++++++----- source/common/router/scoped_rds.h | 21 +++++++++++---------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index e1dbe9d3b8fae..9abb0ea28a7c6 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -18,11 +18,13 @@ using Envoy::Config::ConfigProviderPtr; namespace Envoy { namespace Router { -ConfigProviderPtr ScopedRoutesConfigProviderUtil::maybeCreate( - const envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& - config, - Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, - ConfigProviderManager& scoped_routes_config_provider_manager) { +namespace ScopedRoutesConfigProviderUtil { + +ConfigProviderPtr +create(const envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& + config, + Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, + ConfigProviderManager& scoped_routes_config_provider_manager) { ASSERT(config.route_specifier_case() == envoy::config::filter::network::http_connection_manager:: v2::HttpConnectionManager::kScopedRoutes); @@ -63,6 +65,8 @@ ConfigProviderPtr ScopedRoutesConfigProviderUtil::maybeCreate( } } +} // namespace ScopedRoutesConfigProviderUtil + InlineScopedRoutesConfigProvider::InlineScopedRoutesConfigProvider( ProtobufTypes::ConstMessagePtrVector&& config_protos, std::string name, Server::Configuration::FactoryContext& factory_context, diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index 2627a2461b198..8cb205808400d 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -15,16 +15,17 @@ namespace Envoy { namespace Router { // Scoped routing configuration utilities. -class ScopedRoutesConfigProviderUtil { -public: - // If enabled in the HttpConnectionManager config, returns a ConfigProvider for scoped routing - // configuration. - static Envoy::Config::ConfigProviderPtr maybeCreate( - const envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& - config, - Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, - Envoy::Config::ConfigProviderManager& scoped_routes_config_provider_manager); -}; +namespace ScopedRoutesConfigProviderUtil { + +// If enabled in the HttpConnectionManager config, returns a ConfigProvider for scoped routing +// configuration. +Envoy::Config::ConfigProviderPtr +create(const envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& + config, + Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, + Envoy::Config::ConfigProviderManager& scoped_routes_config_provider_manager); + +} // namespace ScopedRoutesConfigProviderUtil class ScopedRoutesConfigProviderManager; From 68ed29bc2f5c474b71b5d9299c972226fb2e84bd Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Mon, 20 May 2019 20:53:40 -0400 Subject: [PATCH 32/37] Fix clang-tidy issues. Signed-off-by: Andres Guedez --- source/common/router/scoped_config_manager.cc | 2 +- source/common/router/scoped_config_manager.h | 4 ++-- test/common/router/scoped_rds_test.cc | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/common/router/scoped_config_manager.cc b/source/common/router/scoped_config_manager.cc index e8cdc959792f1..2a5b75f3b29ce 100644 --- a/source/common/router/scoped_config_manager.cc +++ b/source/common/router/scoped_config_manager.cc @@ -9,7 +9,7 @@ namespace Router { ScopedRouteInfoConstSharedPtr ScopedConfigManager::addOrUpdateRoutingScope( const envoy::api::v2::ScopedRouteConfiguration& config_proto, const std::string&) { - auto scoped_route_info = std::make_shared(std::move(config_proto)); + auto scoped_route_info = std::make_shared(config_proto); scoped_route_map_[config_proto.name()] = scoped_route_info; return scoped_route_info; } diff --git a/source/common/router/scoped_config_manager.h b/source/common/router/scoped_config_manager.h index c6c3b3570f582..5f8dd6fda878c 100644 --- a/source/common/router/scoped_config_manager.h +++ b/source/common/router/scoped_config_manager.h @@ -12,8 +12,8 @@ namespace Router { // proto. class ScopedRouteInfo { public: - ScopedRouteInfo(const envoy::api::v2::ScopedRouteConfiguration&& config_proto) - : config_proto_(std::move(config_proto)) {} + ScopedRouteInfo(const envoy::api::v2::ScopedRouteConfiguration& config_proto) + : config_proto_(config_proto) {} // TODO(AndresGuedez): Add the necessary APIs required for the scoped routing logic. diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index 309cfa6190c5a..15b558e507950 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -444,7 +444,7 @@ using ScopedRoutesConfigProviderManagerDeathTest = ScopedRoutesConfigProviderMan // Tests that SRDS only allows creation of delta static config providers. TEST_F(ScopedRoutesConfigProviderManagerDeathTest, DeltaStaticConfigProviderOnly) { - EXPECT_DEBUG_DEATH( + EXPECT_DEATH( config_provider_manager_->createStaticConfigProvider( parseScopedRouteConfigurationFromYaml(R"EOF( name: dynamic-foo From a5c87fc2cc5e315c22436679cec817abdc73d549 Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Mon, 20 May 2019 21:12:19 -0400 Subject: [PATCH 33/37] Cleanup. Signed-off-by: Andres Guedez --- source/common/router/scoped_rds.cc | 1 + source/common/router/scoped_rds.h | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 9abb0ea28a7c6..05576543c720c 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -6,6 +6,7 @@ #include "envoy/api/v2/srds.pb.validate.h" #include "common/common/assert.h" +#include "common/common/logger.h" #include "common/config/subscription_factory.h" // Types are deeply nested under Envoy::Config::ConfigProvider; use 'using-directives' across all diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index 8cb205808400d..6b66952c64ff1 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -7,7 +7,6 @@ #include "envoy/config/subscription.h" #include "envoy/stats/scope.h" -#include "common/common/logger.h" #include "common/config/config_provider_impl.h" #include "common/router/scoped_config_impl.h" From 28f39a661ad5ac8424653eb116589865949468d7 Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Mon, 20 May 2019 23:49:17 -0400 Subject: [PATCH 34/37] Remove dead code and other minor cleanup. Signed-off-by: Andres Guedez --- source/common/router/scoped_rds.cc | 53 +-------------------------- source/common/router/scoped_rds.h | 13 ------- test/common/router/scoped_rds_test.cc | 9 ++--- 3 files changed, 5 insertions(+), 70 deletions(-) diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 05576543c720c..d57ad4a0ed7b2 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -19,55 +19,6 @@ using Envoy::Config::ConfigProviderPtr; namespace Envoy { namespace Router { -namespace ScopedRoutesConfigProviderUtil { - -ConfigProviderPtr -create(const envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& - config, - Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, - ConfigProviderManager& scoped_routes_config_provider_manager) { - ASSERT(config.route_specifier_case() == envoy::config::filter::network::http_connection_manager:: - v2::HttpConnectionManager::kScopedRoutes); - - switch (config.scoped_routes().config_specifier_case()) { - case envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: - kScopedRouteConfigurationsList: { - const envoy::config::filter::network::http_connection_manager::v2:: - ScopedRouteConfigurationsList& scoped_route_list = - config.scoped_routes().scoped_route_configurations_list(); - ProtobufTypes::ConstMessagePtrVector config_protos( - scoped_route_list.scoped_route_configurations().size()); - for (const auto& it : scoped_route_list.scoped_route_configurations()) { - Protobuf::Message* clone = it.New(); - clone->CopyFrom(it); - config_protos.push_back(std::unique_ptr(clone)); - } - - return scoped_routes_config_provider_manager.createStaticConfigProvider( - std::move(config_protos), factory_context, - ScopedRoutesConfigProviderManagerOptArg(config.scoped_routes().name(), - config.scoped_routes().rds_config_source(), - config.scoped_routes().scope_key_builder())); - } - - case envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::kScopedRds: - return scoped_routes_config_provider_manager.createXdsConfigProvider( - config.scoped_routes().scoped_rds(), factory_context, stat_prefix, - ScopedRoutesConfigProviderManagerOptArg(config.scoped_routes().name(), - config.scoped_routes().rds_config_source(), - config.scoped_routes().scope_key_builder())); - - case envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes:: - CONFIG_SPECIFIER_NOT_SET: - return nullptr; - - default: - NOT_REACHED_GCOVR_EXCL_LINE; - } -} - -} // namespace ScopedRoutesConfigProviderUtil - InlineScopedRoutesConfigProvider::InlineScopedRoutesConfigProvider( ProtobufTypes::ConstMessagePtrVector&& config_protos, std::string name, Server::Configuration::FactoryContext& factory_context, @@ -213,9 +164,7 @@ ProtobufTypes::MessagePtr ScopedRoutesConfigProviderManager::dumpConfigs() const for (const auto& provider : immutableConfigProviders(ConfigProviderInstanceType::Inline)) { const auto protos_info = provider->configProtoInfoVector(); - if (protos_info == absl::nullopt) { - continue; - } + ASSERT(protos_info != absl::nullopt); auto* inline_config = config_dump->mutable_inline_scoped_route_configs()->Add(); inline_config->set_name(static_cast(provider)->name()); for (const auto& config_proto : protos_info.value().config_protos_) { diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index 6b66952c64ff1..c32ec1012ce5e 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -13,19 +13,6 @@ namespace Envoy { namespace Router { -// Scoped routing configuration utilities. -namespace ScopedRoutesConfigProviderUtil { - -// If enabled in the HttpConnectionManager config, returns a ConfigProvider for scoped routing -// configuration. -Envoy::Config::ConfigProviderPtr -create(const envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& - config, - Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, - Envoy::Config::ConfigProviderManager& scoped_routes_config_provider_manager); - -} // namespace ScopedRoutesConfigProviderUtil - class ScopedRoutesConfigProviderManager; // A ConfigProvider for inline scoped routing configuration. diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index 15b558e507950..83bd3f9365d3e 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -444,16 +444,15 @@ using ScopedRoutesConfigProviderManagerDeathTest = ScopedRoutesConfigProviderMan // Tests that SRDS only allows creation of delta static config providers. TEST_F(ScopedRoutesConfigProviderManagerDeathTest, DeltaStaticConfigProviderOnly) { - EXPECT_DEATH( - config_provider_manager_->createStaticConfigProvider( - parseScopedRouteConfigurationFromYaml(R"EOF( + EXPECT_DEATH(config_provider_manager_->createStaticConfigProvider( + parseScopedRouteConfigurationFromYaml(R"EOF( name: dynamic-foo route_configuration_name: static-foo-route-config key: fragments: { string_key: "172.30.30.10" } )EOF"), - factory_context_, Envoy::Config::ConfigProviderManager::NullOptionalArg()), - ".*SRDS supports delta updates and requires the use of the createStaticConfigProvider().*"); + factory_context_, Envoy::Config::ConfigProviderManager::NullOptionalArg()), + ".*(SRDS.*requires the use of the createStaticConfigProvider()|panic).*"); } } // namespace From a8f97f6b8e9e571d80760d8f15c5c1482b1744e3 Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Tue, 21 May 2019 11:50:25 -0400 Subject: [PATCH 35/37] Remove unused code and other minor cleanup. Signed-off-by: Andres Guedez --- include/envoy/router/scopes.h | 2 +- source/common/router/scoped_config_impl.cc | 2 +- source/common/router/scoped_config_impl.h | 6 ++--- source/common/router/scoped_rds.cc | 31 +++++++--------------- 4 files changed, 13 insertions(+), 28 deletions(-) diff --git a/include/envoy/router/scopes.h b/include/envoy/router/scopes.h index d3207036f5f1c..9eda8e8e3d9b4 100644 --- a/include/envoy/router/scopes.h +++ b/include/envoy/router/scopes.h @@ -21,7 +21,7 @@ class ScopedConfig : public Envoy::Config::ConfigProvider::Config { * @param headers the request headers to match the scoped routing configuration against. * @return ConfigConstSharedPtr the router's Config matching the request headers. */ - virtual ConfigConstSharedPtr getRouterConfig(const Http::HeaderMap& headers) const PURE; + virtual ConfigConstSharedPtr getRouteConfig(const Http::HeaderMap& headers) const PURE; }; using ScopedConfigConstSharedPtr = std::shared_ptr; diff --git a/source/common/router/scoped_config_impl.cc b/source/common/router/scoped_config_impl.cc index 04b153176b041..0274381ae8501 100644 --- a/source/common/router/scoped_config_impl.cc +++ b/source/common/router/scoped_config_impl.cc @@ -8,7 +8,7 @@ void ThreadLocalScopedConfigImpl::addOrUpdateRoutingScope(const ScopedRouteInfoC void ThreadLocalScopedConfigImpl::removeRoutingScope(const std::string&) {} Router::ConfigConstSharedPtr -ThreadLocalScopedConfigImpl::getRouterConfig(const Http::HeaderMap&) const { +ThreadLocalScopedConfigImpl::getRouteConfig(const Http::HeaderMap&) const { return std::make_shared(); } diff --git a/source/common/router/scoped_config_impl.h b/source/common/router/scoped_config_impl.h index 8a812bc71ada9..94d376b2a354e 100644 --- a/source/common/router/scoped_config_impl.h +++ b/source/common/router/scoped_config_impl.h @@ -28,13 +28,11 @@ class ThreadLocalScopedConfigImpl : public ScopedConfig, public ThreadLocal::Thr scope_key_builder) : scope_key_builder_(std::move(scope_key_builder)) {} - ~ThreadLocalScopedConfigImpl() override = default; - void addOrUpdateRoutingScope(const ScopedRouteInfoConstSharedPtr& scoped_route_info); void removeRoutingScope(const std::string& scope_name); // Envoy::Router::ScopedConfig - Router::ConfigConstSharedPtr getRouterConfig(const Http::HeaderMap& headers) const override; + Router::ConfigConstSharedPtr getRouteConfig(const Http::HeaderMap& headers) const override; private: const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder @@ -46,7 +44,7 @@ class ThreadLocalScopedConfigImpl : public ScopedConfig, public ThreadLocal::Thr */ class NullScopedConfigImpl : public ScopedConfig { public: - Router::ConfigConstSharedPtr getRouterConfig(const Http::HeaderMap&) const override { + Router::ConfigConstSharedPtr getRouteConfig(const Http::HeaderMap&) const override { return std::make_shared(); } }; diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index d57ad4a0ed7b2..3a5ac1d76ff20 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -83,24 +83,15 @@ void ScopedRdsConfigSubscription::onConfigUpdate( scoped_config_manager_.scopedRouteMap(); for (auto& scoped_route : scoped_routes) { const std::string& scoped_route_name = scoped_route.name(); - try { - scoped_routes_to_remove.erase(scoped_route_name); - ScopedRouteInfoConstSharedPtr scoped_route_info = - scoped_config_manager_.addOrUpdateRoutingScope(scoped_route, version_info); - if (scoped_route_info == nullptr) { - throw EnvoyException( - fmt::format("failed to create/update global routing scope {}", scoped_route_name)); - } - ENVOY_LOG(debug, "srds: add/update scoped_route '{}'", scoped_route_name); - applyDeltaConfigUpdate( - [scoped_route_info](const ConfigProvider::ConfigConstSharedPtr& config) { - auto* thread_local_scoped_config = const_cast( - static_cast(config.get())); - thread_local_scoped_config->addOrUpdateRoutingScope(scoped_route_info); - }); - } catch (const EnvoyException& ex) { - exception_msgs.push_back(fmt::format("{}: {}", scoped_route_name, ex.what())); - } + scoped_routes_to_remove.erase(scoped_route_name); + ScopedRouteInfoConstSharedPtr scoped_route_info = + scoped_config_manager_.addOrUpdateRoutingScope(scoped_route, version_info); + ENVOY_LOG(debug, "srds: add/update scoped_route '{}'", scoped_route_name); + applyDeltaConfigUpdate([scoped_route_info](const ConfigProvider::ConfigConstSharedPtr& config) { + auto* thread_local_scoped_config = const_cast( + static_cast(config.get())); + thread_local_scoped_config->addOrUpdateRoutingScope(scoped_route_info); + }); } for (const auto& scoped_route : scoped_routes_to_remove) { @@ -116,10 +107,6 @@ void ScopedRdsConfigSubscription::onConfigUpdate( ConfigSubscriptionCommonBase::onConfigUpdate(); setLastConfigInfo(absl::optional({absl::nullopt, version_info})); - if (!exception_msgs.empty()) { - throw EnvoyException(fmt::format("Error adding/updating scoped route(s) {}", - StringUtil::join(exception_msgs, ", "))); - } stats_.config_reload_.inc(); } From bed4340eba0d83191566980a9636145ea7e508aa Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Tue, 21 May 2019 11:56:55 -0400 Subject: [PATCH 36/37] Cleanup. Signed-off-by: Andres Guedez --- test/common/router/scoped_rds_test.cc | 11 +---------- test/mocks/router/mocks.cc | 3 +++ test/mocks/router/mocks.h | 4 ++-- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index 83bd3f9365d3e..2c1f7fcb105cf 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -84,10 +84,6 @@ class ScopedRoutesTestBase : public testing::Test { class ScopedRdsTest : public ScopedRoutesTestBase { protected: - ScopedRdsTest() = default; - - ~ScopedRdsTest() override = default; - void setup() { InSequence s; @@ -265,12 +261,7 @@ TEST_F(ScopedRdsTest, ConfigUpdateFailure) { time); } -class ScopedRoutesConfigProviderManagerTest : public ScopedRoutesTestBase { -public: - ScopedRoutesConfigProviderManagerTest() = default; - - ~ScopedRoutesConfigProviderManagerTest() override = default; -}; +using ScopedRoutesConfigProviderManagerTest = ScopedRoutesTestBase; // Tests that the /config_dump handler returns the corresponding scoped routing config. TEST_F(ScopedRoutesConfigProviderManagerTest, ConfigDump) { diff --git a/test/mocks/router/mocks.cc b/test/mocks/router/mocks.cc index 099d1095c7e70..31c8037701ff5 100644 --- a/test/mocks/router/mocks.cc +++ b/test/mocks/router/mocks.cc @@ -109,5 +109,8 @@ MockRoute::~MockRoute() {} MockRouteConfigProviderManager::MockRouteConfigProviderManager() {} MockRouteConfigProviderManager::~MockRouteConfigProviderManager() {} +MockScopedConfig::MockScopedConfig() {} +MockScopedConfig::~MockScopedConfig() {} + } // namespace Router } // namespace Envoy diff --git a/test/mocks/router/mocks.h b/test/mocks/router/mocks.h index 3cb443786a172..308da62a36c9e 100644 --- a/test/mocks/router/mocks.h +++ b/test/mocks/router/mocks.h @@ -365,8 +365,8 @@ class MockRouteConfigProviderManager : public RouteConfigProviderManager { class MockScopedConfig : public ScopedConfig { public: - MockScopedConfig() = default; - ~MockScopedConfig() override = default; + MockScopedConfig(); + ~MockScopedConfig(); MOCK_CONST_METHOD1(getRouterConfig, ConfigConstSharedPtr(const Http::HeaderMap& headers)); }; From 6716559277c64cebcaa10d7b1b06887b81261338 Mon Sep 17 00:00:00 2001 From: Andres Guedez Date: Tue, 21 May 2019 14:59:59 -0400 Subject: [PATCH 37/37] Fix coverage test. Signed-off-by: Andres Guedez --- test/common/router/scoped_rds_test.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index 2c1f7fcb105cf..7165ee9a1c7d6 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -435,6 +435,7 @@ using ScopedRoutesConfigProviderManagerDeathTest = ScopedRoutesConfigProviderMan // Tests that SRDS only allows creation of delta static config providers. TEST_F(ScopedRoutesConfigProviderManagerDeathTest, DeltaStaticConfigProviderOnly) { + // Use match all regex due to lack of distinctive matchable output for coverage test. EXPECT_DEATH(config_provider_manager_->createStaticConfigProvider( parseScopedRouteConfigurationFromYaml(R"EOF( name: dynamic-foo @@ -443,7 +444,7 @@ route_configuration_name: static-foo-route-config fragments: { string_key: "172.30.30.10" } )EOF"), factory_context_, Envoy::Config::ConfigProviderManager::NullOptionalArg()), - ".*(SRDS.*requires the use of the createStaticConfigProvider()|panic).*"); + ".*"); } } // namespace