diff --git a/envoy/rds/BUILD b/envoy/rds/BUILD new file mode 100644 index 0000000000000..9f20ca9bdff57 --- /dev/null +++ b/envoy/rds/BUILD @@ -0,0 +1,19 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_cc_library( + name = "rds_interface", + hdrs = [ + "config.h", + "config_traits.h", + "route_config_provider.h", + "route_config_update_receiver.h", + ], +) diff --git a/envoy/rds/config.h b/envoy/rds/config.h new file mode 100644 index 0000000000000..8a230a9763c83 --- /dev/null +++ b/envoy/rds/config.h @@ -0,0 +1,19 @@ +#pragma once + +#include + +namespace Envoy { +namespace Rds { + +/** + * Base class for router configuration classes used with Rds. + */ +class Config { +public: + virtual ~Config() = default; +}; + +using ConfigConstSharedPtr = std::shared_ptr; + +} // namespace Rds +} // namespace Envoy diff --git a/envoy/rds/config_traits.h b/envoy/rds/config_traits.h new file mode 100644 index 0000000000000..b7d79018cb5b6 --- /dev/null +++ b/envoy/rds/config_traits.h @@ -0,0 +1,61 @@ +#pragma once + +#include + +#include "envoy/common/pure.h" +#include "envoy/rds/config.h" + +#include "source/common/protobuf/protobuf.h" + +namespace Envoy { +namespace Rds { + +/** + * Traits of the protocol specific route configuration and proto. + * The generic rds classes will call the methods of this interface + * to get information which is not visible for them directly. + */ +class ProtoTraits { +public: + virtual ~ProtoTraits() = default; + + /** + * Give the full name of the route configuration proto description. + * For example 'envoy.config.route.v3.RouteConfiguration' + */ + virtual const std::string& resourceType() const PURE; + + /** + * Gives back the name field tag number of the route configuration proto. + */ + virtual int resourceNameFieldNumber() const PURE; + + /** + * Create an empty route configuration proto object. + */ + virtual ProtobufTypes::MessagePtr createEmptyProto() const PURE; +}; + +class ConfigTraits { +public: + virtual ~ConfigTraits() = default; + + /** + * Create a dummy config object without actual route configuration. + * This object will be used before the first valid route configuration is fetched. + */ + virtual ConfigConstSharedPtr createNullConfig() const PURE; + + /** + * Create a config object based on a route configuration. + * The full name of the type of the parameter message is + * guaranteed to match with the return value of ProtoTraits::resourceType. + * Both dynamic or static cast can be applied to downcast the message + * to the corresponding route configuration class. + * @throw EnvoyException if the new config can't be applied of. + */ + virtual ConfigConstSharedPtr createConfig(const Protobuf::Message& rc) const PURE; +}; + +} // namespace Rds +} // namespace Envoy diff --git a/envoy/rds/route_config_provider.h b/envoy/rds/route_config_provider.h new file mode 100644 index 0000000000000..9b60e3f9780af --- /dev/null +++ b/envoy/rds/route_config_provider.h @@ -0,0 +1,61 @@ +#pragma once + +#include + +#include "envoy/common/time.h" +#include "envoy/rds/config.h" + +#include "source/common/protobuf/protobuf.h" + +#include "absl/types/optional.h" + +namespace Envoy { +namespace Rds { + +/** + * A provider for constant route configurations. + */ +class RouteConfigProvider { +public: + struct ConfigInfo { + // A reference to the currently loaded route configuration. Do not hold this reference beyond + // the caller of configInfo()'s scope. + const Protobuf::Message& config_; + + // The discovery version that supplied this route. This will be set to "" in the case of + // static clusters. + const std::string version_; + }; + + virtual ~RouteConfigProvider() = default; + + /** + * @return ConfigConstSharedPtr a route configuration for use during a single request. The + * returned config may be different on a subsequent call, so a new config should be acquired for + * each request flow. + */ + virtual ConfigConstSharedPtr config() const PURE; + + /** + * @return the configuration information for the currently loaded route configuration. Note that + * if the provider has not yet performed an initial configuration load, no information will be + * returned. + */ + virtual const absl::optional& configInfo() const PURE; + + /** + * @return the last time this RouteConfigProvider was updated. Used for config dumps. + */ + virtual SystemTime lastUpdated() const PURE; + + /** + * Callback used to notify RouteConfigProvider about configuration changes. + */ + virtual void onConfigUpdate() PURE; +}; + +using RouteConfigProviderPtr = std::unique_ptr; +using RouteConfigProviderSharedPtr = std::shared_ptr; + +} // namespace Rds +} // namespace Envoy diff --git a/envoy/rds/route_config_update_receiver.h b/envoy/rds/route_config_update_receiver.h new file mode 100644 index 0000000000000..c4fd2792dc658 --- /dev/null +++ b/envoy/rds/route_config_update_receiver.h @@ -0,0 +1,63 @@ +#pragma once + +#include + +#include "envoy/common/pure.h" +#include "envoy/common/time.h" +#include "envoy/rds/route_config_provider.h" + +#include "absl/types/optional.h" + +namespace Envoy { +namespace Rds { + +/** + * A primitive that keeps track of updates to a RouteConfiguration. + */ +class RouteConfigUpdateReceiver { +public: + virtual ~RouteConfigUpdateReceiver() = default; + + /** + * Called on updates via RDS. + * @param rc supplies the RouteConfiguration. + * @param version_info supplies RouteConfiguration version. + * @return bool whether the hash of the new config has been different than + * the hash of the current one and RouteConfiguration has been updated. + * @throw EnvoyException if the new config is invalid and can't be applied. + */ + virtual bool onRdsUpdate(const Protobuf::Message& rc, const std::string& version_info) PURE; + + /** + * @return uint64_t the hash value of RouteConfiguration. + */ + virtual uint64_t configHash() const PURE; + + /** + * @return absl::optional containing an instance of + * RouteConfigProvider::ConfigInfo if RouteConfiguration has been updated at least once. Otherwise + * returns an empty absl::optional. + */ + virtual const absl::optional& configInfo() const PURE; + + /** + * @return Protobuf::Message& current RouteConfiguration. + */ + virtual const Protobuf::Message& protobufConfiguration() const PURE; + + /** + * @return ConfigConstSharedPtr a parsed and validated copy of current RouteConfiguration. + * @see protobufConfiguration() + */ + virtual ConfigConstSharedPtr parsedConfiguration() const PURE; + + /** + * @return SystemTime the time of the last update. + */ + virtual SystemTime lastUpdated() const PURE; +}; + +using RouteConfigUpdatePtr = std::unique_ptr; + +} // namespace Rds +} // namespace Envoy diff --git a/envoy/router/BUILD b/envoy/router/BUILD index 86bc52bc99f04..31ae02a84d81c 100644 --- a/envoy/router/BUILD +++ b/envoy/router/BUILD @@ -19,6 +19,7 @@ envoy_cc_library( deps = [ ":router_interface", "//envoy/http:filter_interface", + "//envoy/rds:rds_interface", "@envoy_api//envoy/config/route/v3:pkg_cc_proto", ], ) @@ -70,6 +71,7 @@ envoy_cc_library( "//envoy/http:conn_pool_interface", "//envoy/http:hash_policy_interface", "//envoy/http:header_map_interface", + "//envoy/rds:rds_interface", "//envoy/tcp:conn_pool_interface", "//envoy/tracing:http_tracer_interface", "//envoy/upstream:resource_manager_interface", diff --git a/envoy/router/rds.h b/envoy/router/rds.h index 1b4d0ac0431c1..f8a7ad81def14 100644 --- a/envoy/router/rds.h +++ b/envoy/router/rds.h @@ -4,6 +4,7 @@ #include "envoy/config/route/v3/route.pb.h" #include "envoy/http/filter.h" +#include "envoy/rds/route_config_provider.h" #include "envoy/router/router.h" namespace Envoy { @@ -12,43 +13,14 @@ namespace Router { /** * A provider for constant route configurations. */ -class RouteConfigProvider { +class RouteConfigProvider : public Rds::RouteConfigProvider { public: - struct ConfigInfo { - // A reference to the currently loaded route configuration. Do not hold this reference beyond - // the caller of configInfo()'s scope. - const envoy::config::route::v3::RouteConfiguration& config_; - - // The discovery version that supplied this route. This will be set to "" in the case of - // static clusters. - std::string version_; - }; - - virtual ~RouteConfigProvider() = default; - - /** - * @return Router::ConfigConstSharedPtr a route configuration for use during a single request. The - * returned config may be different on a subsequent call, so a new config should be acquired for - * each request flow. - */ - virtual Router::ConfigConstSharedPtr config() PURE; - - /** - * @return the configuration information for the currently loaded route configuration. Note that - * if the provider has not yet performed an initial configuration load, no information will be - * returned. - */ - virtual absl::optional configInfo() const PURE; - - /** - * @return the last time this RouteConfigProvider was updated. Used for config dumps. - */ - virtual SystemTime lastUpdated() const PURE; - /** - * Callback used to notify RouteConfigProvider about configuration changes. + * Same purpose as Rds::RouteConfigProvider::config() + * but the return is downcasted to proper type. + * @return downcasted ConfigConstSharedPtr from Rds::ConfigConstSharedPtr */ - virtual void onConfigUpdate() PURE; + virtual ConfigConstSharedPtr configCast() const PURE; /** * Callback used to request an update to the route configuration from the management server. diff --git a/envoy/router/route_config_update_receiver.h b/envoy/router/route_config_update_receiver.h index a81f33a16f9da..23a2fbe3f9058 100644 --- a/envoy/router/route_config_update_receiver.h +++ b/envoy/router/route_config_update_receiver.h @@ -5,7 +5,7 @@ #include "envoy/common/pure.h" #include "envoy/common/time.h" #include "envoy/config/route/v3/route.pb.h" -#include "envoy/router/rds.h" +#include "envoy/rds/route_config_update_receiver.h" #include "envoy/service/discovery/v3/discovery.pb.h" #include "source/common/protobuf/protobuf.h" @@ -18,19 +18,15 @@ namespace Router { /** * A primitive that keeps track of updates to a RouteConfiguration. */ -class RouteConfigUpdateReceiver { +class RouteConfigUpdateReceiver : public Rds::RouteConfigUpdateReceiver { public: - virtual ~RouteConfigUpdateReceiver() = default; - /** - * Called on updates via RDS. - * @param rc supplies the RouteConfiguration. - * @param version_info supplies RouteConfiguration version. - * @return bool whether RouteConfiguration has been updated. - * @throw EnvoyException if the new config can't be applied. + * Same purpose as Rds::RouteConfigUpdateReceiver::protobufConfiguration() + * but the return is downcasted to proper type. + * @return current RouteConfiguration downcasted from Protobuf::Message& */ - virtual bool onRdsUpdate(const envoy::config::route::v3::RouteConfiguration& rc, - const std::string& version_info) PURE; + virtual const envoy::config::route::v3::RouteConfiguration& + protobufConfigurationCast() const PURE; using VirtualHostRefVector = std::vector>; @@ -48,16 +44,6 @@ class RouteConfigUpdateReceiver { const Protobuf::RepeatedPtrField& removed_resources, const std::string& version_info) PURE; - /** - * @return std::string& the name of RouteConfiguration. - */ - virtual const std::string& routeConfigName() const PURE; - - /** - * @return std::string& the version of RouteConfiguration. - */ - virtual const std::string& configVersion() const PURE; - /** * @return bool return whether VHDS configuration has been changed in the last RDS update. */ @@ -66,34 +52,6 @@ class RouteConfigUpdateReceiver { // intent and the lifecycle of the "last update state" less muddled. virtual bool vhdsConfigurationChanged() const PURE; - /** - * @return uint64_t the hash value of RouteConfiguration. - */ - virtual uint64_t configHash() const PURE; - - /** - * @return absl::optional containing an instance of - * RouteConfigProvider::ConfigInfo if RouteConfiguration has been updated at least once. Otherwise - * returns an empty absl::optional. - */ - virtual absl::optional configInfo() const PURE; - - /** - * @return envoy::config::route::v3::RouteConfiguration& current RouteConfiguration. - */ - virtual const envoy::config::route::v3::RouteConfiguration& protobufConfiguration() PURE; - - /** - * @return Router::ConfigConstSharedPtr a parsed and validated copy of current RouteConfiguration. - * @see protobufConfiguration() - */ - virtual ConfigConstSharedPtr parsedConfiguration() const PURE; - - /** - * @return SystemTime the time of the last update. - */ - virtual SystemTime lastUpdated() const PURE; - /** * @return the union of all resource names and aliases (if any) received with the last VHDS * update. diff --git a/envoy/router/router.h b/envoy/router/router.h index c829262d1403e..a0d30f0c37bec 100644 --- a/envoy/router/router.h +++ b/envoy/router/router.h @@ -18,6 +18,7 @@ #include "envoy/http/codes.h" #include "envoy/http/conn_pool.h" #include "envoy/http/hash_policy.h" +#include "envoy/rds/config.h" #include "envoy/router/internal_redirect.h" #include "envoy/tcp/conn_pool.h" #include "envoy/tracing/http_tracer.h" @@ -1093,10 +1094,8 @@ using RouteCallback = std::functionconfig(); + snapped_route_config_ = connection_manager_.config_.routeConfigProvider()->configCast(); } else if (connection_manager_.config_.scopedRouteConfigProvider() != nullptr) { snapped_scoped_routes_config_ = connection_manager_.config_.scopedRouteConfigProvider()->config(); snapScopedRouteConfig(); } } else { - snapped_route_config_ = connection_manager_.config_.routeConfigProvider()->config(); + snapped_route_config_ = connection_manager_.config_.routeConfigProvider()->configCast(); } ENVOY_STREAM_LOG(debug, "request headers complete (end_stream={}):\n{}", *this, end_stream, @@ -1319,7 +1319,7 @@ void ConnectionManagerImpl::ActiveStream::requestRouteConfigUpdate( absl::optional ConnectionManagerImpl::ActiveStream::routeConfig() { if (connection_manager_.config_.routeConfigProvider() != nullptr) { return absl::optional( - connection_manager_.config_.routeConfigProvider()->config()); + connection_manager_.config_.routeConfigProvider()->configCast()); } return {}; } diff --git a/source/common/rds/BUILD b/source/common/rds/BUILD new file mode 100644 index 0000000000000..d870af8bbe1f3 --- /dev/null +++ b/source/common/rds/BUILD @@ -0,0 +1,48 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_cc_library( + name = "rds_lib", + srcs = [ + "rds_route_config_provider_impl.cc", + "rds_route_config_subscription.cc", + "route_config_provider_manager.cc", + "route_config_update_receiver_impl.cc", + "static_route_config_provider_impl.cc", + "util.cc", + ], + hdrs = [ + "rds_route_config_provider_impl.h", + "rds_route_config_subscription.h", + "route_config_provider_manager.h", + "route_config_update_receiver_impl.h", + "static_route_config_provider_impl.h", + "util.h", + ], + deps = [ + "//envoy/rds:rds_interface", + "//envoy/singleton:instance_interface", + "//envoy/thread_local:thread_local_interface", + "//source/common/common:assert_lib", + "//source/common/common:callback_impl_lib", + "//source/common/common:cleanup_lib", + "//source/common/common:minimal_logger_lib", + "//source/common/config:api_version_lib", + "//source/common/config:subscription_base_interface", + "//source/common/config:subscription_factory_lib", + "//source/common/config:utility_lib", + "//source/common/init:manager_lib", + "//source/common/init:target_lib", + "//source/common/init:watcher_lib", + "//source/common/protobuf:utility_lib", + "@envoy_api//envoy/admin/v3:pkg_cc_proto", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + ], +) diff --git a/source/common/rds/rds_route_config_provider_impl.cc b/source/common/rds/rds_route_config_provider_impl.cc new file mode 100644 index 0000000000000..cc594c7fe0ac6 --- /dev/null +++ b/source/common/rds/rds_route_config_provider_impl.cc @@ -0,0 +1,38 @@ +#include "source/common/rds/rds_route_config_provider_impl.h" + +namespace Envoy { +namespace Rds { + +RdsRouteConfigProviderImpl::RdsRouteConfigProviderImpl( + RdsRouteConfigSubscriptionSharedPtr&& subscription, + Server::Configuration::ServerFactoryContext& factory_context) + : subscription_(std::move(subscription)), + config_update_info_(subscription_->routeConfigUpdate()), tls_(factory_context.threadLocal()) { + + auto initial_config = config_update_info_->parsedConfiguration(); + ASSERT(initial_config); + tls_.set([initial_config](Event::Dispatcher&) { + return std::make_shared(initial_config); + }); + // It should be 1:1 mapping due to shared rds config. + ASSERT(!subscription_->routeConfigProvider().has_value()); + subscription_->routeConfigProvider().emplace(this); +} + +RdsRouteConfigProviderImpl::~RdsRouteConfigProviderImpl() { + ASSERT(subscription_->routeConfigProvider().has_value()); + subscription_->routeConfigProvider().reset(); +} + +const absl::optional& +RdsRouteConfigProviderImpl::configInfo() const { + return config_update_info_->configInfo(); +} + +void RdsRouteConfigProviderImpl::onConfigUpdate() { + tls_.runOnAllThreads([new_config = config_update_info_->parsedConfiguration()]( + OptRef tls) { tls->config_ = new_config; }); +} + +} // namespace Rds +} // namespace Envoy diff --git a/source/common/rds/rds_route_config_provider_impl.h b/source/common/rds/rds_route_config_provider_impl.h new file mode 100644 index 0000000000000..e3ed7c2cc7b54 --- /dev/null +++ b/source/common/rds/rds_route_config_provider_impl.h @@ -0,0 +1,50 @@ +#pragma once + +#include +#include + +#include "envoy/rds/route_config_provider.h" +#include "envoy/rds/route_config_update_receiver.h" +#include "envoy/server/factory_context.h" +#include "envoy/thread_local/thread_local.h" + +#include "source/common/rds/rds_route_config_subscription.h" + +namespace Envoy { +namespace Rds { + +/** + * Implementation of RouteConfigProvider that fetches the route configuration dynamically using + * the subscription. + */ +class RdsRouteConfigProviderImpl : public RouteConfigProvider, + Logger::Loggable { +public: + RdsRouteConfigProviderImpl(RdsRouteConfigSubscriptionSharedPtr&& subscription, + Server::Configuration::ServerFactoryContext& factory_context); + + ~RdsRouteConfigProviderImpl() override; + + RdsRouteConfigSubscription& subscription() { return *subscription_; } + + // RouteConfigProvider + ConfigConstSharedPtr config() const override { return tls_->config_; } + + const absl::optional& configInfo() const override; + SystemTime lastUpdated() const override { return config_update_info_->lastUpdated(); } + void onConfigUpdate() override; + +private: + struct ThreadLocalConfig : public ThreadLocal::ThreadLocalObject { + ThreadLocalConfig(std::shared_ptr initial_config) + : config_(std::move(initial_config)) {} + ConfigConstSharedPtr config_; + }; + + RdsRouteConfigSubscriptionSharedPtr subscription_; + RouteConfigUpdatePtr& config_update_info_; + ThreadLocal::TypedSlot tls_; +}; + +} // namespace Rds +} // namespace Envoy diff --git a/source/common/rds/rds_route_config_subscription.cc b/source/common/rds/rds_route_config_subscription.cc new file mode 100644 index 0000000000000..b61a31921b5ed --- /dev/null +++ b/source/common/rds/rds_route_config_subscription.cc @@ -0,0 +1,137 @@ +#include "source/common/rds/rds_route_config_subscription.h" + +#include "source/common/common/logger.h" +#include "source/common/rds/util.h" + +namespace Envoy { +namespace Rds { + +RdsRouteConfigSubscription::RdsRouteConfigSubscription( + RouteConfigUpdatePtr&& config_update, + std::unique_ptr&& resource_decoder, + const envoy::config::core::v3::ConfigSource& config_source, + const std::string& route_config_name, const uint64_t manager_identifier, + Server::Configuration::ServerFactoryContext& factory_context, const std::string& stat_prefix, + const std::string& rds_type, RouteConfigProviderManager& route_config_provider_manager) + : route_config_name_(route_config_name), + scope_(factory_context.scope().createScope(stat_prefix + route_config_name_ + ".")), + factory_context_(factory_context), + parent_init_target_( + fmt::format("RdsRouteConfigSubscription {} init {}", rds_type, route_config_name_), + [this]() { local_init_manager_.initialize(local_init_watcher_); }), + local_init_watcher_(fmt::format("{} local-init-watcher {}", rds_type, route_config_name_), + [this]() { parent_init_target_.ready(); }), + local_init_target_(fmt::format("RdsRouteConfigSubscription {} local-init-target {}", rds_type, + route_config_name_), + [this]() { subscription_->start({route_config_name_}); }), + local_init_manager_(fmt::format("{} local-init-manager {}", rds_type, route_config_name_)), + stat_prefix_(stat_prefix), rds_type_(rds_type), + stats_({ALL_RDS_STATS(POOL_COUNTER(*scope_), POOL_GAUGE(*scope_))}), + route_config_provider_manager_(route_config_provider_manager), + manager_identifier_(manager_identifier), config_update_info_(std::move(config_update)), + resource_decoder_(std::move(resource_decoder)) { + const auto resource_type = route_config_provider_manager_.protoTraits().resourceType(); + subscription_ = + factory_context.clusterManager().subscriptionFactory().subscriptionFromConfigSource( + config_source, Envoy::Grpc::Common::typeUrl(resource_type), *scope_, *this, + *resource_decoder_, {}); + local_init_manager_.add(local_init_target_); +} + +RdsRouteConfigSubscription::~RdsRouteConfigSubscription() { + // If we get destroyed during initialization, make sure we signal that we "initialized". + local_init_target_.ready(); + + // The ownership of RdsRouteConfigProviderImpl is shared among all HttpConnectionManagers that + // hold a shared_ptr to it. The RouteConfigProviderManager holds weak_ptrs to the + // RdsRouteConfigProviders. Therefore, the map entry for the RdsRouteConfigProvider has to get + // cleaned by the RdsRouteConfigProvider's destructor. + route_config_provider_manager_.eraseDynamicProvider(manager_identifier_); +} + +absl::optional& RdsRouteConfigSubscription::routeConfigProvider() { + return route_config_provider_opt_; +} + +void RdsRouteConfigSubscription::onConfigUpdate( + const std::vector& resources, + const std::string& version_info) { + if (!validateUpdateSize(resources.size())) { + return; + } + const auto& route_config = resources[0].get().resource(); + if (route_config.GetDescriptor()->full_name() != + route_config_provider_manager_.protoTraits().resourceType()) { + throw EnvoyException(fmt::format("Unexpected {} configuration type (expecting {}): {}", + rds_type_, + route_config_provider_manager_.protoTraits().resourceType(), + route_config.GetDescriptor()->full_name())); + } + if (resourceName(route_config_provider_manager_.protoTraits(), route_config) != + route_config_name_) { + throw EnvoyException( + fmt::format("Unexpected {} configuration (expecting {}): {}", rds_type_, route_config_name_, + resourceName(route_config_provider_manager_.protoTraits(), route_config))); + } + std::unique_ptr noop_init_manager; + std::unique_ptr resume_rds; + if (config_update_info_->onRdsUpdate(route_config, version_info)) { + stats_.config_reload_.inc(); + stats_.config_reload_time_ms_.set(DateUtil::nowToMilliseconds(factory_context_.timeSource())); + + beforeProviderUpdate(noop_init_manager, resume_rds); + + ENVOY_LOG(debug, "rds: loading new configuration: config_name={} hash={}", route_config_name_, + config_update_info_->configHash()); + + if (route_config_provider_opt_.has_value()) { + route_config_provider_opt_.value()->onConfigUpdate(); + } + + afterProviderUpdate(); + } + + local_init_target_.ready(); +} + +void RdsRouteConfigSubscription::onConfigUpdate( + const std::vector& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, const std::string&) { + if (!removed_resources.empty()) { + // TODO(#2500) when on-demand resource loading is supported, an RDS removal may make sense + // (see discussion in #6879), and so we should do something other than ignoring here. + ENVOY_LOG(error, + "Server sent a delta {} update attempting to remove a resource (name: {}). Ignoring.", + rds_type_, removed_resources[0]); + } + if (!added_resources.empty()) { + onConfigUpdate(added_resources, added_resources[0].get().version()); + } +} + +void RdsRouteConfigSubscription::onConfigUpdateFailed( + Envoy::Config::ConfigUpdateFailureReason reason, const EnvoyException*) { + ASSERT(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure != reason); + // We need to allow server startup to continue, even if we have a bad + // config. + local_init_target_.ready(); +} + +bool RdsRouteConfigSubscription::validateUpdateSize(int num_resources) { + if (num_resources == 0) { + ENVOY_LOG(debug, "Missing {} RouteConfiguration for {} in onConfigUpdate()", rds_type_, + route_config_name_); + stats_.update_empty_.inc(); + local_init_target_.ready(); + return false; + } + if (num_resources != 1) { + throw EnvoyException( + fmt::format("Unexpected {} resource length: {}", rds_type_, num_resources)); + // (would be a return false here) + } + return true; +} + +} // namespace Rds +} // namespace Envoy diff --git a/source/common/rds/rds_route_config_subscription.h b/source/common/rds/rds_route_config_subscription.h new file mode 100644 index 0000000000000..d1753d105a783 --- /dev/null +++ b/source/common/rds/rds_route_config_subscription.h @@ -0,0 +1,108 @@ +#pragma once + +#include +#include + +#include "envoy/config/core/v3/config_source.pb.h" +#include "envoy/config/subscription.h" +#include "envoy/rds/route_config_provider.h" +#include "envoy/rds/route_config_update_receiver.h" +#include "envoy/server/factory_context.h" +#include "envoy/stats/scope.h" + +#include "source/common/grpc/common.h" +#include "source/common/init/manager_impl.h" +#include "source/common/init/target_impl.h" +#include "source/common/init/watcher_impl.h" +#include "source/common/rds/route_config_provider_manager.h" + +#include "absl/types/optional.h" + +namespace Envoy { +namespace Rds { + +/** + * All RDS stats. @see stats_macros.h + */ +#define ALL_RDS_STATS(COUNTER, GAUGE) \ + COUNTER(config_reload) \ + COUNTER(update_empty) \ + GAUGE(config_reload_time_ms, NeverImport) + +/** + * Struct definition for all RDS stats. @see stats_macros.h + */ +struct RdsStats { + ALL_RDS_STATS(GENERATE_COUNTER_STRUCT, GENERATE_GAUGE_STRUCT) +}; + +/** + * A class that fetches the route configuration dynamically using the RDS API and updates them to + * RDS config providers. + */ +class RdsRouteConfigSubscription : Envoy::Config::SubscriptionCallbacks, + protected Logger::Loggable { +public: + RdsRouteConfigSubscription( + RouteConfigUpdatePtr&& config_update, + std::unique_ptr&& resource_decoder, + const envoy::config::core::v3::ConfigSource& config_source, + const std::string& route_config_name, const uint64_t manager_identifier, + Server::Configuration::ServerFactoryContext& factory_context, const std::string& stat_prefix, + const std::string& rds_type, RouteConfigProviderManager& route_config_provider_manager); + + ~RdsRouteConfigSubscription() override; + + absl::optional& routeConfigProvider(); + + RouteConfigUpdatePtr& routeConfigUpdate() { return config_update_info_; } + + const Init::Target& initTarget() { return parent_init_target_; } + +private: + // Config::SubscriptionCallbacks + void onConfigUpdate(const std::vector& resources, + const std::string& version_info) override; + + void onConfigUpdate(const std::vector& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string&) override; + + void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason reason, + const EnvoyException*) override; + + bool validateUpdateSize(int num_resources); + + virtual void beforeProviderUpdate(std::unique_ptr&, + std::unique_ptr&) {} + virtual void afterProviderUpdate() {} + +protected: + const std::string route_config_name_; + // This scope must outlive the subscription_ below as the subscription has derived stats. + Stats::ScopePtr scope_; + Envoy::Config::SubscriptionPtr subscription_; + Server::Configuration::ServerFactoryContext& factory_context_; + + // Init target used to notify the parent init manager that the subscription [and its sub resource] + // is ready. + Init::SharedTargetImpl parent_init_target_; + // Init watcher on RDS and VHDS ready event. This watcher marks parent_init_target_ ready. + Init::WatcherImpl local_init_watcher_; + // Target which starts the RDS subscription. + Init::TargetImpl local_init_target_; + Init::ManagerImpl local_init_manager_; + const std::string stat_prefix_; + const std::string rds_type_; + RdsStats stats_; + RouteConfigProviderManager& route_config_provider_manager_; + const uint64_t manager_identifier_; + absl::optional route_config_provider_opt_; + RouteConfigUpdatePtr config_update_info_; + std::unique_ptr resource_decoder_; +}; + +using RdsRouteConfigSubscriptionSharedPtr = std::shared_ptr; + +} // namespace Rds +} // namespace Envoy diff --git a/source/common/rds/route_config_provider_manager.cc b/source/common/rds/route_config_provider_manager.cc new file mode 100644 index 0000000000000..cdc9d05e4205f --- /dev/null +++ b/source/common/rds/route_config_provider_manager.cc @@ -0,0 +1,111 @@ +#include "source/common/rds/route_config_provider_manager.h" + +#include "source/common/rds/util.h" + +namespace Envoy { +namespace Rds { + +RouteConfigProviderManager::RouteConfigProviderManager(Server::Admin& admin, + const std::string& config_tracker_key, + ProtoTraits& proto_traits) + : config_tracker_entry_(admin.getConfigTracker().add( + config_tracker_key, + [this](const Matchers::StringMatcher& matcher) { return dumpRouteConfigs(matcher); })), + proto_traits_(proto_traits) { + // ConfigTracker keys must be unique. We are asserting that no one has stolen the "routes" key + // from us, since the returned entry will be nullptr if the key already exists. + RELEASE_ASSERT(config_tracker_entry_, ""); +} + +void RouteConfigProviderManager::eraseStaticProvider(RouteConfigProvider* provider) { + static_route_config_providers_.erase(provider); +} + +void RouteConfigProviderManager::eraseDynamicProvider(uint64_t manager_identifier) { + dynamic_route_config_providers_.erase(manager_identifier); +} + +std::unique_ptr +RouteConfigProviderManager::dumpRouteConfigs(const Matchers::StringMatcher& name_matcher) const { + auto config_dump = std::make_unique(); + + for (const auto& element : dynamic_route_config_providers_) { + const auto provider = element.second.first.lock(); + // Because the RouteConfigProviderManager's weak_ptrs only get cleaned up + // in the RdsRouteConfigSubscription destructor, and the single threaded nature + // of this code, locking the weak_ptr will not fail. + ASSERT(provider); + + if (provider->configInfo()) { + if (!name_matcher.match( + resourceName(proto_traits_, provider->configInfo().value().config_))) { + continue; + } + auto* dynamic_config = config_dump->mutable_dynamic_route_configs()->Add(); + dynamic_config->set_version_info(provider->configInfo().value().version_); + dynamic_config->mutable_route_config()->PackFrom(provider->configInfo().value().config_); + TimestampUtil::systemClockToTimestamp(provider->lastUpdated(), + *dynamic_config->mutable_last_updated()); + } + } + + for (const auto& provider : static_route_config_providers_) { + ASSERT(provider->configInfo()); + if (!name_matcher.match(resourceName(proto_traits_, provider->configInfo().value().config_))) { + continue; + } + auto* static_config = config_dump->mutable_static_route_configs()->Add(); + static_config->mutable_route_config()->PackFrom(provider->configInfo().value().config_); + TimestampUtil::systemClockToTimestamp(provider->lastUpdated(), + *static_config->mutable_last_updated()); + } + + return config_dump; +} + +RouteConfigProviderPtr RouteConfigProviderManager::addStaticProvider( + std::function create_static_provider) { + auto provider = create_static_provider(); + static_route_config_providers_.insert(provider.get()); + return provider; +} + +RouteConfigProviderSharedPtr RouteConfigProviderManager::addDynamicProvider( + const Protobuf::Message& rds, const std::string& route_config_name, Init::Manager& init_manager, + std::function< + std::pair(uint64_t manager_identifier)> + create_dynamic_provider) { + // RdsRouteConfigSubscriptions are unique based on their serialized RDS config. + const uint64_t manager_identifier = MessageUtil::hash(rds); + auto existing_provider = + reuseDynamicProvider(manager_identifier, init_manager, route_config_name); + + if (existing_provider) { + return existing_provider; + } + auto new_provider = create_dynamic_provider(manager_identifier); + init_manager.add(*new_provider.second); + dynamic_route_config_providers_.insert({manager_identifier, new_provider}); + return new_provider.first; +} + +RouteConfigProviderSharedPtr +RouteConfigProviderManager::reuseDynamicProvider(uint64_t manager_identifier, + Init::Manager& init_manager, + const std::string& route_config_name) { + auto it = dynamic_route_config_providers_.find(manager_identifier); + if (it == dynamic_route_config_providers_.end()) { + return nullptr; + } + // Because the RouteConfigProviderManager's weak_ptrs only get cleaned up + // in the RdsRouteConfigSubscription destructor, and the single threaded nature + // of this code, locking the weak_ptr will not fail. + auto existing_provider = it->second.first.lock(); + RELEASE_ASSERT(existing_provider != nullptr, + absl::StrCat("cannot find subscribed rds resource ", route_config_name)); + init_manager.add(*it->second.second); + return existing_provider; +} + +} // namespace Rds +} // namespace Envoy diff --git a/source/common/rds/route_config_provider_manager.h b/source/common/rds/route_config_provider_manager.h new file mode 100644 index 0000000000000..e94bb36b06ad4 --- /dev/null +++ b/source/common/rds/route_config_provider_manager.h @@ -0,0 +1,60 @@ +#pragma once + +#include +#include + +#include "envoy/admin/v3/config_dump.pb.h" +#include "envoy/init/manager.h" +#include "envoy/init/target.h" +#include "envoy/rds/config_traits.h" +#include "envoy/rds/route_config_provider.h" +#include "envoy/server/admin.h" + +#include "source/common/common/matchers.h" +#include "source/common/protobuf/utility.h" + +#include "absl/container/node_hash_map.h" +#include "absl/container/node_hash_set.h" + +namespace Envoy { +namespace Rds { + +class RouteConfigProviderManager { +public: + RouteConfigProviderManager(Server::Admin& admin, const std::string& config_tracker_key, + ProtoTraits& proto_traits); + + void eraseStaticProvider(RouteConfigProvider* provider); + void eraseDynamicProvider(uint64_t manager_identifier); + + ProtoTraits& protoTraits() { return proto_traits_; } + + std::unique_ptr + dumpRouteConfigs(const Matchers::StringMatcher& name_matcher) const; + + RouteConfigProviderPtr + addStaticProvider(std::function create_static_provider); + RouteConfigProviderSharedPtr + addDynamicProvider(const Protobuf::Message& rds, const std::string& route_config_name, + Init::Manager& init_manager, + std::function( + uint64_t manager_identifier)> + create_dynamic_provider); + +private: + // TODO(jsedgwick) These two members are prime candidates for the owned-entry list/map + // as in ConfigTracker. I.e. the ProviderImpls would have an EntryOwner for these lists + // Then the lifetime management stuff is centralized and opaque. + absl::node_hash_map, const Init::Target*>> + dynamic_route_config_providers_; + absl::node_hash_set static_route_config_providers_; + Server::ConfigTracker::EntryOwnerPtr config_tracker_entry_; + ProtoTraits& proto_traits_; + + RouteConfigProviderSharedPtr reuseDynamicProvider(uint64_t manager_identifier, + Init::Manager& init_manager, + const std::string& route_config_name); +}; + +} // namespace Rds +} // namespace Envoy diff --git a/source/common/rds/route_config_update_receiver_impl.cc b/source/common/rds/route_config_update_receiver_impl.cc new file mode 100644 index 0000000000000..dee58fa4cbeeb --- /dev/null +++ b/source/common/rds/route_config_update_receiver_impl.cc @@ -0,0 +1,48 @@ +#include "source/common/rds/route_config_update_receiver_impl.h" + +#include "source/common/rds/util.h" + +namespace Envoy { +namespace Rds { + +RouteConfigUpdateReceiverImpl::RouteConfigUpdateReceiverImpl( + ConfigTraits& config_traits, ProtoTraits& proto_traits, + Server::Configuration::ServerFactoryContext& factory_context) + : config_traits_(config_traits), proto_traits_(proto_traits), + time_source_(factory_context.timeSource()), + route_config_proto_(proto_traits_.createEmptyProto()), last_config_hash_(0ull), + config_(config_traits_.createNullConfig()) {} + +void RouteConfigUpdateReceiverImpl::updateConfig( + std::unique_ptr&& route_config_proto) { + config_ = config_traits_.createConfig(*route_config_proto); + // If the above create config doesn't raise exception, update the + // other cached config entries. + route_config_proto_ = std::move(route_config_proto); +} + +void RouteConfigUpdateReceiverImpl::onUpdateCommon(const std::string& version_info) { + last_updated_ = time_source_.systemTime(); + config_info_.emplace(RouteConfigProvider::ConfigInfo{*route_config_proto_, version_info}); +} + +// Rds::RouteConfigUpdateReceiver +bool RouteConfigUpdateReceiverImpl::onRdsUpdate(const Protobuf::Message& rc, + const std::string& version_info) { + uint64_t new_hash = getHash(rc); + if (!checkHash(new_hash)) { + return false; + } + updateConfig(cloneProto(proto_traits_, rc)); + updateHash(new_hash); + onUpdateCommon(version_info); + return true; +} + +const absl::optional& +RouteConfigUpdateReceiverImpl::configInfo() const { + return config_info_; +} + +} // namespace Rds +} // namespace Envoy diff --git a/source/common/rds/route_config_update_receiver_impl.h b/source/common/rds/route_config_update_receiver_impl.h new file mode 100644 index 0000000000000..27cd5c1383855 --- /dev/null +++ b/source/common/rds/route_config_update_receiver_impl.h @@ -0,0 +1,44 @@ +#pragma once + +#include + +#include "envoy/rds/config_traits.h" +#include "envoy/rds/route_config_update_receiver.h" +#include "envoy/server/factory_context.h" + +namespace Envoy { +namespace Rds { + +class RouteConfigUpdateReceiverImpl : public RouteConfigUpdateReceiver { +public: + RouteConfigUpdateReceiverImpl(ConfigTraits& config_traits, ProtoTraits& proto_traits, + Server::Configuration::ServerFactoryContext& factory_context); + + uint64_t getHash(const Protobuf::Message& rc) const { return MessageUtil::hash(rc); } + bool checkHash(uint64_t new_hash) const { return (new_hash != last_config_hash_); } + void updateHash(uint64_t hash) { last_config_hash_ = hash; } + void updateConfig(std::unique_ptr&& route_config_proto); + void onUpdateCommon(const std::string& version_info); + + // RouteConfigUpdateReceiver + bool onRdsUpdate(const Protobuf::Message& rc, const std::string& version_info) override; + + uint64_t configHash() const override { return last_config_hash_; } + const absl::optional& configInfo() const override; + const Protobuf::Message& protobufConfiguration() const override { return *route_config_proto_; } + ConfigConstSharedPtr parsedConfiguration() const override { return config_; } + SystemTime lastUpdated() const override { return last_updated_; } + +private: + ConfigTraits& config_traits_; + ProtoTraits& proto_traits_; + TimeSource& time_source_; + ProtobufTypes::MessagePtr route_config_proto_; + uint64_t last_config_hash_; + SystemTime last_updated_; + absl::optional config_info_; + ConfigConstSharedPtr config_; +}; + +} // namespace Rds +} // namespace Envoy diff --git a/source/common/rds/static_route_config_provider_impl.cc b/source/common/rds/static_route_config_provider_impl.cc new file mode 100644 index 0000000000000..fe43895a0adfc --- /dev/null +++ b/source/common/rds/static_route_config_provider_impl.cc @@ -0,0 +1,24 @@ +#include "source/common/rds/static_route_config_provider_impl.h" + +#include "source/common/rds/util.h" + +namespace Envoy { +namespace Rds { + +StaticRouteConfigProviderImpl::StaticRouteConfigProviderImpl( + const Protobuf::Message& route_config_proto, ConfigTraits& config_traits, + Server::Configuration::ServerFactoryContext& factory_context, + RouteConfigProviderManager& route_config_provider_manager) + : route_config_proto_( + cloneProto(route_config_provider_manager.protoTraits(), route_config_proto)), + config_(config_traits.createConfig(*route_config_proto_)), + last_updated_(factory_context.timeSource().systemTime()), + config_info_(ConfigInfo{*route_config_proto_, ""}), + route_config_provider_manager_(route_config_provider_manager) {} + +StaticRouteConfigProviderImpl::~StaticRouteConfigProviderImpl() { + route_config_provider_manager_.eraseStaticProvider(this); +} + +} // namespace Rds +} // namespace Envoy diff --git a/source/common/rds/static_route_config_provider_impl.h b/source/common/rds/static_route_config_provider_impl.h new file mode 100644 index 0000000000000..196f7a9efb11e --- /dev/null +++ b/source/common/rds/static_route_config_provider_impl.h @@ -0,0 +1,39 @@ +#pragma once + +#include "envoy/rds/config_traits.h" +#include "envoy/rds/route_config_provider.h" +#include "envoy/server/factory_context.h" + +#include "source/common/rds/route_config_provider_manager.h" + +namespace Envoy { +namespace Rds { + +/** + * Implementation of RouteConfigProvider that holds a static route configuration. + */ +class StaticRouteConfigProviderImpl : public RouteConfigProvider { +public: + StaticRouteConfigProviderImpl(const Protobuf::Message& route_config_proto, + ConfigTraits& config_traits, + Server::Configuration::ServerFactoryContext& factory_context, + RouteConfigProviderManager& route_config_provider_manager); + + ~StaticRouteConfigProviderImpl() override; + + // Router::RouteConfigProvider + ConfigConstSharedPtr config() const override { return config_; } + const absl::optional& configInfo() const override { return config_info_; } + SystemTime lastUpdated() const override { return last_updated_; } + void onConfigUpdate() override {} + +private: + ProtobufTypes::MessagePtr route_config_proto_; + ConfigConstSharedPtr config_; + SystemTime last_updated_; + absl::optional config_info_; + RouteConfigProviderManager& route_config_provider_manager_; +}; + +} // namespace Rds +} // namespace Envoy diff --git a/source/common/rds/util.cc b/source/common/rds/util.cc new file mode 100644 index 0000000000000..c9c8d017a2ac0 --- /dev/null +++ b/source/common/rds/util.cc @@ -0,0 +1,23 @@ +#include "source/common/rds/util.h" + +namespace Envoy { +namespace Rds { + +ProtobufTypes::MessagePtr cloneProto(ProtoTraits& proto_traits, const Protobuf::Message& rc) { + auto clone = proto_traits.createEmptyProto(); + clone->CopyFrom(rc); + return clone; +} + +std::string resourceName(ProtoTraits& proto_traits, const Protobuf::Message& rc) { + const Protobuf::FieldDescriptor* field = + rc.GetDescriptor()->FindFieldByNumber(proto_traits.resourceNameFieldNumber()); + if (!field) { + return std::string(); + } + const Protobuf::Reflection* reflection = rc.GetReflection(); + return reflection->GetString(rc, field); +} + +} // namespace Rds +} // namespace Envoy diff --git a/source/common/rds/util.h b/source/common/rds/util.h new file mode 100644 index 0000000000000..71c9e673028e0 --- /dev/null +++ b/source/common/rds/util.h @@ -0,0 +1,12 @@ +#pragma once + +#include "envoy/rds/config_traits.h" + +namespace Envoy { +namespace Rds { + +ProtobufTypes::MessagePtr cloneProto(ProtoTraits& proto_traits, const Protobuf::Message& rc); +std::string resourceName(ProtoTraits& proto_traits, const Protobuf::Message& rc); + +} // namespace Rds +} // namespace Envoy diff --git a/source/common/router/BUILD b/source/common/router/BUILD index cc7b7b979edcc..4c1cd46d09772 100644 --- a/source/common/router/BUILD +++ b/source/common/router/BUILD @@ -128,6 +128,7 @@ envoy_cc_library( "//source/common/common:assert_lib", "//source/common/common:minimal_logger_lib", "//source/common/protobuf:utility_lib", + "//source/common/rds:rds_lib", "@envoy_api//envoy/config/route/v3:pkg_cc_proto", "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", ], @@ -174,20 +175,7 @@ envoy_cc_library( "//envoy/router:route_config_provider_manager_interface", "//envoy/router:route_config_update_info_interface", "//envoy/server:admin_interface", - "//envoy/singleton:instance_interface", - "//envoy/thread_local:thread_local_interface", - "//source/common/common:assert_lib", - "//source/common/common:callback_impl_lib", - "//source/common/common:cleanup_lib", - "//source/common/common:minimal_logger_lib", - "//source/common/config:api_version_lib", - "//source/common/config:subscription_base_interface", - "//source/common/config:subscription_factory_lib", - "//source/common/config:utility_lib", - "//source/common/init:manager_lib", - "//source/common/init:target_lib", - "//source/common/init:watcher_lib", - "//source/common/protobuf:utility_lib", + "//source/common/rds:rds_lib", "//source/common/router:route_config_update_impl_lib", "//source/common/router:vhds_lib", "@envoy_api//envoy/admin/v3:pkg_cc_proto", diff --git a/source/common/router/rds_impl.cc b/source/common/router/rds_impl.cc index 00cf44c82907b..7c8887fec9bae 100644 --- a/source/common/router/rds_impl.cc +++ b/source/common/router/rds_impl.cc @@ -17,6 +17,7 @@ #include "source/common/http/header_map_impl.h" #include "source/common/protobuf/utility.h" #include "source/common/router/config_impl.h" +#include "source/common/router/route_config_update_receiver_impl.h" namespace Envoy { namespace Router { @@ -51,110 +52,61 @@ RouteConfigProviderSharedPtr RouteConfigProviderUtil::create( } StaticRouteConfigProviderImpl::StaticRouteConfigProviderImpl( - const envoy::config::route::v3::RouteConfiguration& config, - const OptionalHttpFilters& optional_http_filters, + const envoy::config::route::v3::RouteConfiguration& config, Rds::ConfigTraits& config_traits, Server::Configuration::ServerFactoryContext& factory_context, - ProtobufMessage::ValidationVisitor& validator, - RouteConfigProviderManagerImpl& route_config_provider_manager) - : config_(new ConfigImpl(config, optional_http_filters, factory_context, validator, true)), - route_config_proto_{config}, last_updated_(factory_context.timeSource().systemTime()), - route_config_provider_manager_(route_config_provider_manager) { - route_config_provider_manager_.static_route_config_providers_.insert(this); -} + Rds::RouteConfigProviderManager& route_config_provider_manager) + : base_(config, config_traits, factory_context, route_config_provider_manager), + route_config_provider_manager_(route_config_provider_manager) {} StaticRouteConfigProviderImpl::~StaticRouteConfigProviderImpl() { - route_config_provider_manager_.static_route_config_providers_.erase(this); + route_config_provider_manager_.eraseStaticProvider(this); +} + +ConfigConstSharedPtr StaticRouteConfigProviderImpl::configCast() const { + ASSERT(dynamic_cast(StaticRouteConfigProviderImpl::config().get())); + return std::static_pointer_cast(StaticRouteConfigProviderImpl::config()); } // TODO(htuch): If support for multiple clusters is added per #1170 cluster_name_ RdsRouteConfigSubscription::RdsRouteConfigSubscription( + RouteConfigUpdatePtr&& config_update, + std::unique_ptr&& resource_decoder, const envoy::extensions::filters::network::http_connection_manager::v3::Rds& rds, const uint64_t manager_identifier, Server::Configuration::ServerFactoryContext& factory_context, - const std::string& stat_prefix, const OptionalHttpFilters& optional_http_filters, - Envoy::Router::RouteConfigProviderManagerImpl& route_config_provider_manager) - : Envoy::Config::SubscriptionBase( - factory_context.messageValidationContext().dynamicValidationVisitor(), "name"), - route_config_name_(rds.route_config_name()), - scope_(factory_context.scope().createScope(stat_prefix + "rds." + route_config_name_ + ".")), - factory_context_(factory_context), - parent_init_target_(fmt::format("RdsRouteConfigSubscription init {}", route_config_name_), - [this]() { local_init_manager_.initialize(local_init_watcher_); }), - local_init_watcher_(fmt::format("RDS local-init-watcher {}", rds.route_config_name()), - [this]() { parent_init_target_.ready(); }), - local_init_target_( - fmt::format("RdsRouteConfigSubscription local-init-target {}", route_config_name_), - [this]() { subscription_->start({route_config_name_}); }), - local_init_manager_(fmt::format("RDS local-init-manager {}", route_config_name_)), - stat_prefix_(stat_prefix), - stats_({ALL_RDS_STATS(POOL_COUNTER(*scope_), POOL_GAUGE(*scope_))}), - route_config_provider_manager_(route_config_provider_manager), - manager_identifier_(manager_identifier), optional_http_filters_(optional_http_filters) { - const auto resource_name = getResourceName(); - subscription_ = - factory_context.clusterManager().subscriptionFactory().subscriptionFromConfigSource( - rds.config_source(), Grpc::Common::typeUrl(resource_name), *scope_, *this, - resource_decoder_, {}); - local_init_manager_.add(local_init_target_); - config_update_info_ = - std::make_unique(factory_context, optional_http_filters_); -} - -RdsRouteConfigSubscription::~RdsRouteConfigSubscription() { - // If we get destroyed during initialization, make sure we signal that we "initialized". - local_init_target_.ready(); - - // The ownership of RdsRouteConfigProviderImpl is shared among all HttpConnectionManagers that - // hold a shared_ptr to it. The RouteConfigProviderManager holds weak_ptrs to the - // RdsRouteConfigProviders. Therefore, the map entry for the RdsRouteConfigProvider has to get - // cleaned by the RdsRouteConfigProvider's destructor. - route_config_provider_manager_.dynamic_route_config_providers_.erase(manager_identifier_); -} - -void RdsRouteConfigSubscription::onConfigUpdate( - const std::vector& resources, - const std::string& version_info) { - if (!validateUpdateSize(resources.size())) { - return; + const std::string& stat_prefix, Rds::RouteConfigProviderManager& route_config_provider_manager) + : Rds::RdsRouteConfigSubscription(std::move(config_update), std::move(resource_decoder), + rds.config_source(), rds.route_config_name(), + manager_identifier, factory_context, stat_prefix + "rds.", + "RDS", route_config_provider_manager), + config_update_info_(static_cast( + Rds::RdsRouteConfigSubscription::config_update_info_.get())) {} + +RdsRouteConfigSubscription::~RdsRouteConfigSubscription() { config_update_info_.release(); } + +void RdsRouteConfigSubscription::beforeProviderUpdate( + std::unique_ptr& noop_init_manager, std::unique_ptr& resume_rds) { + if (config_update_info_->protobufConfigurationCast().has_vhds() && + config_update_info_->vhdsConfigurationChanged()) { + ENVOY_LOG(debug, + "rds: vhds configuration present/changed, (re)starting vhds: config_name={} hash={}", + route_config_name_, routeConfigUpdate()->configHash()); + ASSERT(config_update_info_->configInfo().has_value()); + maybeCreateInitManager(routeConfigUpdate()->configInfo().value().version_, noop_init_manager, + resume_rds); + vhds_subscription_ = std::make_unique( + config_update_info_, factory_context_, stat_prefix_, route_config_provider_opt_); + vhds_subscription_->registerInitTargetWithInitManager( + noop_init_manager == nullptr ? local_init_manager_ : *noop_init_manager); } - const auto& route_config = dynamic_cast( - resources[0].get().resource()); - if (route_config.name() != route_config_name_) { - throw EnvoyException(fmt::format("Unexpected RDS configuration (expecting {}): {}", - route_config_name_, route_config.name())); - } - std::unique_ptr noop_init_manager; - std::unique_ptr resume_rds; - if (config_update_info_->onRdsUpdate(route_config, version_info)) { - stats_.config_reload_.inc(); - stats_.config_reload_time_ms_.set(DateUtil::nowToMilliseconds(factory_context_.timeSource())); - if (config_update_info_->protobufConfiguration().has_vhds() && - config_update_info_->vhdsConfigurationChanged()) { - ENVOY_LOG( - debug, - "rds: vhds configuration present/changed, (re)starting vhds: config_name={} hash={}", - route_config_name_, config_update_info_->configHash()); - maybeCreateInitManager(version_info, noop_init_manager, resume_rds); - vhds_subscription_ = std::make_unique( - config_update_info_, factory_context_, stat_prefix_, route_config_provider_opt_); - vhds_subscription_->registerInitTargetWithInitManager( - noop_init_manager == nullptr ? local_init_manager_ : *noop_init_manager); - } - - ENVOY_LOG(debug, "rds: loading new configuration: config_name={} hash={}", route_config_name_, - config_update_info_->configHash()); - - if (route_config_provider_opt_.has_value()) { - route_config_provider_opt_.value()->onConfigUpdate(); - } - // RDS update removed VHDS configuration - if (!config_update_info_->protobufConfiguration().has_vhds()) { - vhds_subscription_.release(); - } +} - update_callback_manager_.runCallbacks(); +void RdsRouteConfigSubscription::afterProviderUpdate() { + // RDS update removed VHDS configuration + if (!config_update_info_->protobufConfigurationCast().has_vhds()) { + vhds_subscription_.release(); } - local_init_target_.ready(); + update_callback_manager_.runCallbacks(); } // Initialize a no-op InitManager in case the one in the factory_context has completed @@ -178,30 +130,6 @@ void RdsRouteConfigSubscription::maybeCreateInitManager( } } -void RdsRouteConfigSubscription::onConfigUpdate( - const std::vector& added_resources, - const Protobuf::RepeatedPtrField& removed_resources, const std::string&) { - if (!removed_resources.empty()) { - // TODO(#2500) when on-demand resource loading is supported, an RDS removal may make sense - // (see discussion in #6879), and so we should do something other than ignoring here. - ENVOY_LOG( - error, - "Server sent a delta RDS update attempting to remove a resource (name: {}). Ignoring.", - removed_resources[0]); - } - if (!added_resources.empty()) { - onConfigUpdate(added_resources, added_resources[0].get().version()); - } -} - -void RdsRouteConfigSubscription::onConfigUpdateFailed( - Envoy::Config::ConfigUpdateFailureReason reason, const EnvoyException*) { - ASSERT(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure != reason); - // We need to allow server startup to continue, even if we have a bad - // config. - local_init_target_.ready(); -} - void RdsRouteConfigSubscription::updateOnDemand(const std::string& aliases) { if (vhds_subscription_.get() == nullptr) { return; @@ -209,54 +137,23 @@ void RdsRouteConfigSubscription::updateOnDemand(const std::string& aliases) { vhds_subscription_->updateOnDemand(aliases); } -bool RdsRouteConfigSubscription::validateUpdateSize(int num_resources) { - if (num_resources == 0) { - ENVOY_LOG(debug, "Missing RouteConfiguration for {} in onConfigUpdate()", route_config_name_); - stats_.update_empty_.inc(); - local_init_target_.ready(); - return false; - } - if (num_resources != 1) { - throw EnvoyException(fmt::format("Unexpected RDS resource length: {}", num_resources)); - // (would be a return false here) - } - return true; -} - RdsRouteConfigProviderImpl::RdsRouteConfigProviderImpl( RdsRouteConfigSubscriptionSharedPtr&& subscription, - Server::Configuration::ServerFactoryContext& factory_context, - const OptionalHttpFilters& optional_http_filters) - : subscription_(std::move(subscription)), - config_update_info_(subscription_->routeConfigUpdate()), factory_context_(factory_context), - validator_(factory_context.messageValidationContext().dynamicValidationVisitor()), - tls_(factory_context.threadLocal()), optional_http_filters_(optional_http_filters) { - ConfigConstSharedPtr initial_config; - if (config_update_info_->configInfo().has_value()) { - initial_config = - std::make_shared(config_update_info_->protobufConfiguration(), - optional_http_filters_, factory_context_, validator_, false); - } else { - initial_config = std::make_shared(); - } - tls_.set([initial_config](Event::Dispatcher&) { - return std::make_shared(initial_config); - }); - // It should be 1:1 mapping due to shared rds config. - ASSERT(!subscription_->routeConfigProvider().has_value()); - subscription_->routeConfigProvider().emplace(this); + Server::Configuration::ServerFactoryContext& factory_context) + : base_(subscription, factory_context), config_update_info_(subscription->routeConfigUpdate()), + factory_context_(factory_context) { + // The subscription referenced by the 'base_' and by 'this' is the same. + // In it the provider is already set by the 'base_' so it points to that. + // Need to set again to point to 'this'. + base_.subscription().routeConfigProvider().emplace(this); } -RdsRouteConfigProviderImpl::~RdsRouteConfigProviderImpl() { - ASSERT(subscription_->routeConfigProvider().has_value()); - subscription_->routeConfigProvider().reset(); +RdsRouteConfigSubscription& RdsRouteConfigProviderImpl::subscription() { + return static_cast(base_.subscription()); } -Router::ConfigConstSharedPtr RdsRouteConfigProviderImpl::config() { return tls_->config_; } - void RdsRouteConfigProviderImpl::onConfigUpdate() { - tls_.runOnAllThreads([new_config = config_update_info_->parsedConfiguration()]( - OptRef tls) { tls->config_ = new_config; }); + base_.onConfigUpdate(); const auto aliases = config_update_info_->resourceIdsInLastVhdsUpdate(); // Regular (non-VHDS) RDS updates don't populate aliases fields in resources. @@ -289,13 +186,18 @@ void RdsRouteConfigProviderImpl::onConfigUpdate() { } } +ConfigConstSharedPtr RdsRouteConfigProviderImpl::configCast() const { + ASSERT(dynamic_cast(RdsRouteConfigProviderImpl::config().get())); + return std::static_pointer_cast(RdsRouteConfigProviderImpl::config()); +} + // Schedules a VHDS request on the main thread and queues up the callback to use when the VHDS // response has been propagated to the worker thread that was the request origin. void RdsRouteConfigProviderImpl::requestVirtualHostsUpdate( const std::string& for_domain, Event::Dispatcher& thread_local_dispatcher, std::weak_ptr route_config_updated_cb) { - auto alias = - VhdsSubscription::domainNameToAlias(config_update_info_->routeConfigName(), for_domain); + auto alias = VhdsSubscription::domainNameToAlias( + config_update_info_->protobufConfigurationCast().name(), for_domain); // The RdsRouteConfigProviderImpl instance can go away before the dispatcher has a chance to // execute the callback. still_alive shared_ptr will be deallocated when the current instance of // the RdsRouteConfigProviderImpl is deallocated; we rely on a weak_ptr to still_alive flag to @@ -306,53 +208,46 @@ void RdsRouteConfigProviderImpl::requestVirtualHostsUpdate( alias, &thread_local_dispatcher, route_config_updated_cb]() -> void { if (maybe_still_alive.lock()) { - subscription_->updateOnDemand(alias); + subscription().updateOnDemand(alias); config_update_callbacks_.push_back({alias, thread_local_dispatcher, route_config_updated_cb}); } }); } -RouteConfigProviderManagerImpl::RouteConfigProviderManagerImpl(Server::Admin& admin) { - config_tracker_entry_ = - admin.getConfigTracker().add("routes", [this](const Matchers::StringMatcher& matcher) { - return dumpRouteConfigs(matcher); - }); - // ConfigTracker keys must be unique. We are asserting that no one has stolen the "routes" key - // from us, since the returned entry will be nullptr if the key already exists. - RELEASE_ASSERT(config_tracker_entry_, ""); +ProtoTraitsImpl::ProtoTraitsImpl() + : resource_type_( + Envoy::Config::getResourceName()) {} + +ProtobufTypes::MessagePtr ProtoTraitsImpl::createEmptyProto() const { + return std::make_unique(); } +RouteConfigProviderManagerImpl::RouteConfigProviderManagerImpl(Server::Admin& admin) + : manager_(admin, "routes", proto_traits_) {} + Router::RouteConfigProviderSharedPtr RouteConfigProviderManagerImpl::createRdsRouteConfigProvider( const envoy::extensions::filters::network::http_connection_manager::v3::Rds& rds, const OptionalHttpFilters& optional_http_filters, Server::Configuration::ServerFactoryContext& factory_context, const std::string& stat_prefix, Init::Manager& init_manager) { - // RdsRouteConfigSubscriptions are unique based on their serialized RDS config. - const uint64_t manager_identifier = MessageUtil::hash(rds); - auto it = dynamic_route_config_providers_.find(manager_identifier); - - if (it == dynamic_route_config_providers_.end()) { - // std::make_shared does not work for classes with private constructors. There are ways - // around it. However, since this is not a performance critical path we err on the side - // of simplicity. - RdsRouteConfigSubscriptionSharedPtr subscription(new RdsRouteConfigSubscription( - rds, manager_identifier, factory_context, stat_prefix, optional_http_filters, *this)); - init_manager.add(subscription->parent_init_target_); - RdsRouteConfigProviderImplSharedPtr new_provider{new RdsRouteConfigProviderImpl( - std::move(subscription), factory_context, optional_http_filters)}; - dynamic_route_config_providers_.insert( - {manager_identifier, std::weak_ptr(new_provider)}); - return new_provider; - } else { - // Because the RouteConfigProviderManager's weak_ptrs only get cleaned up - // in the RdsRouteConfigSubscription destructor, and the single threaded nature - // of this code, locking the weak_ptr will not fail. - auto existing_provider = it->second.lock(); - RELEASE_ASSERT(existing_provider != nullptr, - absl::StrCat("cannot find subscribed rds resource ", rds.route_config_name())); - init_manager.add(existing_provider->subscription_->parent_init_target_); - return existing_provider; - } + auto provider = manager_.addDynamicProvider( + rds, rds.route_config_name(), init_manager, + [&optional_http_filters, &factory_context, &rds, &stat_prefix, + this](uint64_t manager_identifier) { + auto config_update = std::make_unique( + proto_traits_, factory_context, optional_http_filters); + auto resource_decoder = std::make_unique< + Envoy::Config::OpaqueResourceDecoderImpl>( + factory_context.messageValidationContext().dynamicValidationVisitor(), "name"); + auto subscription = std::make_shared( + std::move(config_update), std::move(resource_decoder), rds, manager_identifier, + factory_context, stat_prefix, manager_); + auto provider = + std::make_shared(std::move(subscription), factory_context); + return std::make_pair(provider, &provider->subscription().initTarget()); + }); + ASSERT(dynamic_cast(provider.get())); + return std::static_pointer_cast(provider); } RouteConfigProviderPtr RouteConfigProviderManagerImpl::createStaticRouteConfigProvider( @@ -360,50 +255,14 @@ RouteConfigProviderPtr RouteConfigProviderManagerImpl::createStaticRouteConfigPr const OptionalHttpFilters& optional_http_filters, Server::Configuration::ServerFactoryContext& factory_context, ProtobufMessage::ValidationVisitor& validator) { - auto provider = std::make_unique( - route_config, optional_http_filters, factory_context, validator, *this); - static_route_config_providers_.insert(provider.get()); - return provider; -} - -std::unique_ptr -RouteConfigProviderManagerImpl::dumpRouteConfigs( - const Matchers::StringMatcher& name_matcher) const { - auto config_dump = std::make_unique(); - - for (const auto& element : dynamic_route_config_providers_) { - const auto& subscription = element.second.lock()->subscription_; - // Because the RouteConfigProviderManager's weak_ptrs only get cleaned up - // in the RdsRouteConfigSubscription destructor, and the single threaded nature - // of this code, locking the weak_ptr will not fail. - ASSERT(subscription); - ASSERT(subscription->route_config_provider_opt_.has_value()); - - if (subscription->routeConfigUpdate()->configInfo()) { - if (!name_matcher.match(subscription->routeConfigUpdate()->protobufConfiguration().name())) { - continue; - } - auto* dynamic_config = config_dump->mutable_dynamic_route_configs()->Add(); - dynamic_config->set_version_info(subscription->routeConfigUpdate()->configVersion()); - dynamic_config->mutable_route_config()->PackFrom( - subscription->routeConfigUpdate()->protobufConfiguration()); - TimestampUtil::systemClockToTimestamp(subscription->routeConfigUpdate()->lastUpdated(), - *dynamic_config->mutable_last_updated()); - } - } - - for (const auto& provider : static_route_config_providers_) { - ASSERT(provider->configInfo()); - if (!name_matcher.match(provider->configInfo().value().config_.name())) { - continue; - } - auto* static_config = config_dump->mutable_static_route_configs()->Add(); - static_config->mutable_route_config()->PackFrom(provider->configInfo().value().config_); - TimestampUtil::systemClockToTimestamp(provider->lastUpdated(), - *static_config->mutable_last_updated()); - } - - return config_dump; + auto provider = manager_.addStaticProvider( + [&optional_http_filters, &factory_context, &validator, &route_config, this]() { + ConfigTraitsImpl config_traits(optional_http_filters, factory_context, validator, true); + return std::make_unique(route_config, config_traits, + factory_context, manager_); + }); + ASSERT(dynamic_cast(provider.get())); + return RouteConfigProviderPtr(static_cast(provider.release())); } } // namespace Router diff --git a/source/common/router/rds_impl.h b/source/common/router/rds_impl.h index 8b99ce2f741e4..c207c69ea1402 100644 --- a/source/common/router/rds_impl.h +++ b/source/common/router/rds_impl.h @@ -31,7 +31,11 @@ #include "source/common/init/target_impl.h" #include "source/common/init/watcher_impl.h" #include "source/common/protobuf/utility.h" -#include "source/common/router/route_config_update_receiver_impl.h" +#include "source/common/rds/rds_route_config_provider_impl.h" +#include "source/common/rds/rds_route_config_subscription.h" +#include "source/common/rds/route_config_provider_manager.h" +#include "source/common/rds/route_config_update_receiver_impl.h" +#include "source/common/rds/static_route_config_provider_impl.h" #include "source/common/router/vhds.h" #include "absl/container/node_hash_map.h" @@ -60,67 +64,49 @@ class RouteConfigProviderUtil { const std::string& stat_prefix, RouteConfigProviderManager& route_config_provider_manager); }; -class RouteConfigProviderManagerImpl; - /** * Implementation of RouteConfigProvider that holds a static route configuration. */ class StaticRouteConfigProviderImpl : public RouteConfigProvider { public: StaticRouteConfigProviderImpl(const envoy::config::route::v3::RouteConfiguration& config, - const OptionalHttpFilters& http_filters, + Rds::ConfigTraits& config_traits, Server::Configuration::ServerFactoryContext& factory_context, - ProtobufMessage::ValidationVisitor& validator, - RouteConfigProviderManagerImpl& route_config_provider_manager); + Rds::RouteConfigProviderManager& route_config_provider_manager); ~StaticRouteConfigProviderImpl() override; // Router::RouteConfigProvider - Router::ConfigConstSharedPtr config() override { return config_; } - absl::optional configInfo() const override { - return ConfigInfo{route_config_proto_, ""}; - } - SystemTime lastUpdated() const override { return last_updated_; } - void onConfigUpdate() override {} + Rds::ConfigConstSharedPtr config() const override { return base_.config(); } + const absl::optional& configInfo() const override { return base_.configInfo(); } + SystemTime lastUpdated() const override { return base_.lastUpdated(); } + void onConfigUpdate() override { base_.onConfigUpdate(); } + ConfigConstSharedPtr configCast() const override; void requestVirtualHostsUpdate(const std::string&, Event::Dispatcher&, std::weak_ptr) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } private: - ConfigConstSharedPtr config_; - envoy::config::route::v3::RouteConfiguration route_config_proto_; - SystemTime last_updated_; - RouteConfigProviderManagerImpl& route_config_provider_manager_; -}; - -/** - * All RDS stats. @see stats_macros.h - */ -#define ALL_RDS_STATS(COUNTER, GAUGE) \ - COUNTER(config_reload) \ - COUNTER(update_empty) \ - GAUGE(config_reload_time_ms, NeverImport) - -/** - * Struct definition for all RDS stats. @see stats_macros.h - */ -struct RdsStats { - ALL_RDS_STATS(GENERATE_COUNTER_STRUCT, GENERATE_GAUGE_STRUCT) + Rds::StaticRouteConfigProviderImpl base_; + Rds::RouteConfigProviderManager& route_config_provider_manager_; }; -class RdsRouteConfigProviderImpl; - /** * A class that fetches the route configuration dynamically using the RDS API and updates them to * RDS config providers. */ -class RdsRouteConfigSubscription - : Envoy::Config::SubscriptionBase, - Logger::Loggable { + +class RdsRouteConfigSubscription : public Rds::RdsRouteConfigSubscription { public: + RdsRouteConfigSubscription( + RouteConfigUpdatePtr&& config_update, + std::unique_ptr&& resource_decoder, + const envoy::extensions::filters::network::http_connection_manager::v3::Rds& rds, + const uint64_t manager_identifier, + Server::Configuration::ServerFactoryContext& factory_context, const std::string& stat_prefix, + Rds::RouteConfigProviderManager& route_config_provider_manager); ~RdsRouteConfigSubscription() override; - absl::optional& routeConfigProvider() { return route_config_provider_opt_; } RouteConfigUpdatePtr& routeConfigUpdate() { return config_update_info_; } void updateOnDemand(const std::string& aliases); void maybeCreateInitManager(const std::string& version_info, @@ -128,53 +114,18 @@ class RdsRouteConfigSubscription std::unique_ptr& resume_rds); private: - // Config::SubscriptionCallbacks - void onConfigUpdate(const std::vector& resources, - const std::string& version_info) override; - void onConfigUpdate(const std::vector& added_resources, - const Protobuf::RepeatedPtrField& removed_resources, - const std::string& system_version_info) override; - void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason reason, - const EnvoyException* e) override; + void beforeProviderUpdate(std::unique_ptr& noop_init_manager, + std::unique_ptr& resume_rds) override; + void afterProviderUpdate() override; ABSL_MUST_USE_RESULT Common::CallbackHandlePtr addUpdateCallback(std::function callback) { return update_callback_manager_.add(callback); } - RdsRouteConfigSubscription( - const envoy::extensions::filters::network::http_connection_manager::v3::Rds& rds, - const uint64_t manager_identifier, - Server::Configuration::ServerFactoryContext& factory_context, const std::string& stat_prefix, - const OptionalHttpFilters& optional_http_filters, - RouteConfigProviderManagerImpl& route_config_provider_manager); - - bool validateUpdateSize(int num_resources); - - const std::string route_config_name_; - // This scope must outlive the subscription_ below as the subscription has derived stats. - Stats::ScopePtr scope_; - Envoy::Config::SubscriptionPtr subscription_; - Server::Configuration::ServerFactoryContext& factory_context_; - - // Init target used to notify the parent init manager that the subscription [and its sub resource] - // is ready. - Init::SharedTargetImpl parent_init_target_; - // Init watcher on RDS and VHDS ready event. This watcher marks parent_init_target_ ready. - Init::WatcherImpl local_init_watcher_; - // Target which starts the RDS subscription. - Init::TargetImpl local_init_target_; - Init::ManagerImpl local_init_manager_; - std::string stat_prefix_; - RdsStats stats_; - RouteConfigProviderManagerImpl& route_config_provider_manager_; - const uint64_t manager_identifier_; - absl::optional route_config_provider_opt_; VhdsSubscriptionPtr vhds_subscription_; RouteConfigUpdatePtr config_update_info_; Common::CallbackManager<> update_callback_manager_; - const OptionalHttpFilters optional_http_filters_; - friend class RouteConfigProviderManagerImpl; // Access to addUpdateCallback friend class ScopedRdsConfigSubscription; }; @@ -194,54 +145,55 @@ struct UpdateOnDemandCallback { class RdsRouteConfigProviderImpl : public RouteConfigProvider, Logger::Loggable { public: - ~RdsRouteConfigProviderImpl() override; + RdsRouteConfigProviderImpl(RdsRouteConfigSubscriptionSharedPtr&& subscription, + Server::Configuration::ServerFactoryContext& factory_context); - RdsRouteConfigSubscription& subscription() { return *subscription_; } + RdsRouteConfigSubscription& subscription(); // Router::RouteConfigProvider - Router::ConfigConstSharedPtr config() override; - absl::optional configInfo() const override { - return config_update_info_->configInfo(); - } - SystemTime lastUpdated() const override { return config_update_info_->lastUpdated(); } + Rds::ConfigConstSharedPtr config() const override { return base_.config(); } + const absl::optional& configInfo() const override { return base_.configInfo(); } + SystemTime lastUpdated() const override { return base_.lastUpdated(); } + void onConfigUpdate() override; + ConfigConstSharedPtr configCast() const override; void requestVirtualHostsUpdate( const std::string& for_domain, Event::Dispatcher& thread_local_dispatcher, std::weak_ptr route_config_updated_cb) override; private: - struct ThreadLocalConfig : public ThreadLocal::ThreadLocalObject { - ThreadLocalConfig(ConfigConstSharedPtr initial_config) : config_(std::move(initial_config)) {} - ConfigConstSharedPtr config_; - }; + Rds::RdsRouteConfigProviderImpl base_; - RdsRouteConfigProviderImpl(RdsRouteConfigSubscriptionSharedPtr&& subscription, - Server::Configuration::ServerFactoryContext& factory_context, - const OptionalHttpFilters& optional_http_filters); - - RdsRouteConfigSubscriptionSharedPtr subscription_; RouteConfigUpdatePtr& config_update_info_; Server::Configuration::ServerFactoryContext& factory_context_; - ProtobufMessage::ValidationVisitor& validator_; - ThreadLocal::TypedSlot tls_; std::list config_update_callbacks_; // A flag used to determine if this instance of RdsRouteConfigProviderImpl hasn't been // deallocated. Please also see a comment in requestVirtualHostsUpdate() method implementation. std::shared_ptr still_alive_{std::make_shared(true)}; - const OptionalHttpFilters optional_http_filters_; - - friend class RouteConfigProviderManagerImpl; }; using RdsRouteConfigProviderImplSharedPtr = std::shared_ptr; +class ProtoTraitsImpl : public Rds::ProtoTraits { +public: + ProtoTraitsImpl(); + const std::string& resourceType() const override { return resource_type_; } + int resourceNameFieldNumber() const override { return 1; } + ProtobufTypes::MessagePtr createEmptyProto() const override; + +private: + const std::string resource_type_; +}; + class RouteConfigProviderManagerImpl : public RouteConfigProviderManager, public Singleton::Instance { public: RouteConfigProviderManagerImpl(Server::Admin& admin); std::unique_ptr - dumpRouteConfigs(const Matchers::StringMatcher& name_matcher) const; + dumpRouteConfigs(const Matchers::StringMatcher& name_matcher) const { + return manager_.dumpRouteConfigs(name_matcher); + } // RouteConfigProviderManager RouteConfigProviderSharedPtr createRdsRouteConfigProvider( @@ -257,16 +209,8 @@ class RouteConfigProviderManagerImpl : public RouteConfigProviderManager, ProtobufMessage::ValidationVisitor& validator) override; private: - // TODO(jsedgwick) These two members are prime candidates for the owned-entry list/map - // as in ConfigTracker. I.e. the ProviderImpls would have an EntryOwner for these lists - // Then the lifetime management stuff is centralized and opaque. - absl::node_hash_map> - dynamic_route_config_providers_; - absl::node_hash_set static_route_config_providers_; - Server::ConfigTracker::EntryOwnerPtr config_tracker_entry_; - - friend class RdsRouteConfigSubscription; - friend class StaticRouteConfigProviderImpl; + ProtoTraitsImpl proto_traits_; + Rds::RouteConfigProviderManager manager_; }; using RouteConfigProviderManagerImplPtr = std::unique_ptr; diff --git a/source/common/router/route_config_update_receiver_impl.cc b/source/common/router/route_config_update_receiver_impl.cc index baec90c90bbc3..488fa5f663c90 100644 --- a/source/common/router/route_config_update_receiver_impl.cc +++ b/source/common/router/route_config_update_receiver_impl.cc @@ -9,6 +9,7 @@ #include "source/common/common/assert.h" #include "source/common/common/fmt.h" #include "source/common/common/thread.h" +#include "source/common/config/resource_name.h" #include "source/common/protobuf/utility.h" #include "source/common/router/config_impl.h" @@ -34,32 +35,39 @@ void rebuildRouteConfigVirtualHosts( } // namespace -bool RouteConfigUpdateReceiverImpl::onRdsUpdate( - const envoy::config::route::v3::RouteConfiguration& rc, const std::string& version_info) { - const uint64_t new_hash = MessageUtil::hash(rc); - if (new_hash == last_config_hash_) { +Rds::ConfigConstSharedPtr ConfigTraitsImpl::createNullConfig() const { + return std::make_shared(); +} + +Rds::ConfigConstSharedPtr ConfigTraitsImpl::createConfig(const Protobuf::Message& rc) const { + ASSERT(dynamic_cast(&rc)); + return std::make_shared( + static_cast(rc), optional_http_filters_, + factory_context_, validator_, validate_clusters_default_); +} + +bool RouteConfigUpdateReceiverImpl::onRdsUpdate(const Protobuf::Message& rc, + const std::string& version_info) { + uint64_t new_hash = base_.getHash(rc); + if (!base_.checkHash(new_hash)) { return false; } - const uint64_t new_vhds_config_hash = rc.has_vhds() ? MessageUtil::hash(rc.vhds()) : 0ul; + auto new_route_config = std::make_unique(); + new_route_config->CopyFrom(rc); + const uint64_t new_vhds_config_hash = + new_route_config->has_vhds() ? MessageUtil::hash(new_route_config->vhds()) : 0ul; std::map rds_virtual_hosts; - for (const auto& vhost : rc.virtual_hosts()) { + for (const auto& vhost : new_route_config->virtual_hosts()) { rds_virtual_hosts.emplace(vhost.name(), vhost); } - envoy::config::route::v3::RouteConfiguration new_route_config = rc; - rebuildRouteConfigVirtualHosts(rds_virtual_hosts, *vhds_virtual_hosts_, new_route_config); - auto new_config = std::make_shared( - new_route_config, optional_http_filters_, factory_context_, - factory_context_.messageValidationContext().dynamicValidationVisitor(), false); - // If the above validation/validation doesn't raise exception, update the - // other cached config entries. - config_ = new_config; + rebuildRouteConfigVirtualHosts(rds_virtual_hosts, *vhds_virtual_hosts_, *new_route_config); + base_.updateConfig(std::move(new_route_config)); + base_.updateHash(new_hash); rds_virtual_hosts_ = std::move(rds_virtual_hosts); - last_config_hash_ = new_hash; - *route_config_proto_ = std::move(new_route_config); vhds_configuration_changed_ = new_vhds_config_hash != last_vhds_config_hash_; last_vhds_config_hash_ = new_vhds_config_hash; - onUpdateCommon(version_info); + base_.onUpdateCommon(version_info); return true; } @@ -76,30 +84,19 @@ bool RouteConfigUpdateReceiverImpl::onVhdsUpdate( auto route_config_after_this_update = std::make_unique(); - route_config_after_this_update->CopyFrom(*route_config_proto_); + route_config_after_this_update->CopyFrom(base_.protobufConfiguration()); rebuildRouteConfigVirtualHosts(rds_virtual_hosts_, *vhosts_after_this_update, *route_config_after_this_update); - auto new_config = std::make_shared( - *route_config_after_this_update, optional_http_filters_, factory_context_, - factory_context_.messageValidationContext().dynamicValidationVisitor(), false); - + base_.updateConfig(std::move(route_config_after_this_update)); // No exception, route_config_after_this_update is valid, can update the state. vhds_virtual_hosts_ = std::move(vhosts_after_this_update); - route_config_proto_ = std::move(route_config_after_this_update); - config_ = new_config; resource_ids_in_last_update_ = added_resource_ids; - onUpdateCommon(version_info); + base_.onUpdateCommon(version_info); return removed || updated || !resource_ids_in_last_update_.empty(); } -void RouteConfigUpdateReceiverImpl::onUpdateCommon(const std::string& version_info) { - last_config_version_ = version_info; - last_updated_ = time_source_.systemTime(); - config_info_.emplace(RouteConfigProvider::ConfigInfo{*route_config_proto_, last_config_version_}); -} - bool RouteConfigUpdateReceiverImpl::removeVhosts( std::map& vhosts, const Protobuf::RepeatedPtrField& removed_vhost_names) { diff --git a/source/common/router/route_config_update_receiver_impl.h b/source/common/router/route_config_update_receiver_impl.h index 721e46095dfc0..fa9501b29a0ce 100644 --- a/source/common/router/route_config_update_receiver_impl.h +++ b/source/common/router/route_config_update_receiver_impl.h @@ -4,6 +4,7 @@ #include "envoy/config/route/v3/route.pb.h" #include "envoy/config/route/v3/route_components.pb.h" +#include "envoy/rds/config_traits.h" #include "envoy/router/rds.h" #include "envoy/router/route_config_update_receiver.h" #include "envoy/server/factory_context.h" @@ -11,67 +12,87 @@ #include "source/common/common/logger.h" #include "source/common/protobuf/utility.h" +#include "source/common/rds/route_config_update_receiver_impl.h" #include "source/common/router/config_impl.h" namespace Envoy { namespace Router { +class ConfigTraitsImpl : public Rds::ConfigTraits { +public: + ConfigTraitsImpl(const OptionalHttpFilters& optional_http_filters, + Server::Configuration::ServerFactoryContext& factory_context, + ProtobufMessage::ValidationVisitor& validator, bool validate_clusters_default) + : optional_http_filters_(optional_http_filters), factory_context_(factory_context), + validator_(validator), validate_clusters_default_(validate_clusters_default) {} + + Rds::ConfigConstSharedPtr createNullConfig() const override; + Rds::ConfigConstSharedPtr createConfig(const Protobuf::Message& rc) const override; + +private: + const OptionalHttpFilters optional_http_filters_; + Server::Configuration::ServerFactoryContext& factory_context_; + ProtobufMessage::ValidationVisitor& validator_; + bool validate_clusters_default_; +}; + class RouteConfigUpdateReceiverImpl : public RouteConfigUpdateReceiver { public: - RouteConfigUpdateReceiverImpl(Server::Configuration::ServerFactoryContext& factory_context, + RouteConfigUpdateReceiverImpl(Rds::ProtoTraits& proto_traits, + Server::Configuration::ServerFactoryContext& factory_context, const OptionalHttpFilters& optional_http_filters) - : factory_context_(factory_context), time_source_(factory_context.timeSource()), - route_config_proto_(std::make_unique()), - last_config_hash_(0ull), last_vhds_config_hash_(0ul), + : config_traits_(optional_http_filters, factory_context, + factory_context.messageValidationContext().dynamicValidationVisitor(), + false), + base_(config_traits_, proto_traits, factory_context), last_vhds_config_hash_(0ul), vhds_virtual_hosts_( std::make_unique>()), - vhds_configuration_changed_(true), optional_http_filters_(optional_http_filters) {} + vhds_configuration_changed_(true) {} bool removeVhosts(std::map& vhosts, const Protobuf::RepeatedPtrField& removed_vhost_names); bool updateVhosts(std::map& vhosts, const VirtualHostRefVector& added_vhosts); bool onDemandFetchFailed(const envoy::service::discovery::v3::Resource& resource) const; - void onUpdateCommon(const std::string& version_info); // Router::RouteConfigUpdateReceiver - bool onRdsUpdate(const envoy::config::route::v3::RouteConfiguration& rc, - const std::string& version_info) override; + bool onRdsUpdate(const Protobuf::Message& rc, const std::string& version_info) override; bool onVhdsUpdate(const VirtualHostRefVector& added_vhosts, const std::set& added_resource_ids, const Protobuf::RepeatedPtrField& removed_resources, const std::string& version_info) override; - const std::string& routeConfigName() const override { return route_config_proto_->name(); } - const std::string& configVersion() const override { return last_config_version_; } - uint64_t configHash() const override { return last_config_hash_; } - absl::optional configInfo() const override { - return config_info_; + uint64_t configHash() const override { return base_.configHash(); } + const absl::optional& configInfo() const override { + return base_.configInfo(); } bool vhdsConfigurationChanged() const override { return vhds_configuration_changed_; } - const envoy::config::route::v3::RouteConfiguration& protobufConfiguration() override { - return static_cast(*route_config_proto_); + const Protobuf::Message& protobufConfiguration() const override { + return base_.protobufConfiguration(); + } + Rds::ConfigConstSharedPtr parsedConfiguration() const override { + return base_.parsedConfiguration(); } - ConfigConstSharedPtr parsedConfiguration() const override { return config_; } - SystemTime lastUpdated() const override { return last_updated_; } + SystemTime lastUpdated() const override { return base_.lastUpdated(); } const std::set& resourceIdsInLastVhdsUpdate() override { return resource_ids_in_last_update_; } + const envoy::config::route::v3::RouteConfiguration& protobufConfigurationCast() const override { + ASSERT(dynamic_cast( + &RouteConfigUpdateReceiverImpl::protobufConfiguration())); + return static_cast( + RouteConfigUpdateReceiverImpl::protobufConfiguration()); + } private: - Server::Configuration::ServerFactoryContext& factory_context_; - TimeSource& time_source_; - std::unique_ptr route_config_proto_; - uint64_t last_config_hash_; + ConfigTraitsImpl config_traits_; + + Rds::RouteConfigUpdateReceiverImpl base_; + uint64_t last_vhds_config_hash_; - std::string last_config_version_; - SystemTime last_updated_; std::map rds_virtual_hosts_; std::unique_ptr> vhds_virtual_hosts_; - absl::optional config_info_; std::set resource_ids_in_last_update_; bool vhds_configuration_changed_; - ConfigConstSharedPtr config_; - const OptionalHttpFilters& optional_http_filters_; }; } // namespace Router diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index de3e5832130ac..c18e6ee3c258c 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -103,7 +103,7 @@ makeScopedRouteInfos(ProtobufTypes::ConstMessagePtrVector&& config_protos, scoped_route_config.route_configuration(), optional_http_filters, factory_context, factory_context.messageValidationContext().staticValidationVisitor()); scopes.push_back(std::make_shared(scoped_route_config, - route_config_provider->config())); + route_config_provider->configCast())); } return scopes; @@ -230,7 +230,7 @@ void ScopedRdsConfigSubscription::RdsRouteConfigProviderHelper::initRdsConfigPro rds_update_callback_handle_ = route_provider_->subscription().addUpdateCallback([this]() { // Subscribe to RDS update. - parent_.onRdsConfigUpdate(scope_name_, route_provider_->config()); + parent_.onRdsConfigUpdate(scope_name_, route_provider_->configCast()); }); parent_.stats_.active_scopes_.inc(); } @@ -263,7 +263,7 @@ void ScopedRdsConfigSubscription::RdsRouteConfigProviderHelper::maybeInitRdsConf return; } // If RouteConfiguration has been initialized, apply update to all the threads. - parent_.onRdsConfigUpdate(scope_name_, route_provider_->config()); + parent_.onRdsConfigUpdate(scope_name_, route_provider_->configCast()); } bool ScopedRdsConfigSubscription::addOrUpdateScopes( diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index eb53c8b57fe89..2b542ff403ffe 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -153,7 +153,7 @@ class ScopedRdsConfigSubscription parent_.stats_.on_demand_scopes_.dec(); } } - ConfigConstSharedPtr routeConfig() { return route_provider_->config(); } + ConfigConstSharedPtr routeConfig() { return route_provider_->configCast(); } void addOnDemandUpdateCallback(std::function callback); diff --git a/source/common/router/vhds.cc b/source/common/router/vhds.cc index 324d952c42b6d..c76efc08d9e4a 100644 --- a/source/common/router/vhds.cc +++ b/source/common/router/vhds.cc @@ -20,20 +20,24 @@ namespace Envoy { namespace Router { // Implements callbacks to handle DeltaDiscovery protocol for VirtualHostDiscoveryService -VhdsSubscription::VhdsSubscription(RouteConfigUpdatePtr& config_update_info, - Server::Configuration::ServerFactoryContext& factory_context, - const std::string& stat_prefix, - absl::optional& route_config_provider_opt) +VhdsSubscription::VhdsSubscription( + RouteConfigUpdatePtr& config_update_info, + Server::Configuration::ServerFactoryContext& factory_context, const std::string& stat_prefix, + absl::optional& route_config_provider_opt) : Envoy::Config::SubscriptionBase( factory_context.messageValidationContext().dynamicValidationVisitor(), "name"), config_update_info_(config_update_info), - scope_(factory_context.scope().createScope(stat_prefix + "vhds." + - config_update_info_->routeConfigName() + ".")), + scope_(factory_context.scope().createScope( + stat_prefix + "vhds." + config_update_info_->protobufConfigurationCast().name() + ".")), stats_({ALL_VHDS_STATS(POOL_COUNTER(*scope_))}), - init_target_(fmt::format("VhdsConfigSubscription {}", config_update_info_->routeConfigName()), - [this]() { subscription_->start({config_update_info_->routeConfigName()}); }), + init_target_(fmt::format("VhdsConfigSubscription {}", + config_update_info_->protobufConfigurationCast().name()), + [this]() { + subscription_->start( + {config_update_info_->protobufConfigurationCast().name()}); + }), route_config_provider_opt_(route_config_provider_opt) { - const auto& config_source = config_update_info_->protobufConfiguration() + const auto& config_source = config_update_info_->protobufConfigurationCast() .vhds() .config_source() .api_config_source() @@ -46,7 +50,7 @@ VhdsSubscription::VhdsSubscription(RouteConfigUpdatePtr& config_update_info, options.use_namespace_matching_ = true; subscription_ = factory_context.clusterManager().subscriptionFactory().subscriptionFromConfigSource( - config_update_info_->protobufConfiguration().vhds().config_source(), + config_update_info_->protobufConfigurationCast().vhds().config_source(), Grpc::Common::typeUrl(resource_name), *scope_, *this, resource_decoder_, options); } @@ -84,7 +88,8 @@ void VhdsSubscription::onConfigUpdate( version_info)) { stats_.config_reload_.inc(); ENVOY_LOG(debug, "vhds: loading new configuration: config_name={} hash={}", - config_update_info_->routeConfigName(), config_update_info_->configHash()); + config_update_info_->protobufConfigurationCast().name(), + config_update_info_->configHash()); if (route_config_provider_opt_.has_value()) { route_config_provider_opt_.value()->onConfigUpdate(); } diff --git a/source/common/router/vhds.h b/source/common/router/vhds.h index e1d36da175afc..3a1e43bf924ed 100644 --- a/source/common/router/vhds.h +++ b/source/common/router/vhds.h @@ -42,7 +42,8 @@ class VhdsSubscription : Envoy::Config::SubscriptionBase& route_config_providers); + absl::optional& route_config_providers); + ~VhdsSubscription() override { init_target_.ready(); } void registerInitTargetWithInitManager(Init::Manager& m) { m.add(init_target_); } @@ -72,7 +73,7 @@ class VhdsSubscription : Envoy::Config::SubscriptionBase& route_config_provider_opt_; + absl::optional& route_config_provider_opt_; }; using VhdsSubscriptionPtr = std::unique_ptr; diff --git a/source/server/admin/admin.h b/source/server/admin/admin.h index 15e06294dc9d2..682304eadf9c7 100644 --- a/source/server/admin/admin.h +++ b/source/server/admin/admin.h @@ -226,16 +226,18 @@ class AdminImpl : public Admin, NullRouteConfigProvider(TimeSource& time_source); // Router::RouteConfigProvider - Router::ConfigConstSharedPtr config() override { return config_; } - absl::optional configInfo() const override { return {}; } + Rds::ConfigConstSharedPtr config() const override { return config_; } + const absl::optional& configInfo() const override { return config_info_; } SystemTime lastUpdated() const override { return time_source_.systemTime(); } void onConfigUpdate() override {} + Router::ConfigConstSharedPtr configCast() const override { return config_; } void requestVirtualHostsUpdate(const std::string&, Event::Dispatcher&, std::weak_ptr) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } Router::ConfigConstSharedPtr config_; + absl::optional config_info_; TimeSource& time_source_; }; diff --git a/test/common/http/conn_manager_impl_fuzz_test.cc b/test/common/http/conn_manager_impl_fuzz_test.cc index 4177b065e607c..70d54e6fff521 100644 --- a/test/common/http/conn_manager_impl_fuzz_test.cc +++ b/test/common/http/conn_manager_impl_fuzz_test.cc @@ -49,19 +49,6 @@ namespace Http { class FuzzConfig : public ConnectionManagerConfig { public: - struct RouteConfigProvider : public Router::RouteConfigProvider { - RouteConfigProvider(TimeSource& time_source) : time_source_(time_source) {} - - // Router::RouteConfigProvider - Router::ConfigConstSharedPtr config() override { return route_config_; } - absl::optional configInfo() const override { return {}; } - SystemTime lastUpdated() const override { return time_source_.systemTime(); } - void onConfigUpdate() override {} - - TimeSource& time_source_; - std::shared_ptr route_config_{new NiceMock()}; - }; - FuzzConfig(envoy::extensions::filters::network::http_connection_manager::v3:: HttpConnectionManager::ForwardClientCertDetails forward_client_cert) : stats_({ALL_HTTP_CONN_MAN_STATS(POOL_COUNTER(fake_stats_), POOL_GAUGE(fake_stats_), diff --git a/test/common/http/conn_manager_impl_test_base.h b/test/common/http/conn_manager_impl_test_base.h index 6225b61d854d2..3c50a650e1169 100644 --- a/test/common/http/conn_manager_impl_test_base.h +++ b/test/common/http/conn_manager_impl_test_base.h @@ -28,19 +28,6 @@ namespace Http { class HttpConnectionManagerImplTest : public testing::Test, public ConnectionManagerConfig { public: - struct RouteConfigProvider : public Router::RouteConfigProvider { - RouteConfigProvider(TimeSource& time_source) : time_source_(time_source) {} - - // Router::RouteConfigProvider - Router::ConfigConstSharedPtr config() override { return route_config_; } - absl::optional configInfo() const override { return {}; } - SystemTime lastUpdated() const override { return time_source_.systemTime(); } - void onConfigUpdate() override {} - - TimeSource& time_source_; - std::shared_ptr route_config_{new NiceMock()}; - }; - HttpConnectionManagerImplTest(); ~HttpConnectionManagerImplTest() override; Tracing::CustomTagConstSharedPtr requestHeaderCustomTag(const std::string& header); diff --git a/test/common/rds/BUILD b/test/common/rds/BUILD new file mode 100644 index 0000000000000..e12ec953362ac --- /dev/null +++ b/test/common/rds/BUILD @@ -0,0 +1,19 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_test", + "envoy_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_cc_test( + name = "rds_test", + srcs = ["rds_test.cc"], + deps = [ + "//source/common/rds:rds_lib", + "//test/mocks/server:instance_mocks", + "@envoy_api//envoy/config/route/v3:pkg_cc_proto", + ], +) diff --git a/test/common/rds/rds_test.cc b/test/common/rds/rds_test.cc new file mode 100644 index 0000000000000..7e7fe0bad6a69 --- /dev/null +++ b/test/common/rds/rds_test.cc @@ -0,0 +1,269 @@ +#include +#include +#include + +#include "envoy/config/route/v3/route.pb.h" +#include "envoy/config/route/v3/route.pb.validate.h" +#include "envoy/stats/scope.h" + +#include "source/common/config/opaque_resource_decoder_impl.h" +#include "source/common/rds/rds_route_config_provider_impl.h" +#include "source/common/rds/rds_route_config_subscription.h" +#include "source/common/rds/route_config_provider_manager.h" +#include "source/common/rds/route_config_update_receiver_impl.h" +#include "source/common/rds/static_route_config_provider_impl.h" + +#include "test/mocks/protobuf/mocks.h" +#include "test/mocks/server/instance.h" +#include "test/test_common/simulated_time_system.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::ReturnRef; + +namespace Envoy { +namespace Rds { +namespace { + +class RdsTestBase : public testing::Test { +public: + RdsTestBase() { + ON_CALL(server_factory_context_, scope()).WillByDefault(ReturnRef(scope_)); + ON_CALL(server_factory_context_, messageValidationContext()) + .WillByDefault(ReturnRef(validation_context_)); + } + + ~RdsTestBase() override { server_factory_context_.thread_local_.shutdownThread(); } + + Event::SimulatedTimeSystem& timeSystem() { return time_system_; } + + Event::SimulatedTimeSystem time_system_; + NiceMock validation_context_; + NiceMock server_factory_context_; + NiceMock scope_; +}; + +class TestConfig : public Config { +public: + TestConfig() = default; + TestConfig(const envoy::config::route::v3::RouteConfiguration& rc) : rc_(rc) {} + const std::string* route(const std::string& name) const { + for (const auto& virtual_host_config : rc_.virtual_hosts()) { + if (virtual_host_config.name() == name) { + return &virtual_host_config.name(); + } + } + return nullptr; + } + +private: + envoy::config::route::v3::RouteConfiguration rc_; +}; + +class TestTraits : public ConfigTraits, public ProtoTraits { +public: + TestTraits() { + resource_type_ = Envoy::Config::getResourceName(); + } + + const std::string& resourceType() const override { return resource_type_; } + int resourceNameFieldNumber() const override { return resource_name_field_index_; } + ConfigConstSharedPtr createNullConfig() const override { + return std::make_shared(); + } + ProtobufTypes::MessagePtr createEmptyProto() const override { + return std::make_unique(); + } + ConfigConstSharedPtr createConfig(const Protobuf::Message& rc) const override { + return std::make_shared( + dynamic_cast(rc)); + } + + std::string resource_type_; + int resource_name_field_index_ = 1; +}; + +class RdsConfigUpdateReceiverTest : public RdsTestBase { +public: + void setup() { + config_update_ = + std::make_unique(traits_, traits_, server_factory_context_); + } + + const std::string* route(const std::string& path) { + return std::static_pointer_cast(config_update_->parsedConfiguration()) + ->route(path); + } + + TestTraits traits_; + RouteConfigUpdatePtr config_update_; +}; + +TEST_F(RdsConfigUpdateReceiverTest, OnRdsUpdate) { + setup(); + + EXPECT_TRUE(config_update_->parsedConfiguration()); + EXPECT_EQ(nullptr, route("foo")); + EXPECT_FALSE(config_update_->configInfo().has_value()); + + const std::string response1_json = R"EOF( +{ + "name": "foo_route_config", + "virtual_hosts": null +} +)EOF"; + auto response1 = + TestUtility::parseYaml(response1_json); + + SystemTime time1(std::chrono::milliseconds(1234567891234)); + timeSystem().setSystemTime(time1); + + EXPECT_TRUE(config_update_->onRdsUpdate(response1, "1")); + EXPECT_EQ(nullptr, route("foo")); + EXPECT_TRUE(config_update_->configInfo().has_value()); + EXPECT_EQ("1", config_update_->configInfo().value().version_); + EXPECT_EQ(time1, config_update_->lastUpdated()); + + EXPECT_FALSE(config_update_->onRdsUpdate(response1, "2")); + EXPECT_EQ(nullptr, route("foo")); + EXPECT_EQ("1", config_update_->configInfo().value().version_); + + std::shared_ptr config = config_update_->parsedConfiguration(); + EXPECT_EQ(2, config.use_count()); + + const std::string response2_json = R"EOF( +{ + "name": "foo_route_config", + "virtual_hosts": [ + { + "name": "foo", + "domains": [ + "*" + ], + } + ] +} + )EOF"; + + auto response2 = + TestUtility::parseYaml(response2_json); + + SystemTime time2(std::chrono::milliseconds(1234567891235)); + timeSystem().setSystemTime(time2); + + EXPECT_TRUE(config_update_->onRdsUpdate(response2, "2")); + EXPECT_EQ("foo", *route("foo")); + EXPECT_TRUE(config_update_->configInfo().has_value()); + EXPECT_EQ("2", config_update_->configInfo().value().version_); + EXPECT_EQ(time2, config_update_->lastUpdated()); + + EXPECT_EQ(1, config.use_count()); +} + +class RdsConfigProviderManagerTest : public RdsTestBase { +public: + RdsConfigProviderManagerTest() : manager_(server_factory_context_.admin_, "test", traits_) {} + + RouteConfigProviderSharedPtr createDynamic() { + envoy::config::core::v3::ConfigSource config_source; + config_source.set_path("test_path"); + return manager_.addDynamicProvider( + config_source, "test_route", outer_init_manager_, + [&config_source, this](uint64_t manager_identifier) { + auto config_update = std::make_unique( + traits_, traits_, server_factory_context_); + auto resource_decoder = std::make_unique>( + server_factory_context_.messageValidationContext().dynamicValidationVisitor(), + "name"); + auto subscription = std::make_shared( + std::move(config_update), std::move(resource_decoder), config_source, + route_config_name_, manager_identifier, server_factory_context_, "test_stat", + rds_type_, manager_); + auto provider = std::make_shared(std::move(subscription), + server_factory_context_); + return std::make_pair(provider, &provider->subscription().initTarget()); + }); + } + + RouteConfigProviderSharedPtr createStatic() { + return manager_.addStaticProvider([this]() { + envoy::config::route::v3::RouteConfiguration route_config; + return std::make_unique(route_config, traits_, + server_factory_context_, manager_); + }); + } + + void setConfigToDynamicProvider() { + const std::string response_json = R"EOF( +{ + "version_info": "1", + "resources": [ + { + "@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration", + "name": "test_route", + "virtual_hosts": null + } + ] +} +)EOF"; + auto response = + TestUtility::parseYaml(response_json); + const auto decoded_resources = + TestUtility::decodeResources(response); + server_factory_context_.cluster_manager_.subscription_factory_.callbacks_->onConfigUpdate( + decoded_resources.refvec_, response.version_info()); + } + + NiceMock outer_init_manager_; + TestTraits traits_; + RouteConfigProviderManager manager_; + const std::string rds_type_ = "TestRDS"; + const std::string route_config_name_ = "test_route"; +}; + +TEST_F(RdsConfigProviderManagerTest, ProviderErase) { + Matchers::UniversalStringMatcher universal_name_matcher; + + auto dump = manager_.dumpRouteConfigs(universal_name_matcher); + EXPECT_EQ(0, dump->dynamic_route_configs().size()); + EXPECT_EQ(0, dump->static_route_configs().size()); + + RouteConfigProviderSharedPtr static_provider = createStatic(); + RouteConfigProviderSharedPtr dynamic_provider = createDynamic(); + setConfigToDynamicProvider(); + + dump = manager_.dumpRouteConfigs(universal_name_matcher); + EXPECT_EQ(1, dump->dynamic_route_configs().size()); + EXPECT_EQ(1, dump->static_route_configs().size()); + + static_provider.reset(); + dump = manager_.dumpRouteConfigs(universal_name_matcher); + EXPECT_EQ(1, dump->dynamic_route_configs().size()); + EXPECT_EQ(0, dump->static_route_configs().size()); + + dynamic_provider.reset(); + dump = manager_.dumpRouteConfigs(universal_name_matcher); + EXPECT_EQ(0, dump->dynamic_route_configs().size()); + EXPECT_EQ(0, dump->static_route_configs().size()); +} + +TEST_F(RdsConfigProviderManagerTest, FailureInvalidResourceType) { + RouteConfigProviderSharedPtr dynamic_provider = createDynamic(); + + traits_.resource_name_field_index_ = 0; + EXPECT_THROW_WITH_MESSAGE(setConfigToDynamicProvider(), EnvoyException, + "Unexpected " + rds_type_ + " configuration (expecting " + + route_config_name_ + "): "); + + traits_.resource_type_ = "EXPECTED_resource_type"; + EXPECT_THROW_WITH_MESSAGE(setConfigToDynamicProvider(), EnvoyException, + "Unexpected " + rds_type_ + " configuration type (expecting " + + traits_.resource_type_ + + "): envoy.config.route.v3.RouteConfiguration"); +} + +} // namespace +} // namespace Rds +} // namespace Envoy diff --git a/test/common/router/rds_impl_test.cc b/test/common/router/rds_impl_test.cc index 62eb723c12685..9d935dd632268 100644 --- a/test/common/router/rds_impl_test.cc +++ b/test/common/router/rds_impl_test.cc @@ -122,7 +122,7 @@ stat_prefix: foo RouteConstSharedPtr route(Http::TestRequestHeaderMapImpl headers) { NiceMock stream_info; headers.addCopy("x-forwarded-proto", "http"); - return rds_->config()->route(headers, stream_info, 0); + return rds_->configCast()->route(headers, stream_info, 0); } NiceMock server_; @@ -246,7 +246,7 @@ TEST_F(RdsImplTest, Basic) { // Load the config and verified shared count. // ConfigConstSharedPtr is shared between: RouteConfigUpdateReceiverImpl, rds_ (via tls_), and // config local var below. - ConfigConstSharedPtr config = rds_->config(); + ConfigConstSharedPtr config = rds_->configCast(); EXPECT_EQ(3, config.use_count()); // Third request. @@ -644,7 +644,7 @@ TEST_F(RdsImplTest, VHDSandRDSupdateTogether) { EXPECT_CALL(init_watcher_, ready()); rds_callbacks_->onConfigUpdate(decoded_resources.refvec_, response1.version_info()); - EXPECT_TRUE(rds_->config()->usesVhds()); + EXPECT_TRUE(rds_->configCast()->usesVhds()); EXPECT_EQ("foo", route(Http::TestRequestHeaderMapImpl{{":authority", "foo"}, {":path", "/foo"}}) ->routeEntry() @@ -700,6 +700,14 @@ TEST_F(RdsImplTest, VirtualHostUpdateWhenProviderHasBeenDeallocated) { EXPECT_NO_THROW(post_cb()); } +TEST_F(RdsImplTest, RdsRouteConfigProviderImplSubscriptionSetup) { + setup(); + EXPECT_CALL(init_watcher_, ready()); + RdsRouteConfigSubscription& subscription = + dynamic_cast(*rds_).subscription(); + EXPECT_EQ(rds_.get(), subscription.routeConfigProvider().value()); +} + class RdsRouteConfigSubscriptionTest : public RdsTestBase { public: RdsRouteConfigSubscriptionTest() { diff --git a/test/common/router/vhds_test.cc b/test/common/router/vhds_test.cc index 28e933675b189..69d28aaa0d9d0 100644 --- a/test/common/router/vhds_test.cc +++ b/test/common/router/vhds_test.cc @@ -10,6 +10,7 @@ #include "source/common/config/utility.h" #include "source/common/protobuf/protobuf.h" #include "source/common/router/rds_impl.h" +#include "source/common/router/route_config_update_receiver_impl.h" #include "source/server/admin/admin.h" #include "test/mocks/config/mocks.h" @@ -74,17 +75,18 @@ name: my_route } RouteConfigUpdatePtr makeRouteConfigUpdate(const envoy::config::route::v3::RouteConfiguration& rc) { - RouteConfigUpdatePtr config_update_info = - std::make_unique(factory_context_, OptionalHttpFilters()); + RouteConfigUpdatePtr config_update_info = std::make_unique( + proto_traits_, factory_context_, OptionalHttpFilters()); config_update_info->onRdsUpdate(rc, "1"); return config_update_info; } + ProtoTraitsImpl proto_traits_; NiceMock factory_context_; Init::ExpectableWatcherImpl init_watcher_; Init::TargetHandlePtr init_target_handle_; const std::string context_ = "vhds_test"; - absl::optional provider_; + absl::optional provider_; Protobuf::util::MessageDifferencer messageDifferencer_; std::string default_vhds_config_; NiceMock subscription_factory_; @@ -125,7 +127,7 @@ TEST_F(VhdsTest, VhdsAddsVirtualHosts) { RouteConfigUpdatePtr config_update_info = makeRouteConfigUpdate(route_config); VhdsSubscription subscription(config_update_info, factory_context_, context_, provider_); - EXPECT_EQ(0UL, config_update_info->protobufConfiguration().virtual_hosts_size()); + EXPECT_EQ(0UL, config_update_info->protobufConfigurationCast().virtual_hosts_size()); auto vhost = buildVirtualHost("vhost1", "vhost.first"); const auto& added_resources = buildAddedResources({vhost}); @@ -135,9 +137,9 @@ TEST_F(VhdsTest, VhdsAddsVirtualHosts) { factory_context_.cluster_manager_.subscription_factory_.callbacks_->onConfigUpdate( decoded_resources.refvec_, removed_resources, "1"); - EXPECT_EQ(1UL, config_update_info->protobufConfiguration().virtual_hosts_size()); + EXPECT_EQ(1UL, config_update_info->protobufConfigurationCast().virtual_hosts_size()); EXPECT_TRUE(messageDifferencer_.Equals( - vhost, config_update_info->protobufConfiguration().virtual_hosts(0))); + vhost, config_update_info->protobufConfigurationCast().virtual_hosts(0))); } // verify that an RDS update of virtual hosts leaves VHDS virtual hosts intact @@ -184,8 +186,8 @@ name: my_route RouteConfigUpdatePtr config_update_info = makeRouteConfigUpdate(route_config); VhdsSubscription subscription(config_update_info, factory_context_, context_, provider_); - EXPECT_EQ(1UL, config_update_info->protobufConfiguration().virtual_hosts_size()); - EXPECT_EQ("vhost_rds1", config_update_info->protobufConfiguration().virtual_hosts(0).name()); + EXPECT_EQ(1UL, config_update_info->protobufConfigurationCast().virtual_hosts_size()); + EXPECT_EQ("vhost_rds1", config_update_info->protobufConfigurationCast().virtual_hosts(0).name()); auto vhost = buildVirtualHost("vhost_vhds1", "vhost.first"); const auto& added_resources = buildAddedResources({vhost}); @@ -194,14 +196,14 @@ name: my_route const Protobuf::RepeatedPtrField removed_resources; factory_context_.cluster_manager_.subscription_factory_.callbacks_->onConfigUpdate( decoded_resources.refvec_, removed_resources, "1"); - EXPECT_EQ(2UL, config_update_info->protobufConfiguration().virtual_hosts_size()); + EXPECT_EQ(2UL, config_update_info->protobufConfigurationCast().virtual_hosts_size()); config_update_info->onRdsUpdate(updated_route_config, "2"); - EXPECT_EQ(3UL, config_update_info->protobufConfiguration().virtual_hosts_size()); - auto actual_vhost_0 = config_update_info->protobufConfiguration().virtual_hosts(0); - auto actual_vhost_1 = config_update_info->protobufConfiguration().virtual_hosts(1); - auto actual_vhost_2 = config_update_info->protobufConfiguration().virtual_hosts(2); + EXPECT_EQ(3UL, config_update_info->protobufConfigurationCast().virtual_hosts_size()); + auto actual_vhost_0 = config_update_info->protobufConfigurationCast().virtual_hosts(0); + auto actual_vhost_1 = config_update_info->protobufConfigurationCast().virtual_hosts(1); + auto actual_vhost_2 = config_update_info->protobufConfigurationCast().virtual_hosts(2); EXPECT_TRUE("vhost_rds1" == actual_vhost_0.name() || "vhost_rds1" == actual_vhost_1.name() || "vhost_rds1" == actual_vhost_2.name()); EXPECT_TRUE("vhost_rds2" == actual_vhost_0.name() || "vhost_rds2" == actual_vhost_1.name() || diff --git a/test/mocks/router/mocks.cc b/test/mocks/router/mocks.cc index 8ab3d486fba10..5ae26ddc415dc 100644 --- a/test/mocks/router/mocks.cc +++ b/test/mocks/router/mocks.cc @@ -136,6 +136,7 @@ MockRoute::~MockRoute() = default; MockRouteConfigProvider::MockRouteConfigProvider() { ON_CALL(*this, config()).WillByDefault(Return(route_config_)); + ON_CALL(*this, configCast()).WillByDefault(Return(route_config_)); } MockRouteConfigProvider::~MockRouteConfigProvider() = default; diff --git a/test/mocks/router/mocks.h b/test/mocks/router/mocks.h index e3ea24f69e9d2..81f08aa509680 100644 --- a/test/mocks/router/mocks.h +++ b/test/mocks/router/mocks.h @@ -502,10 +502,11 @@ class MockRouteConfigProvider : public RouteConfigProvider { MockRouteConfigProvider(); ~MockRouteConfigProvider() override; - MOCK_METHOD(ConfigConstSharedPtr, config, ()); - MOCK_METHOD(absl::optional, configInfo, (), (const)); + MOCK_METHOD(Rds::ConfigConstSharedPtr, config, (), (const)); + MOCK_METHOD(const absl::optional&, configInfo, (), (const)); MOCK_METHOD(SystemTime, lastUpdated, (), (const)); MOCK_METHOD(void, onConfigUpdate, ()); + MOCK_METHOD(ConfigConstSharedPtr, configCast, (), (const)); MOCK_METHOD(void, requestVirtualHostsUpdate, (const std::string&, Event::Dispatcher&, std::weak_ptr route_config_updated_cb));