diff --git a/include/envoy/config/BUILD b/include/envoy/config/BUILD index bf35d5762ff5b..5c41821d10c80 100644 --- a/include/envoy/config/BUILD +++ b/include/envoy/config/BUILD @@ -33,3 +33,23 @@ envoy_cc_library( "//source/common/protobuf", ], ) + +envoy_cc_library( + name = "config_provider_interface", + hdrs = ["config_provider.h"], + external_deps = ["abseil_optional"], + deps = [ + "//include/envoy/common:time_interface", + "//source/common/protobuf", + ], +) + +envoy_cc_library( + name = "config_provider_manager_interface", + hdrs = ["config_provider_manager.h"], + deps = [ + ":config_provider_interface", + "//include/envoy/server:filter_config_interface", + "//source/common/protobuf", + ], +) diff --git a/include/envoy/config/config_provider.h b/include/envoy/config/config_provider.h new file mode 100644 index 0000000000000..a42f512dea4e9 --- /dev/null +++ b/include/envoy/config/config_provider.h @@ -0,0 +1,113 @@ +#pragma once + +#include + +#include "envoy/common/time.h" + +#include "common/protobuf/protobuf.h" + +#include "absl/types/optional.h" + +namespace Envoy { +namespace Config { + +/** + * A provider for configuration obtained statically (via static resources in the bootstrap config), + * inline with a higher level resource or dynamically via xDS APIs. + * + * The ConfigProvider is an abstraction layer which higher level components such as the + * HttpConnectionManager, Listener, etc can leverage to interface with Envoy's configuration + * mechanisms. Implementations of this interface build upon lower level abstractions such as + * Envoy::Config::Subscription and Envoy::Config::SubscriptionCallbacks. + * + * The interface exposed below allows xDS providers to share the underlying config protos and + * resulting config implementations (i.e., the ConfigProvider::Config); this enables linear memory + * scaling based on the size of the configuration set, regardless of the number of threads/workers. + * + * Use config() to obtain a shared_ptr to the implementation of the config, and configProtoInfo() to + * obtain a reference to the underlying config proto and version (applicable only to dynamic config + * providers). + */ +class ConfigProvider { +public: + /** + * The "implementation" of the configuration. + * Use config() to obtain a typed object that corresponds to the specific configuration + * represented by this abstract type. + */ + class Config { + public: + virtual ~Config() = default; + }; + using ConfigConstSharedPtr = std::shared_ptr; + + /** + * Stores the config proto as well as the associated version. + */ + template struct ConfigProtoInfo { + const P& config_proto_; + + // Only populated by dynamic config providers. + std::string version_; + }; + + virtual ~ConfigProvider() = default; + + /** + * Returns a ConfigProtoInfo associated with the provider. + * @return absl::optional> an optional ConfigProtoInfo; the value is set when a + * config is available. + */ + template absl::optional> configProtoInfo() const { + static_assert(std::is_base_of::value, + "Proto type must derive from Protobuf::Message"); + + const auto* config_proto = dynamic_cast(getConfigProto()); + if (config_proto == nullptr) { + return absl::nullopt; + } + return ConfigProtoInfo

{*config_proto, getConfigVersion()}; + } + + /** + * Returns the Config corresponding to the provider. + * @return std::shared_ptr a shared pointer to the Config. + */ + template std::shared_ptr config() const { + static_assert(std::is_base_of::value, + "Config type must derive from ConfigProvider::Config"); + + return std::dynamic_pointer_cast(getConfig()); + } + + /** + * Returns the timestamp associated with the last update to the Config. + * @return SystemTime the timestamp corresponding to the last config update. + */ + virtual SystemTime lastUpdated() const PURE; + +protected: + /** + * Returns the config proto associated with the provider. + * @return Protobuf::Message* the config proto corresponding to the Config instantiated by the + * provider. + */ + virtual const Protobuf::Message* getConfigProto() const PURE; + + /** + * Returns the config version associated with the provider. + * @return std::string the config version. + */ + virtual std::string getConfigVersion() const PURE; + + /** + * Returns the config implementation associated with the provider. + * @return ConfigConstSharedPtr the config as the base type. + */ + virtual ConfigConstSharedPtr getConfig() const PURE; +}; + +using ConfigProviderPtr = std::unique_ptr; + +} // namespace Config +} // namespace Envoy diff --git a/include/envoy/config/config_provider_manager.h b/include/envoy/config/config_provider_manager.h new file mode 100644 index 0000000000000..3c3eddba9c2f0 --- /dev/null +++ b/include/envoy/config/config_provider_manager.h @@ -0,0 +1,58 @@ +#pragma once + +#include + +#include "envoy/config/config_provider.h" +#include "envoy/server/filter_config.h" + +#include "common/protobuf/protobuf.h" + +namespace Envoy { +namespace Config { + +/** + * A ConfigProvider manager which instantiates static and dynamic (xDS) providers. + * + * ConfigProvider objects are owned by the caller of the + * createXdsConfigProvider()/createStaticConfigProvider() functions. The ConfigProviderManager holds + * raw pointers to those objects. + * + * Configuration implementations returned by ConfigProvider::config() are immutable, which allows + * them to share the underlying objects such as config protos and subscriptions (for dynamic + * providers) without synchronization related performance penalties. This enables linear memory + * growth based on the size of the configuration set, regardless of the number of threads/objects + * that must hold a reference/pointer to them. + */ +class ConfigProviderManager { +public: + virtual ~ConfigProviderManager() = default; + + /** + * Returns a dynamic ConfigProvider which receives configuration via an xDS API. + * A shared ownership model is used, such that the underlying subscription, config proto + * and Config are shared amongst all providers relying on the same config source. + * @param config_source_proto supplies the proto containing the xDS API configuration. + * @param factory_context is the context to use for the provider. + * @param stat_prefix supplies the prefix to use for statistics. + * @return ConfigProviderPtr a newly allocated dynamic config provider which shares underlying + * data structures with other dynamic providers configured with the same + * API source. + */ + virtual ConfigProviderPtr + createXdsConfigProvider(const Protobuf::Message& config_source_proto, + Server::Configuration::FactoryContext& factory_context, + const std::string& stat_prefix) PURE; + + /** + * Returns a ConfigProvider associated with a statically specified configuration. + * @param config_proto supplies the configuration proto. + * @param factory_context is the context to use for the provider. + * @return ConfigProviderPtr a newly allocated static config provider. + */ + virtual ConfigProviderPtr + createStaticConfigProvider(const Protobuf::Message& config_proto, + Server::Configuration::FactoryContext& factory_context) PURE; +}; + +} // namespace Config +} // namespace Envoy diff --git a/source/common/common/utility.h b/source/common/common/utility.h index 86e77eb46d619..89db4c2ae0151 100644 --- a/source/common/common/utility.h +++ b/source/common/common/utility.h @@ -523,6 +523,17 @@ struct StringViewHash { std::size_t operator()(const absl::string_view& k) const { return HashUtil::xxHash64(k); } }; +/** + * Hashing functor for use with enum class types. + * This is needed for GCC 5.X; newer versions of GCC, as well as clang7, provide native hashing + * specializations. + */ +struct EnumClassHash { + template std::size_t operator()(T t) const { + return std::hash()(static_cast(t)); + } +}; + /** * Computes running standard-deviation using Welford's algorithm: * https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Online_algorithm diff --git a/source/common/config/BUILD b/source/common/config/BUILD index c3c391af5797d..921eee5b5dc26 100644 --- a/source/common/config/BUILD +++ b/source/common/config/BUILD @@ -353,3 +353,20 @@ envoy_cc_library( "//source/common/singleton:const_singleton", ], ) + +envoy_cc_library( + name = "config_provider_lib", + srcs = ["config_provider_impl.cc"], + hdrs = ["config_provider_impl.h"], + deps = [ + ":utility_lib", + "//include/envoy/config:config_provider_interface", + "//include/envoy/config:config_provider_manager_interface", + "//include/envoy/init:init_interface", + "//include/envoy/server:admin_interface", + "//include/envoy/server:config_tracker_interface", + "//include/envoy/singleton:instance_interface", + "//include/envoy/thread_local:thread_local_interface", + "//source/common/protobuf", + ], +) diff --git a/source/common/config/config_provider_impl.cc b/source/common/config/config_provider_impl.cc new file mode 100644 index 0000000000000..541c767412aac --- /dev/null +++ b/source/common/config/config_provider_impl.cc @@ -0,0 +1,121 @@ +#include "common/config/config_provider_impl.h" + +namespace Envoy { +namespace Config { + +ImmutableConfigProviderImplBase::ImmutableConfigProviderImplBase( + Server::Configuration::FactoryContext& factory_context, + ConfigProviderManagerImplBase& config_provider_manager, ConfigProviderInstanceType type) + : last_updated_(factory_context.timeSource().systemTime()), + config_provider_manager_(config_provider_manager), type_(type) { + config_provider_manager_.bindImmutableConfigProvider(this); +} + +ImmutableConfigProviderImplBase::~ImmutableConfigProviderImplBase() { + config_provider_manager_.unbindImmutableConfigProvider(this); +} + +ConfigSubscriptionInstanceBase::~ConfigSubscriptionInstanceBase() { + runInitializeCallbackIfAny(); + config_provider_manager_.unbindSubscription(manager_identifier_); +} + +void ConfigSubscriptionInstanceBase::runInitializeCallbackIfAny() { + if (initialize_callback_) { + initialize_callback_(); + initialize_callback_ = nullptr; + } +} + +bool ConfigSubscriptionInstanceBase::checkAndApplyConfig(const Protobuf::Message& config_proto, + const std::string& config_name, + const std::string& version_info) { + const uint64_t new_hash = MessageUtil::hash(config_proto); + if (config_info_ && config_info_.value().last_config_hash_ == new_hash) { + return false; + } + + config_info_ = {new_hash, version_info}; + ENVOY_LOG(debug, "{}: loading new configuration: config_name={} hash={}", name_, config_name, + new_hash); + + ASSERT(!mutable_config_providers_.empty()); + ConfigProvider::ConfigConstSharedPtr new_config; + for (auto* provider : mutable_config_providers_) { + // All bound mutable config providers must be of the same type (see the ASSERT... in + // bindConfigProvider()). + // This makes it safe to call any of the provider's onConfigProtoUpdate() to get a new config + // impl, which can then be passed to all providers. + if (new_config == nullptr) { + if ((new_config = provider->onConfigProtoUpdate(config_proto)) == nullptr) { + return false; + } + } + provider->onConfigUpdate(new_config); + } + + return true; +} + +void ConfigSubscriptionInstanceBase::bindConfigProvider(MutableConfigProviderImplBase* provider) { + // All config providers bound to a ConfigSubscriptionInstanceBase must be of the same concrete + // type; this is assumed by checkAndApplyConfig() and is verified by the assertion below. + // NOTE: an inlined statement ASSERT() triggers a potentially evaluated expression warning from + // clang due to `typeid(**mutable_config_providers_.begin())`. To avoid this, we use a lambda to + // separate the first mutable provider dereference from the typeid() statement. + ASSERT([&]() { + if (!mutable_config_providers_.empty()) { + const auto& first_provider = **mutable_config_providers_.begin(); + return typeid(*provider) == typeid(first_provider); + } + return true; + }()); + mutable_config_providers_.insert(provider); +} + +ConfigProviderManagerImplBase::ConfigProviderManagerImplBase(Server::Admin& admin, + const std::string& config_name) { + config_tracker_entry_ = + admin.getConfigTracker().add(config_name, [this] { return dumpConfigs(); }); + // ConfigTracker keys must be unique. We are asserting that no one has stolen the key + // from us, since the returned entry will be nullptr if the key already exists. + RELEASE_ASSERT(config_tracker_entry_, ""); +} + +const ConfigProviderManagerImplBase::ConfigProviderSet& +ConfigProviderManagerImplBase::immutableConfigProviders(ConfigProviderInstanceType type) const { + static ConfigProviderSet empty_set; + ConfigProviderMap::const_iterator it; + if ((it = immutable_config_providers_map_.find(type)) == immutable_config_providers_map_.end()) { + return empty_set; + } + + return *it->second; +} + +void ConfigProviderManagerImplBase::bindImmutableConfigProvider( + ImmutableConfigProviderImplBase* provider) { + ASSERT(provider->type() == ConfigProviderInstanceType::Static || + provider->type() == ConfigProviderInstanceType::Inline); + ConfigProviderMap::iterator it; + if ((it = immutable_config_providers_map_.find(provider->type())) == + immutable_config_providers_map_.end()) { + immutable_config_providers_map_.insert(std::make_pair( + provider->type(), + std::make_unique(std::initializer_list({provider})))); + } else { + it->second->insert(provider); + } +} + +void ConfigProviderManagerImplBase::unbindImmutableConfigProvider( + ImmutableConfigProviderImplBase* provider) { + ASSERT(provider->type() == ConfigProviderInstanceType::Static || + provider->type() == ConfigProviderInstanceType::Inline); + auto it = immutable_config_providers_map_.find(provider->type()); + ASSERT(it != immutable_config_providers_map_.end()); + it->second->erase(provider); +} + +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/config_provider_impl.h b/source/common/config/config_provider_impl.h new file mode 100644 index 0000000000000..12d6fa73665f6 --- /dev/null +++ b/source/common/config/config_provider_impl.h @@ -0,0 +1,431 @@ +#pragma once + +#include + +#include "envoy/config/config_provider.h" +#include "envoy/config/config_provider_manager.h" +#include "envoy/init/init.h" +#include "envoy/server/admin.h" +#include "envoy/server/config_tracker.h" +#include "envoy/singleton/instance.h" +#include "envoy/thread_local/thread_local.h" + +#include "common/common/thread.h" +#include "common/common/utility.h" +#include "common/config/utility.h" +#include "common/protobuf/protobuf.h" + +namespace Envoy { +namespace Config { + +// This file provides a set of base classes, (ImmutableConfigProviderImplBase, +// MutableConfigProviderImplBase, ConfigProviderManagerImplBase, ConfigSubscriptionInstanceBase), +// conforming to the ConfigProvider/ConfigProviderManager interfaces, which in tandem provide a +// framework for implementing statically defined (i.e., immutable) and dynamic (mutable via +// subscriptions) configuration for Envoy. +// +// The mutability property applies to the ConfigProvider itself and _not_ the underlying config +// proto, which is always immutable. MutableConfigProviderImplBase objects receive config proto +// updates via xDS subscriptions, resulting in new ConfigProvider::Config objects being instantiated +// with the corresponding change in behavior corresponding to updated config. ConfigProvider::Config +// objects must be latched/associated with the appropriate objects in the connection and request +// processing pipeline, such that configuration stays consistent for the lifetime of the connection +// and/or stream/request (if required by the configuration being processed). +// +// Dynamic configuration is distributed via xDS APIs (see +// https://github.com/envoyproxy/data-plane-api/blob/master/XDS_PROTOCOL.md). The framework exposed +// by these classes simplifies creation of client xDS implementations following a shared ownership +// model, where according to the config source specification, a config subscription, config protos +// received over the subscription and the subsequent config "implementation" (i.e., data structures +// and associated business logic) are shared across ConfigProvider objects and Envoy worker threads. +// +// This approach enables linear memory scalability based primarily on the size of the configuration +// set. +// +// A blueprint to follow for implementing {im,}mutable config providers is as follows: +// +// For both: +// 1) Create a class derived from ConfigProviderManagerImplBase and implement the required +// interface. +// When implementing createXdsConfigProvider(), it is expected that getSubscription() will +// be called to fetch either an existing ConfigSubscriptionInstanceBase if the config source +// configuration matches, or a newly instantiated subscription otherwise. +// +// For immutable providers: +// 1) Create a class derived from ImmutableConfigProviderImplBase and implement the required +// interface. +// +// For mutable (xDS) providers: +// 1) Create a class derived from MutableConfigProviderImplBase and implement the required +// interface. +// 2) Create a class derived from ConfigSubscriptionInstanceBase; this is the entity +// responsible for owning and managing the Envoy::Config::Subscription that provides +// the underlying config subscription. +// - When subscription callbacks (onConfigUpdate, onConfigUpdateFailed) are issued by the +// underlying subscription, the corresponding ConfigSubscriptionInstanceBase functions must be +// called as well. +// - On a successful config update, checkAndApplyConfig() should be called to instantiate the +// new config implementation and propagate it to the shared config providers and all +// worker threads. +// - On a successful return from checkAndApplyConfig(), the config proto must be latched into +// this class and returned via the getConfigProto() override. + +class ConfigProviderManagerImplBase; + +/** + * Specifies the type of config associated with a ConfigProvider. + */ +enum class ConfigProviderInstanceType { + // Configuration defined as a static resource in the bootstrap config. + Static, + // Configuration defined inline in a resource that may be specified statically or obtained via + // xDS. + Inline, + // Configuration obtained from an xDS subscription. + Xds +}; + +/** + * ConfigProvider implementation for immutable configuration. + * + * TODO(AndresGuedez): support sharing of config protos and config impls, as is + * done with the MutableConfigProviderImplBase. + * + * This class can not be instantiated directly; instead, it provides the foundation for + * immutable config provider implementations which derive from it. + */ +class ImmutableConfigProviderImplBase : public ConfigProvider { +public: + ~ImmutableConfigProviderImplBase() override; + + // Envoy::Config::ConfigProvider + SystemTime lastUpdated() const override { return last_updated_; } + + ConfigProviderInstanceType type() const { return type_; } + +protected: + ImmutableConfigProviderImplBase(Server::Configuration::FactoryContext& factory_context, + ConfigProviderManagerImplBase& config_provider_manager, + ConfigProviderInstanceType type); + +private: + SystemTime last_updated_; + ConfigProviderManagerImplBase& config_provider_manager_; + ConfigProviderInstanceType type_; +}; + +class MutableConfigProviderImplBase; + +/** + * Provides generic functionality required by all xDS ConfigProvider subscriptions, including + * shared lifetime management via shared_ptr. + * + * To do so, this class keeps track of a set of MutableConfigProviderImplBase instances associated + * with an underlying subscription; providers are bound/unbound as needed as they are created and + * destroyed. + * + * xDS config providers and subscriptions are split to avoid lifetime issues with arguments + * required by the config providers. An example is the Server::Configuration::FactoryContext, which + * is owned by listeners and therefore may be destroyed while an associated config provider is still + * in use (see #3960). This split enables single ownership of the config providers, while enabling + * shared ownership of the underlying subscription. + * + * This class can not be instantiated directly; instead, it provides the foundation for + * config subscription implementations which derive from it. + */ +class ConfigSubscriptionInstanceBase : public Init::Target, + protected Logger::Loggable { +public: + struct LastConfigInfo { + uint64_t last_config_hash_; + std::string last_config_version_; + }; + + ~ConfigSubscriptionInstanceBase() override; + + // Init::Target + void initialize(std::function callback) override { + initialize_callback_ = callback; + start(); + } + + /** + * Starts the subscription corresponding to a config source. + * A derived class must own the configuration proto specific Envoy::Config::Subscription to be + * started. + */ + virtual void start() PURE; + + const SystemTime& lastUpdated() const { return last_updated_; } + + const absl::optional& configInfo() const { return config_info_; } + + /** + * Must be called by derived classes when the onConfigUpdate() callback associated with the + * underlying subscription is issued. + */ + void onConfigUpdate() { + setLastUpdated(); + runInitializeCallbackIfAny(); + } + + /** + * Must be called by derived classes when the onConfigUpdateFailed() callback associated with the + * underlying subscription is issued. + */ + void onConfigUpdateFailed() { runInitializeCallbackIfAny(); } + + /** + * Determines whether a configuration proto is a new update, and if so, propagates it to all + * config providers associated with this subscription. + * @param config_proto supplies the newly received config proto. + * @param config_name supplies the name associated with the config. + * @param version_info supplies the version associated with the config. + * @return bool false when the config proto has no delta from the previous config, true otherwise. + */ + bool checkAndApplyConfig(const Protobuf::Message& config_proto, const std::string& config_name, + const std::string& version_info); + + /** + * Returns one of the bound mutable config providers. + * @return const MutableConfigProviderImplBase* a const pointer to a + * bound MutableConfigProviderImplBase or nullptr when there are none. + */ + const MutableConfigProviderImplBase* getAnyBoundMutableConfigProvider() const { + return !mutable_config_providers_.empty() ? *mutable_config_providers_.begin() : nullptr; + } + +protected: + ConfigSubscriptionInstanceBase(const std::string& name, const std::string& manager_identifier, + ConfigProviderManagerImplBase& config_provider_manager, + TimeSource& time_source, const SystemTime& last_updated, + const LocalInfo::LocalInfo& local_info) + : name_(name), manager_identifier_(manager_identifier), + config_provider_manager_(config_provider_manager), time_source_(time_source), + last_updated_(last_updated) { + Envoy::Config::Utility::checkLocalInfo(name, local_info); + } + + void setLastUpdated() { last_updated_ = time_source_.systemTime(); } + + void runInitializeCallbackIfAny(); + +private: + void registerInitTarget(Init::Manager& init_manager) { init_manager.registerTarget(*this); } + + void bindConfigProvider(MutableConfigProviderImplBase* provider); + + void unbindConfigProvider(MutableConfigProviderImplBase* provider) { + mutable_config_providers_.erase(provider); + } + + const std::string name_; + std::function initialize_callback_; + std::unordered_set mutable_config_providers_; + const std::string manager_identifier_; + ConfigProviderManagerImplBase& config_provider_manager_; + TimeSource& time_source_; + SystemTime last_updated_; + absl::optional config_info_; + + // ConfigSubscriptionInstanceBase, MutableConfigProviderImplBase and ConfigProviderManagerImplBase + // are tightly coupled with the current shared ownership model; use friend classes to explicitly + // denote the binding between them. + // + // TODO(AndresGuedez): Investigate whether a shared ownership model avoiding the s and + // instead centralizing lifetime management in the ConfigProviderManagerImplBase with explicit + // reference counting would be more maintainable. + friend class MutableConfigProviderImplBase; + friend class ConfigProviderManagerImplBase; +}; + +using ConfigSubscriptionInstanceBaseSharedPtr = std::shared_ptr; + +/** + * Provides generic functionality required by all dynamic config providers, including distribution + * of config updates to all workers. + * + * This class can not be instantiated directly; instead, it provides the foundation for + * dynamic config provider implementations which derive from it. + */ +class MutableConfigProviderImplBase : public ConfigProvider { +public: + ~MutableConfigProviderImplBase() override { subscription_->unbindConfigProvider(this); } + + // Envoy::Config::ConfigProvider + SystemTime lastUpdated() const override { return subscription_->lastUpdated(); } + + // Envoy::Config::ConfigProvider + ConfigConstSharedPtr getConfig() const override { + return tls_->getTyped().config_; + } + + /** + * Called when a new config proto is received via an xDS subscription. + * On successful validation of the config, must return a shared_ptr to a ConfigProvider::Config + * implementation that will be propagated to all mutable config providers sharing the + * subscription. + * Note that this function is called _once_ across all shared config providers per xDS + * subscription config update. + * @param config_proto supplies the configuration proto. + * @return ConfigConstSharedPtr the ConfigProvider::Config to share with other providers. + */ + virtual ConfigConstSharedPtr onConfigProtoUpdate(const Protobuf::Message& config_proto) PURE; + + /** + * Must be called by the derived class' constructor. + * @param initial_config supplies an initial Envoy::Config::ConfigProvider::Config associated with + * the underlying subscription. + */ + void initialize(const ConfigConstSharedPtr& initial_config) { + subscription_->bindConfigProvider(this); + tls_->set([initial_config](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { + return std::make_shared(initial_config); + }); + } + + /** + * Propagates a newly instantiated Envoy::Config::ConfigProvider::Config to all workers. + * @param config supplies the newly instantiated config. + */ + void onConfigUpdate(const ConfigConstSharedPtr& config) { + tls_->runOnAllThreads( + [this, config]() -> void { tls_->getTyped().config_ = config; }); + } + +protected: + MutableConfigProviderImplBase(ConfigSubscriptionInstanceBaseSharedPtr&& subscription, + Server::Configuration::FactoryContext& factory_context) + : subscription_(subscription), tls_(factory_context.threadLocal().allocateSlot()) {} + + const ConfigSubscriptionInstanceBaseSharedPtr& subscription() const { return subscription_; } + +private: + struct ThreadLocalConfig : public ThreadLocal::ThreadLocalObject { + ThreadLocalConfig(ConfigProvider::ConfigConstSharedPtr initial_config) + : config_(std::move(initial_config)) {} + + ConfigProvider::ConfigConstSharedPtr config_; + }; + + ConfigSubscriptionInstanceBaseSharedPtr subscription_; + ThreadLocal::SlotPtr tls_; +}; + +/** + * Provides generic functionality required by all config provider managers, such as managing shared + * lifetime of subscriptions and dynamic config providers, along with determining which + * subscriptions should be associated with newly instantiated providers. + * + * The implementation of this class is not thread safe. Note that ImmutableConfigProviderImplBase + * and ConfigSubscriptionInstanceBase call the corresponding {bind,unbind}* functions exposed by + * this class. + * + * All config processing is done on the main thread, so instantiation of *ConfigProvider* objects + * via createStaticConfigProvider() and createXdsConfigProvider() is naturally thread safe. Care + * must be taken with regards to destruction of these objects, since it must also happen on the main + * thread _prior_ to destruction of the ConfigProviderManagerImplBase oject from which they were + * created. + * + * This class can not be instantiated directly; instead, it provides the foundation for + * dynamic config provider implementations which derive from it. + */ +class ConfigProviderManagerImplBase : public ConfigProviderManager, public Singleton::Instance { +public: + ~ConfigProviderManagerImplBase() override = default; + + /** + * This is invoked by the /config_dump admin handler. + * @return ProtobufTypes::MessagePtr the config dump proto corresponding to the associated + * config providers. + */ + virtual ProtobufTypes::MessagePtr dumpConfigs() const PURE; + +protected: + using ConfigProviderSet = std::unordered_set; + using ConfigProviderMap = std::unordered_map, EnumClassHash>; + using ConfigSubscriptionMap = + std::unordered_map>; + + ConfigProviderManagerImplBase(Server::Admin& admin, const std::string& config_name); + + const ConfigSubscriptionMap& configSubscriptions() const { return config_subscriptions_; } + + /** + * Returns the set of bound ImmutableConfigProviderImplBase-derived providers of a given type. + * @param type supplies the type of config providers to return. + * @return const ConfigProviderSet* the set of config providers corresponding to the type. + */ + const ConfigProviderSet& immutableConfigProviders(ConfigProviderInstanceType type) const; + + /** + * Returns the subscription associated with the config_source_proto; if none exists, a new one is + * allocated according to the subscription_factory_fn. + * @param config_source_proto supplies the proto specifying the config subscription paremeters. + * @param init_manager supplies the init manager. + * @param subscription_factory_fn supplies a function to be called when a new subscription needs + * to be allocated. + * @return std::shared_ptr an existing (if a match is found) or newly allocated subscription. + */ + template + std::shared_ptr getSubscription( + const Protobuf::Message& config_source_proto, Init::Manager& init_manager, + const std::function& subscription_factory_fn) { + static_assert(std::is_base_of::value, + "T must be a subclass of ConfigSubscriptionInstanceBase"); + + ConfigSubscriptionInstanceBaseSharedPtr subscription; + const std::string manager_identifier = config_source_proto.SerializeAsString(); + + auto it = config_subscriptions_.find(manager_identifier); + if (it == config_subscriptions_.end()) { + // std::make_shared does not work for classes with private constructors. There are ways + // around it. However, since this is not a performance critical path we err on the side + // of simplicity. + subscription = subscription_factory_fn(manager_identifier, *this); + + subscription->registerInitTarget(init_manager); + + bindSubscription(manager_identifier, subscription); + } else { + // Because the ConfigProviderManagerImplBase's weak_ptrs only get cleaned up + // in the ConfigSubscriptionInstanceBase destructor, and the single threaded nature + // of this code, locking the weak_ptr will not fail. + subscription = it->second.lock(); + } + ASSERT(subscription); + + return std::static_pointer_cast(subscription); + } + +private: + void bindSubscription(const std::string& manager_identifier, + ConfigSubscriptionInstanceBaseSharedPtr& subscription) { + config_subscriptions_.insert({manager_identifier, subscription}); + } + + void unbindSubscription(const std::string& manager_identifier) { + config_subscriptions_.erase(manager_identifier); + } + + void bindImmutableConfigProvider(ImmutableConfigProviderImplBase* provider); + void unbindImmutableConfigProvider(ImmutableConfigProviderImplBase* provider); + + // TODO(jsedgwick) These two members are prime candidates for the owned-entry list/map + // as in ConfigTracker. I.e. the ProviderImpls would have an EntryOwner for these lists + // Then the lifetime management stuff is centralized and opaque. + ConfigSubscriptionMap config_subscriptions_; + ConfigProviderMap immutable_config_providers_map_; + + Server::ConfigTracker::EntryOwnerPtr config_tracker_entry_; + + // See comment for friend classes in the ConfigSubscriptionInstanceBase for more details on the + // use of friends. + friend class ConfigSubscriptionInstanceBase; + friend class ImmutableConfigProviderImplBase; +}; + +} // namespace Config +} // namespace Envoy diff --git a/test/common/config/BUILD b/test/common/config/BUILD index 918e263d850a9..d80c5b07dd781 100644 --- a/test/common/config/BUILD +++ b/test/common/config/BUILD @@ -5,6 +5,7 @@ load( "envoy_cc_test", "envoy_cc_test_library", "envoy_package", + "envoy_proto_library", ) envoy_package() @@ -208,3 +209,19 @@ envoy_cc_test( "//source/common/config:filter_json_lib", ], ) + +envoy_proto_library( + name = "dummy_config_proto", + srcs = ["dummy_config.proto"], +) + +envoy_cc_test( + name = "config_provider_impl_test", + srcs = ["config_provider_impl_test.cc"], + deps = [ + ":dummy_config_proto_cc", + "//source/common/config:config_provider_lib", + "//source/common/protobuf:utility_lib", + "//test/mocks/server:server_mocks", + ], +) diff --git a/test/common/config/config_provider_impl_test.cc b/test/common/config/config_provider_impl_test.cc new file mode 100644 index 0000000000000..7e928963aed9f --- /dev/null +++ b/test/common/config/config_provider_impl_test.cc @@ -0,0 +1,384 @@ +#include + +#include "common/config/config_provider_impl.h" +#include "common/protobuf/utility.h" + +#include "test/common/config/dummy_config.pb.h" +#include "test/mocks/server/mocks.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace Envoy { +namespace Config { +namespace { + +class DummyConfigProviderManager; + +class StaticDummyConfigProvider : public ImmutableConfigProviderImplBase { +public: + StaticDummyConfigProvider(const test::common::config::DummyConfig& config_proto, + Server::Configuration::FactoryContext& factory_context, + DummyConfigProviderManager& config_provider_manager); + + ~StaticDummyConfigProvider() override = default; + + // Envoy::Config::ConfigProvider + const Protobuf::Message* getConfigProto() const override { return &config_proto_; } + + // Envoy::Config::ConfigProvider + std::string getConfigVersion() const override { return ""; } + + // Envoy::Config::ConfigProvider + ConfigConstSharedPtr getConfig() const override { return config_; } + +private: + ConfigConstSharedPtr config_; + test::common::config::DummyConfig config_proto_; +}; + +class DummyConfigSubscription + : public ConfigSubscriptionInstanceBase, + Envoy::Config::SubscriptionCallbacks { +public: + DummyConfigSubscription(const std::string& manager_identifier, + Server::Configuration::FactoryContext& factory_context, + DummyConfigProviderManager& config_provider_manager); + + ~DummyConfigSubscription() override = default; + + // Envoy::Config::ConfigSubscriptionInstanceBase + void start() override {} + + // Envoy::Config::SubscriptionCallbacks + void onConfigUpdate(const ResourceVector& resources, const std::string& version_info) override { + const auto& config = resources[0]; + if (checkAndApplyConfig(config, "dummy_config", version_info)) { + config_proto_ = config; + } + + ConfigSubscriptionInstanceBase::onConfigUpdate(); + } + + // Envoy::Config::SubscriptionCallbacks + void onConfigUpdateFailed(const EnvoyException*) override {} + + // Envoy::Config::SubscriptionCallbacks + std::string resourceName(const ProtobufWkt::Any&) override { return ""; } + + const absl::optional& config_proto() const { + return config_proto_; + } + +private: + absl::optional config_proto_; +}; + +using DummyConfigSubscriptionSharedPtr = std::shared_ptr; + +class DummyConfig : public ConfigProvider::Config { +public: + DummyConfig(const test::common::config::DummyConfig&) {} +}; + +class DummyDynamicConfigProvider : public MutableConfigProviderImplBase { +public: + DummyDynamicConfigProvider(DummyConfigSubscriptionSharedPtr&& subscription, + ConfigConstSharedPtr initial_config, + Server::Configuration::FactoryContext& factory_context) + : MutableConfigProviderImplBase(std::move(subscription), factory_context), + subscription_(static_cast( + MutableConfigProviderImplBase::subscription().get())) { + initialize(initial_config); + } + + ~DummyDynamicConfigProvider() override = default; + + DummyConfigSubscription& subscription() { return *subscription_; } + + // Envoy::Config::MutableConfigProviderImplBase + ConfigProvider::ConfigConstSharedPtr + onConfigProtoUpdate(const Protobuf::Message& config) override { + return std::make_shared( + static_cast(config)); + } + + // Envoy::Config::ConfigProvider + const Protobuf::Message* getConfigProto() const override { + if (!subscription_->config_proto().has_value()) { + return nullptr; + } + return &subscription_->config_proto().value(); + } + + // Envoy::Config::ConfigProvider + std::string getConfigVersion() const override { return ""; } + +private: + // Lifetime of this pointer is owned by the shared_ptr held by the base class. + DummyConfigSubscription* subscription_; +}; + +class DummyConfigProviderManager : public ConfigProviderManagerImplBase { +public: + DummyConfigProviderManager(Server::Admin& admin) + : ConfigProviderManagerImplBase(admin, "dummy") {} + + ~DummyConfigProviderManager() override = default; + + // Envoy::Config::ConfigProviderManagerImplBase + ProtobufTypes::MessagePtr dumpConfigs() const override { + auto config_dump = std::make_unique(); + for (const auto& element : configSubscriptions()) { + auto subscription = element.second.lock(); + ASSERT(subscription); + + if (subscription->configInfo()) { + auto* dynamic_config = config_dump->mutable_dynamic_dummy_configs()->Add(); + dynamic_config->set_version_info(subscription->configInfo().value().last_config_version_); + dynamic_config->mutable_dummy_config()->MergeFrom( + static_cast(subscription.get())->config_proto().value()); + TimestampUtil::systemClockToTimestamp(subscription->lastUpdated(), + *dynamic_config->mutable_last_updated()); + } + } + + for (const auto* provider : immutableConfigProviders(ConfigProviderInstanceType::Static)) { + ASSERT(provider->configProtoInfo()); + auto* static_config = config_dump->mutable_static_dummy_configs()->Add(); + static_config->mutable_dummy_config()->MergeFrom( + provider->configProtoInfo().value().config_proto_); + TimestampUtil::systemClockToTimestamp(provider->lastUpdated(), + *static_config->mutable_last_updated()); + } + + return config_dump; + } + + // Envoy::Config::ConfigProviderManager + ConfigProviderPtr createXdsConfigProvider(const Protobuf::Message& config_source_proto, + Server::Configuration::FactoryContext& factory_context, + const std::string&) override { + DummyConfigSubscriptionSharedPtr subscription = getSubscription( + config_source_proto, factory_context.initManager(), + [&factory_context](const std::string& manager_identifier, + ConfigProviderManagerImplBase& config_provider_manager) + -> ConfigSubscriptionInstanceBaseSharedPtr { + return std::make_shared( + manager_identifier, factory_context, + static_cast(config_provider_manager)); + }); + + ConfigProvider::ConfigConstSharedPtr initial_config; + const MutableConfigProviderImplBase* provider = + subscription->getAnyBoundMutableConfigProvider(); + if (provider) { + initial_config = provider->getConfig(); + } + return std::make_unique(std::move(subscription), initial_config, + factory_context); + } + + // Envoy::Config::ConfigProviderManager + ConfigProviderPtr + createStaticConfigProvider(const Protobuf::Message& config_proto, + Server::Configuration::FactoryContext& factory_context) override { + return std::make_unique( + dynamic_cast(config_proto), factory_context, + *this); + } +}; + +StaticDummyConfigProvider::StaticDummyConfigProvider( + const test::common::config::DummyConfig& config_proto, + Server::Configuration::FactoryContext& factory_context, + DummyConfigProviderManager& config_provider_manager) + : ImmutableConfigProviderImplBase(factory_context, config_provider_manager, + ConfigProviderInstanceType::Static), + config_(std::make_shared(config_proto)), config_proto_(config_proto) {} + +DummyConfigSubscription::DummyConfigSubscription( + const std::string& manager_identifier, Server::Configuration::FactoryContext& factory_context, + DummyConfigProviderManager& config_provider_manager) + : ConfigSubscriptionInstanceBase( + "DummyDS", manager_identifier, config_provider_manager, factory_context.timeSource(), + factory_context.timeSource().systemTime(), factory_context.localInfo()) {} + +class ConfigProviderImplTest : public testing::Test { +public: + ConfigProviderImplTest() { + EXPECT_CALL(factory_context_.admin_.config_tracker_, add_("dummy", _)); + provider_manager_ = std::make_unique(factory_context_.admin_); + } + + Event::SimulatedTimeSystem& timeSystem() { return factory_context_.timeSystem(); } + +protected: + NiceMock factory_context_; + std::unique_ptr provider_manager_; +}; + +test::common::config::DummyConfig parseDummyConfigFromYaml(const std::string& yaml) { + test::common::config::DummyConfig config; + MessageUtil::loadFromYaml(yaml, config); + return config; +} + +// Tests that dynamic config providers share ownership of the config +// subscriptions, config protos and data structures generated as a result of the +// configurations (i.e., the ConfigProvider::Config). +TEST_F(ConfigProviderImplTest, SharedOwnership) { + factory_context_.init_manager_.initialize(); + + envoy::api::v2::core::ApiConfigSource config_source_proto; + config_source_proto.set_api_type(envoy::api::v2::core::ApiConfigSource::GRPC); + ConfigProviderPtr provider1 = provider_manager_->createXdsConfigProvider( + config_source_proto, factory_context_, "dummy_prefix"); + + // No config protos have been received via the subscription yet. + EXPECT_FALSE(provider1->configProtoInfo().has_value()); + + Protobuf::RepeatedPtrField dummy_configs; + dummy_configs.Add()->MergeFrom(parseDummyConfigFromYaml("a: a dummy config")); + + DummyConfigSubscription& subscription = + dynamic_cast(*provider1).subscription(); + subscription.onConfigUpdate(dummy_configs, "1"); + + // Check that a newly created provider with the same config source will share + // the subscription, config proto and resulting ConfigProvider::Config. + ConfigProviderPtr provider2 = provider_manager_->createXdsConfigProvider( + config_source_proto, factory_context_, "dummy_prefix"); + + EXPECT_TRUE(provider2->configProtoInfo().has_value()); + EXPECT_EQ(&dynamic_cast(*provider1).subscription(), + &dynamic_cast(*provider2).subscription()); + EXPECT_EQ(&provider1->configProtoInfo().value().config_proto_, + &provider2->configProtoInfo().value().config_proto_); + EXPECT_EQ(provider1->config().get(), + provider2->config().get()); + + // Change the config source and verify that a new subscription is used. + config_source_proto.set_api_type(envoy::api::v2::core::ApiConfigSource::REST); + ConfigProviderPtr provider3 = provider_manager_->createXdsConfigProvider( + config_source_proto, factory_context_, "dummy_prefix"); + + EXPECT_NE(&dynamic_cast(*provider1).subscription(), + &dynamic_cast(*provider3).subscription()); + EXPECT_NE(provider1->config().get(), + provider3->config().get()); + + dynamic_cast(*provider3) + .subscription() + .onConfigUpdate(dummy_configs, "provider3"); + + EXPECT_EQ(2UL, static_cast( + provider_manager_->dumpConfigs().get()) + ->dynamic_dummy_configs() + .size()); + + // Test that tear down of config providers leads to correctly updating + // centralized state; this is validated using the config dump. + provider1.reset(); + provider2.reset(); + + auto dynamic_dummy_configs = + static_cast(provider_manager_->dumpConfigs().get()) + ->dynamic_dummy_configs(); + EXPECT_EQ(1UL, dynamic_dummy_configs.size()); + + EXPECT_EQ("provider3", dynamic_dummy_configs[0].version_info()); + + provider3.reset(); + + EXPECT_EQ(0UL, static_cast( + provider_manager_->dumpConfigs().get()) + ->dynamic_dummy_configs() + .size()); +} + +// Tests that the base ConfigProvider*s are handling registration with the +// /config_dump admin handler as well as generic bookkeeping such as timestamp +// updates. +TEST_F(ConfigProviderImplTest, ConfigDump) { + // Empty dump first. + auto message_ptr = factory_context_.admin_.config_tracker_.config_tracker_callbacks_["dummy"](); + const auto& dummy_config_dump = + static_cast(*message_ptr); + + test::common::config::DummyConfigsDump expected_config_dump; + MessageUtil::loadFromYaml(R"EOF( +static_dummy_configs: +dynamic_dummy_configs: +)EOF", + expected_config_dump); + EXPECT_EQ(expected_config_dump.DebugString(), dummy_config_dump.DebugString()); + + // Static config dump only. + std::string config_yaml = "a: a static dummy config"; + timeSystem().setSystemTime(std::chrono::milliseconds(1234567891234)); + + ConfigProviderPtr static_config = provider_manager_->createStaticConfigProvider( + parseDummyConfigFromYaml(config_yaml), factory_context_); + message_ptr = factory_context_.admin_.config_tracker_.config_tracker_callbacks_["dummy"](); + const auto& dummy_config_dump2 = + static_cast(*message_ptr); + MessageUtil::loadFromYaml(R"EOF( +static_dummy_configs: + - dummy_config: { a: a static dummy config } + last_updated: { seconds: 1234567891, nanos: 234000000 } +dynamic_dummy_configs: +)EOF", + expected_config_dump); + EXPECT_EQ(expected_config_dump.DebugString(), dummy_config_dump2.DebugString()); + + envoy::api::v2::core::ApiConfigSource config_source_proto; + config_source_proto.set_api_type(envoy::api::v2::core::ApiConfigSource::GRPC); + ConfigProviderPtr dynamic_provider = provider_manager_->createXdsConfigProvider( + config_source_proto, factory_context_, "dummy_prefix"); + + // Static + dynamic config dump. + Protobuf::RepeatedPtrField dummy_configs; + dummy_configs.Add()->MergeFrom(parseDummyConfigFromYaml("a: a dynamic dummy config")); + + timeSystem().setSystemTime(std::chrono::milliseconds(1234567891567)); + DummyConfigSubscription& subscription = + dynamic_cast(*dynamic_provider).subscription(); + subscription.onConfigUpdate(dummy_configs, "v1"); + + message_ptr = factory_context_.admin_.config_tracker_.config_tracker_callbacks_["dummy"](); + const auto& dummy_config_dump3 = + static_cast(*message_ptr); + MessageUtil::loadFromYaml(R"EOF( +static_dummy_configs: + - dummy_config: { a: a static dummy config } + last_updated: { seconds: 1234567891, nanos: 234000000 } +dynamic_dummy_configs: + - version_info: v1 + dummy_config: { a: a dynamic dummy config } + last_updated: { seconds: 1234567891, nanos: 567000000 } +)EOF", + expected_config_dump); + EXPECT_EQ(expected_config_dump.DebugString(), dummy_config_dump3.DebugString()); +} + +// Tests that dynamic config providers enforce that the context's localInfo is +// set, since it is used to obtain the node/cluster attributes required for +// subscriptions. +TEST_F(ConfigProviderImplTest, LocalInfoNotDefined) { + factory_context_.local_info_.node_.set_cluster(""); + factory_context_.local_info_.node_.set_id(""); + + envoy::api::v2::core::ApiConfigSource config_source_proto; + config_source_proto.set_api_type(envoy::api::v2::core::ApiConfigSource::GRPC); + EXPECT_THROW_WITH_MESSAGE( + provider_manager_->createXdsConfigProvider(config_source_proto, factory_context_, + "dummy_prefix"), + EnvoyException, + "DummyDS: node 'id' and 'cluster' are required. Set it either in 'node' config or " + "via --service-node and --service-cluster options."); +} + +} // namespace +} // namespace Config +} // namespace Envoy diff --git a/test/common/config/dummy_config.proto b/test/common/config/dummy_config.proto new file mode 100644 index 0000000000000..fcb2749e4f036 --- /dev/null +++ b/test/common/config/dummy_config.proto @@ -0,0 +1,27 @@ +// Provides protos for testing source/common/config/config_provider_impl.{h,cc}. + +syntax = "proto3"; + +package test.common.config; + +import "google/protobuf/timestamp.proto"; + +message DummyConfig { + string a = 1; +} + +message DummyConfigsDump { + message StaticConfigs { + DummyConfig dummy_config = 1; + google.protobuf.Timestamp last_updated = 2; + } + + message DynamicConfigs { + string version_info = 1; + DummyConfig dummy_config = 2; + google.protobuf.Timestamp last_updated = 3; + } + + repeated StaticConfigs static_dummy_configs = 1; + repeated DynamicConfigs dynamic_dummy_configs = 2; +}