diff --git a/include/envoy/config/BUILD b/include/envoy/config/BUILD index 8c16805bf754f..777cf7a32fb25 100644 --- a/include/envoy/config/BUILD +++ b/include/envoy/config/BUILD @@ -14,6 +14,7 @@ envoy_cc_library( external_deps = ["abseil_optional"], deps = [ "//include/envoy/common:time_interface", + "//source/common/common:assert_lib", "//source/common/protobuf", ], ) diff --git a/include/envoy/config/config_provider.h b/include/envoy/config/config_provider.h index a42f512dea4e9..8a8fbecb2efd1 100644 --- a/include/envoy/config/config_provider.h +++ b/include/envoy/config/config_provider.h @@ -4,6 +4,7 @@ #include "envoy/common/time.h" +#include "common/common/assert.h" #include "common/protobuf/protobuf.h" #include "absl/types/optional.h" @@ -41,6 +42,23 @@ class ConfigProvider { }; using ConfigConstSharedPtr = std::shared_ptr; + /** + * The type of API represented by a ConfigProvider. + */ + enum class ApiType { + /** + * A "Full" API delivers a complete configuration as part of each resource (top level + * config proto); i.e., each resource contains the whole representation of the config intent. An + * example of this type of API is RDS. + */ + Full, + /** + * A "Delta" API delivers a subset of the config intent as part of each resource (top level + * config proto). Examples of this type of API are CDS, LDS and SRDS. + */ + Delta + }; + /** * Stores the config proto as well as the associated version. */ @@ -51,10 +69,26 @@ class ConfigProvider { std::string version_; }; + using ConfigProtoVector = std::vector; + /** + * Stores the config protos associated with a "Delta" API. + */ + template struct ConfigProtoInfoVector { + const std::vector config_protos_; + + // Only populated by dynamic config providers. + std::string version_; + }; + virtual ~ConfigProvider() = default; /** - * Returns a ConfigProtoInfo associated with the provider. + * The type of API. + */ + virtual ApiType apiType() const PURE; + + /** + * Returns a ConfigProtoInfo associated with a ApiType::Full provider. * @return absl::optional> an optional ConfigProtoInfo; the value is set when a * config is available. */ @@ -69,6 +103,27 @@ class ConfigProvider { return ConfigProtoInfo

{*config_proto, getConfigVersion()}; } + /** + * Returns a ConfigProtoInfoVector associated with a ApiType::Delta provider. + * @return absl::optional an optional ConfigProtoInfoVector; the value is + * set when a config is available. + */ + template absl::optional> configProtoInfoVector() const { + static_assert(std::is_base_of::value, + "Proto type must derive from Protobuf::Message"); + + const ConfigProtoVector config_protos = getConfigProtos(); + if (config_protos.empty()) { + return absl::nullopt; + } + std::vector ret_protos; + ret_protos.reserve(config_protos.size()); + for (const auto* elem : config_protos) { + ret_protos.push_back(static_cast(elem)); + } + return ConfigProtoInfoVector

{ret_protos, getConfigVersion()}; + } + /** * Returns the Config corresponding to the provider. * @return std::shared_ptr a shared pointer to the Config. @@ -92,13 +147,20 @@ class ConfigProvider { * @return Protobuf::Message* the config proto corresponding to the Config instantiated by the * provider. */ - virtual const Protobuf::Message* getConfigProto() const PURE; + virtual const Protobuf::Message* getConfigProto() const { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } + + /** + * Returns the config protos associated with the provider. + * @return const ConfigProtoVector the config protos corresponding to the Config instantiated by + * the provider. + */ + virtual ConfigProtoVector getConfigProtos() const { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } /** * Returns the config version associated with the provider. * @return std::string the config version. */ - virtual std::string getConfigVersion() const PURE; + virtual std::string getConfigVersion() const { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } /** * Returns the config implementation associated with the provider. diff --git a/include/envoy/config/config_provider_manager.h b/include/envoy/config/config_provider_manager.h index 3c3eddba9c2f0..81beb1ecb5fbb 100644 --- a/include/envoy/config/config_provider_manager.h +++ b/include/envoy/config/config_provider_manager.h @@ -25,6 +25,17 @@ namespace Config { */ class ConfigProviderManager { public: + class OptionalArg { + public: + virtual ~OptionalArg() = default; + }; + + class NullOptionalArg : public OptionalArg { + public: + NullOptionalArg() = default; + ~NullOptionalArg() override = default; + }; + virtual ~ConfigProviderManager() = default; /** @@ -34,6 +45,7 @@ class ConfigProviderManager { * @param config_source_proto supplies the proto containing the xDS API configuration. * @param factory_context is the context to use for the provider. * @param stat_prefix supplies the prefix to use for statistics. + * @param optarg supplies an optional argument with data specific to the concrete class. * @return ConfigProviderPtr a newly allocated dynamic config provider which shares underlying * data structures with other dynamic providers configured with the same * API source. @@ -41,17 +53,42 @@ class ConfigProviderManager { virtual ConfigProviderPtr createXdsConfigProvider(const Protobuf::Message& config_source_proto, Server::Configuration::FactoryContext& factory_context, - const std::string& stat_prefix) PURE; + const std::string& stat_prefix, const OptionalArg& optarg) PURE; /** * Returns a ConfigProvider associated with a statically specified configuration. * @param config_proto supplies the configuration proto. * @param factory_context is the context to use for the provider. + * @param optarg supplies an optional argument with data specific to the concrete class. * @return ConfigProviderPtr a newly allocated static config provider. */ virtual ConfigProviderPtr createStaticConfigProvider(const Protobuf::Message& config_proto, - Server::Configuration::FactoryContext& factory_context) PURE; + Server::Configuration::FactoryContext& factory_context, + const OptionalArg& optarg) { + UNREFERENCED_PARAMETER(config_proto); + UNREFERENCED_PARAMETER(factory_context); + UNREFERENCED_PARAMETER(optarg); + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; + } + + /** + * Returns a ConfigProvider associated with a statically specified configuration. This is intended + * to be used when a set of configuration protos is required to build the full configuration. + * @param config_protos supplies a vector of configuration protos. + * @param factory_context is the context to use for the provider. + * @param optarg supplies an optional argument with data specific to the concrete class. + * @return ConfigProviderPtr a newly allocated static config provider. + */ + virtual ConfigProviderPtr + createStaticConfigProvider(std::vector>&& config_protos, + Server::Configuration::FactoryContext& factory_context, + const OptionalArg& optarg) { + UNREFERENCED_PARAMETER(config_protos); + UNREFERENCED_PARAMETER(factory_context); + UNREFERENCED_PARAMETER(optarg); + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; + } }; } // namespace Config diff --git a/source/common/config/config_provider_impl.cc b/source/common/config/config_provider_impl.cc index da3d65043a96d..f13058d631dfa 100644 --- a/source/common/config/config_provider_impl.cc +++ b/source/common/config/config_provider_impl.cc @@ -3,29 +3,51 @@ namespace Envoy { namespace Config { -ImmutableConfigProviderImplBase::ImmutableConfigProviderImplBase( +ImmutableConfigProviderBase::ImmutableConfigProviderBase( Server::Configuration::FactoryContext& factory_context, - ConfigProviderManagerImplBase& config_provider_manager, ConfigProviderInstanceType type) + ConfigProviderManagerImplBase& config_provider_manager, + ConfigProviderInstanceType instance_type, ApiType api_type) : last_updated_(factory_context.timeSource().systemTime()), - config_provider_manager_(config_provider_manager), type_(type) { + config_provider_manager_(config_provider_manager), instance_type_(instance_type), + api_type_(api_type) { config_provider_manager_.bindImmutableConfigProvider(this); } -ImmutableConfigProviderImplBase::~ImmutableConfigProviderImplBase() { +ImmutableConfigProviderBase::~ImmutableConfigProviderBase() { config_provider_manager_.unbindImmutableConfigProvider(this); } -ConfigSubscriptionInstanceBase::~ConfigSubscriptionInstanceBase() { +ConfigSubscriptionCommonBase::~ConfigSubscriptionCommonBase() { init_target_.ready(); config_provider_manager_.unbindSubscription(manager_identifier_); } -bool ConfigSubscriptionInstanceBase::checkAndApplyConfig(const Protobuf::Message& config_proto, - const std::string& config_name, - const std::string& version_info) { +void ConfigSubscriptionCommonBase::bindConfigProvider(MutableConfigProviderCommonBase* provider) { + // All config providers bound to a ConfigSubscriptionCommonBase must be of the same concrete + // type; this is assumed by ConfigSubscriptionInstance::checkAndApplyConfigUpdate() and is + // verified by the assertion below. NOTE: an inlined statement ASSERT() triggers a potentially + // evaluated expression warning from clang due to `typeid(**mutable_config_providers_.begin())`. + // To avoid this, we use a lambda to separate the first mutable provider dereference from the + // typeid() statement. + ASSERT([&]() { + if (!mutable_config_providers_.empty()) { + const auto& first_provider = **mutable_config_providers_.begin(); + return typeid(*provider) == typeid(first_provider); + } + return true; + }()); + mutable_config_providers_.insert(provider); +} + +bool ConfigSubscriptionInstance::checkAndApplyConfigUpdate(const Protobuf::Message& config_proto, + const std::string& config_name, + const std::string& version_info) { const uint64_t new_hash = MessageUtil::hash(config_proto); - if (config_info_ && config_info_.value().last_config_hash_ == new_hash) { - return false; + if (config_info_) { + ASSERT(config_info_.value().last_config_hash_.has_value()); + if (config_info_.value().last_config_hash_.value() == new_hash) { + return false; + } } config_info_ = {new_hash, version_info}; @@ -39,31 +61,39 @@ bool ConfigSubscriptionInstanceBase::checkAndApplyConfig(const Protobuf::Message // bindConfigProvider()). // This makes it safe to call any of the provider's onConfigProtoUpdate() to get a new config // impl, which can then be passed to all providers. + auto* typed_provider = static_cast(provider); if (new_config == nullptr) { - if ((new_config = provider->onConfigProtoUpdate(config_proto)) == nullptr) { + if ((new_config = typed_provider->onConfigProtoUpdate(config_proto)) == nullptr) { return false; } } - provider->onConfigUpdate(new_config); + typed_provider->onConfigUpdate(new_config); } return true; } -void ConfigSubscriptionInstanceBase::bindConfigProvider(MutableConfigProviderImplBase* provider) { - // All config providers bound to a ConfigSubscriptionInstanceBase must be of the same concrete - // type; this is assumed by checkAndApplyConfig() and is verified by the assertion below. - // NOTE: an inlined statement ASSERT() triggers a potentially evaluated expression warning from - // clang due to `typeid(**mutable_config_providers_.begin())`. To avoid this, we use a lambda to - // separate the first mutable provider dereference from the typeid() statement. - ASSERT([&]() { - if (!mutable_config_providers_.empty()) { - const auto& first_provider = **mutable_config_providers_.begin(); - return typeid(*provider) == typeid(first_provider); - } - return true; - }()); - mutable_config_providers_.insert(provider); +void DeltaConfigSubscriptionInstance::applyDeltaConfigUpdate( + const std::function& update_fn) { + // The Config implementation is assumed to be shared across the config providers bound to this + // subscription, therefore, simply propagating the update to all worker threads for a single bound + // provider will be sufficient. + if (mutable_config_providers_.size() > 1) { + ASSERT(static_cast(*mutable_config_providers_.begin()) + ->getConfig() == static_cast( + *std::next(mutable_config_providers_.begin())) + ->getConfig()); + } + + // TODO(AndresGuedez): currently, the caller has to compute the differences in resources between + // DS API config updates and passes a granular update_fn() that adds/modifies/removes resources as + // needed. Such logic could be generalized as part of this framework such that this function owns + // the diffing and issues the corresponding call to add/modify/remove a resource according to a + // vector of functions passed by the caller. + auto* typed_provider = + static_cast(getAnyBoundMutableConfigProvider()); + ConfigSharedPtr config = typed_provider->getConfig(); + typed_provider->onConfigUpdate([config, update_fn]() { update_fn(config); }); } ConfigProviderManagerImplBase::ConfigProviderManagerImplBase(Server::Admin& admin, @@ -87,14 +117,14 @@ ConfigProviderManagerImplBase::immutableConfigProviders(ConfigProviderInstanceTy } void ConfigProviderManagerImplBase::bindImmutableConfigProvider( - ImmutableConfigProviderImplBase* provider) { - ASSERT(provider->type() == ConfigProviderInstanceType::Static || - provider->type() == ConfigProviderInstanceType::Inline); + ImmutableConfigProviderBase* provider) { + ASSERT(provider->instanceType() == ConfigProviderInstanceType::Static || + provider->instanceType() == ConfigProviderInstanceType::Inline); ConfigProviderMap::iterator it; - if ((it = immutable_config_providers_map_.find(provider->type())) == + if ((it = immutable_config_providers_map_.find(provider->instanceType())) == immutable_config_providers_map_.end()) { immutable_config_providers_map_.insert(std::make_pair( - provider->type(), + provider->instanceType(), std::make_unique(std::initializer_list({provider})))); } else { it->second->insert(provider); @@ -102,10 +132,10 @@ void ConfigProviderManagerImplBase::bindImmutableConfigProvider( } void ConfigProviderManagerImplBase::unbindImmutableConfigProvider( - ImmutableConfigProviderImplBase* provider) { - ASSERT(provider->type() == ConfigProviderInstanceType::Static || - provider->type() == ConfigProviderInstanceType::Inline); - auto it = immutable_config_providers_map_.find(provider->type()); + ImmutableConfigProviderBase* provider) { + ASSERT(provider->instanceType() == ConfigProviderInstanceType::Static || + provider->instanceType() == ConfigProviderInstanceType::Inline); + auto it = immutable_config_providers_map_.find(provider->instanceType()); ASSERT(it != immutable_config_providers_map_.end()); it->second->erase(provider); } diff --git a/source/common/config/config_provider_impl.h b/source/common/config/config_provider_impl.h index 50e916d1d0348..ddcd1232caa18 100644 --- a/source/common/config/config_provider_impl.h +++ b/source/common/config/config_provider_impl.h @@ -19,14 +19,15 @@ namespace Envoy { namespace Config { -// This file provides a set of base classes, (ImmutableConfigProviderImplBase, -// MutableConfigProviderImplBase, ConfigProviderManagerImplBase, ConfigSubscriptionInstanceBase), -// conforming to the ConfigProvider/ConfigProviderManager interfaces, which in tandem provide a -// framework for implementing statically defined (i.e., immutable) and dynamic (mutable via -// subscriptions) configuration for Envoy. +// This file provides a set of base classes, (ImmutableConfigProviderBase, +// MutableConfigProviderCommonBase, MutableConfigProviderBase, DeltaMutableConfigProviderBase, +// ConfigProviderManagerImplBase, ConfigSubscriptionCommonBase, ConfigSubscriptionInstance, +// DeltaConfigSubscriptionInstance), conforming to the ConfigProvider/ConfigProviderManager +// interfaces, which in tandem provide a framework for implementing statically defined (i.e., +// immutable) and dynamic (mutable via subscriptions) configuration for Envoy. // // The mutability property applies to the ConfigProvider itself and _not_ the underlying config -// proto, which is always immutable. MutableConfigProviderImplBase objects receive config proto +// proto, which is always immutable. MutableConfigProviderCommonBase objects receive config proto // updates via xDS subscriptions, resulting in new ConfigProvider::Config objects being instantiated // with the corresponding change in behavior corresponding to updated config. ConfigProvider::Config // objects must be latched/associated with the appropriate objects in the connection and request @@ -49,27 +50,36 @@ namespace Config { // 1) Create a class derived from ConfigProviderManagerImplBase and implement the required // interface. // When implementing createXdsConfigProvider(), it is expected that getSubscription() will -// be called to fetch either an existing ConfigSubscriptionInstanceBase if the config source -// configuration matches, or a newly instantiated subscription otherwise. +// be called to fetch either an existing ConfigSubscriptionCommonBase if the config +// source configuration matches, or a newly instantiated subscription otherwise. // // For immutable providers: -// 1) Create a class derived from ImmutableConfigProviderImplBase and implement the required +// 1) Create a class derived from ImmutableConfigProviderBase and implement the required // interface. // // For mutable (xDS) providers: -// 1) Create a class derived from MutableConfigProviderImplBase and implement the required -// interface. -// 2) Create a class derived from ConfigSubscriptionInstanceBase; this is the entity -// responsible for owning and managing the Envoy::Config::Subscription that provides -// the underlying config subscription. +// 1) According to the API type, create a class derived from MutableConfigProviderBase or +// DeltaMutableConfigProviderBase and implement the required interface. +// 2) According to the API type, create a class derived from ConfigSubscriptionInstance or +// DeltaConfigSubscriptionInstance; this is the entity responsible for owning and managing the +// Envoy::Config::Subscription that provides the underlying config subscription. +// a) For a ConfigProvider::ApiType::Full subscription instance (i.e., a +// ConfigSubscriptionInstance child): // - When subscription callbacks (onConfigUpdate, onConfigUpdateFailed) are issued by the -// underlying subscription, the corresponding ConfigSubscriptionInstanceBase functions must be -// called as well. -// - On a successful config update, checkAndApplyConfig() should be called to instantiate the -// new config implementation and propagate it to the shared config providers and all -// worker threads. -// - On a successful return from checkAndApplyConfig(), the config proto must be latched into -// this class and returned via the getConfigProto() override. +// underlying subscription, the corresponding ConfigSubscriptionInstance functions +// must be called as well. +// - On a successful config update, checkAndApplyConfigUpdate() should be called to instantiate +// the new config implementation and propagate it to the shared config providers and all worker +// threads. +// - On a successful return from checkAndApplyConfigUpdate(), the config proto must be latched +// into this class and returned via the getConfigProto() override. +// b) For a ConfigProvider::ApiType::Delta subscription instance (i.e., a +// DeltaConfigSubscriptionInstance child): +// - When subscription callbacks (onConfigUpdate, onConfigUpdateFailed) are issued by the +// underlying subscription, the corresponding ConfigSubscriptionInstance functions must be called +// as well. +// - On a successful config update, applyConfigUpdate() should be called to propagate the config +// updates to all bound config providers and worker threads. class ConfigProviderManagerImplBase; @@ -90,38 +100,40 @@ enum class ConfigProviderInstanceType { * ConfigProvider implementation for immutable configuration. * * TODO(AndresGuedez): support sharing of config protos and config impls, as is - * done with the MutableConfigProviderImplBase. + * done with the MutableConfigProviderCommonBase. * * This class can not be instantiated directly; instead, it provides the foundation for * immutable config provider implementations which derive from it. */ -class ImmutableConfigProviderImplBase : public ConfigProvider { +class ImmutableConfigProviderBase : public ConfigProvider { public: - ~ImmutableConfigProviderImplBase() override; + ~ImmutableConfigProviderBase() override; // Envoy::Config::ConfigProvider SystemTime lastUpdated() const override { return last_updated_; } + ApiType apiType() const override { return api_type_; } - ConfigProviderInstanceType type() const { return type_; } + ConfigProviderInstanceType instanceType() const { return instance_type_; } protected: - ImmutableConfigProviderImplBase(Server::Configuration::FactoryContext& factory_context, - ConfigProviderManagerImplBase& config_provider_manager, - ConfigProviderInstanceType type); + ImmutableConfigProviderBase(Server::Configuration::FactoryContext& factory_context, + ConfigProviderManagerImplBase& config_provider_manager, + ConfigProviderInstanceType instance_type, ApiType api_type); private: SystemTime last_updated_; ConfigProviderManagerImplBase& config_provider_manager_; - ConfigProviderInstanceType type_; + ConfigProviderInstanceType instance_type_; + ApiType api_type_; }; -class MutableConfigProviderImplBase; +class MutableConfigProviderCommonBase; /** - * Provides generic functionality required by all xDS ConfigProvider subscriptions, including - * shared lifetime management via shared_ptr. + * Provides common DS API subscription functionality required by the ConfigProvider::ApiType + * specific base classes (see ConfigSubscriptionInstance and DeltaConfigSubscriptionInstance). * - * To do so, this class keeps track of a set of MutableConfigProviderImplBase instances associated + * To do so, this class keeps track of a set of MutableConfigProviderCommonBase instances associated * with an underlying subscription; providers are bound/unbound as needed as they are created and * destroyed. * @@ -134,14 +146,14 @@ class MutableConfigProviderImplBase; * This class can not be instantiated directly; instead, it provides the foundation for * config subscription implementations which derive from it. */ -class ConfigSubscriptionInstanceBase : protected Logger::Loggable { +class ConfigSubscriptionCommonBase : protected Logger::Loggable { public: struct LastConfigInfo { - uint64_t last_config_hash_; + absl::optional last_config_hash_; std::string last_config_version_; }; - virtual ~ConfigSubscriptionInstanceBase(); + virtual ~ConfigSubscriptionCommonBase(); /** * Starts the subscription corresponding to a config source. @@ -167,35 +179,27 @@ class ConfigSubscriptionInstanceBase : protected Logger::Loggable&& config_info) { + config_info_ = std::move(config_info); + } + + const std::string name_; + std::unordered_set mutable_config_providers_; + absl::optional config_info_; + private: - void bindConfigProvider(MutableConfigProviderImplBase* provider); + void bindConfigProvider(MutableConfigProviderCommonBase* provider); - void unbindConfigProvider(MutableConfigProviderImplBase* provider) { + void unbindConfigProvider(MutableConfigProviderCommonBase* provider) { mutable_config_providers_.erase(provider); } - const std::string name_; Init::TargetImpl init_target_; - std::unordered_set mutable_config_providers_; const uint64_t manager_identifier_; ConfigProviderManagerImplBase& config_provider_manager_; TimeSource& time_source_; SystemTime last_updated_; - absl::optional config_info_; - // ConfigSubscriptionInstanceBase, MutableConfigProviderImplBase and ConfigProviderManagerImplBase - // are tightly coupled with the current shared ownership model; use friend classes to explicitly - // denote the binding between them. + // ConfigSubscriptionCommonBase, MutableConfigProviderCommonBase and + // ConfigProviderManagerImplBase are tightly coupled with the current shared ownership model; use + // friend classes to explicitly denote the binding between them. // // TODO(AndresGuedez): Investigate whether a shared ownership model avoiding the s and // instead centralizing lifetime management in the ConfigProviderManagerImplBase with explicit // reference counting would be more maintainable. - friend class MutableConfigProviderImplBase; + friend class MutableConfigProviderCommonBase; + friend class MutableConfigProviderBase; + friend class DeltaMutableConfigProviderBase; friend class ConfigProviderManagerImplBase; + friend class MockMutableConfigProviderBase; }; -using ConfigSubscriptionInstanceBaseSharedPtr = std::shared_ptr; +using ConfigSubscriptionCommonBaseSharedPtr = std::shared_ptr; /** - * Provides generic functionality required by all dynamic config providers, including distribution - * of config updates to all workers. + * Provides common subscription functionality required by ConfigProvider::ApiType::Full DS APIs. + */ +class ConfigSubscriptionInstance : public ConfigSubscriptionCommonBase { +protected: + ConfigSubscriptionInstance(const std::string& name, const uint64_t manager_identifier, + ConfigProviderManagerImplBase& config_provider_manager, + TimeSource& time_source, const SystemTime& last_updated, + const LocalInfo::LocalInfo& local_info) + : ConfigSubscriptionCommonBase(name, manager_identifier, config_provider_manager, time_source, + last_updated, local_info) {} + + ~ConfigSubscriptionInstance() override = default; + + /** + * Determines whether a configuration proto is a new update, and if so, propagates it to all + * config providers associated with this subscription. + * @param config_proto supplies the newly received config proto. + * @param config_name supplies the name associated with the config. + * @param version_info supplies the version associated with the config. + * @return bool false when the config proto has no delta from the previous config, true otherwise. + */ + bool checkAndApplyConfigUpdate(const Protobuf::Message& config_proto, + const std::string& config_name, const std::string& version_info); +}; + +using ConfigSharedPtr = std::shared_ptr; + +/** + * Provides common subscription functionality required by ConfigProvider::ApiType::Delta DS APIs. + */ +class DeltaConfigSubscriptionInstance : public ConfigSubscriptionCommonBase { +protected: + DeltaConfigSubscriptionInstance(const std::string& name, const uint64_t manager_identifier, + ConfigProviderManagerImplBase& config_provider_manager, + TimeSource& time_source, const SystemTime& last_updated, + const LocalInfo::LocalInfo& local_info) + : ConfigSubscriptionCommonBase(name, manager_identifier, config_provider_manager, time_source, + last_updated, local_info) {} + + ~DeltaConfigSubscriptionInstance() override = default; + + /** + * Propagates a config update to the config providers and worker threads associated with the + * subscription. + * + * @param update_fn the callback to run on each worker thread. + */ + void applyDeltaConfigUpdate(const std::function& update_fn); +}; + +/** + * Provides generic functionality required by the ConfigProvider::ApiType specific dynamic config + * providers (see MutableConfigProviderBase and DeltaMutableConfigProviderBase). * * This class can not be instantiated directly; instead, it provides the foundation for * dynamic config provider implementations which derive from it. */ -class MutableConfigProviderImplBase : public ConfigProvider { +class MutableConfigProviderCommonBase : public ConfigProvider { public: - ~MutableConfigProviderImplBase() override { subscription_->unbindConfigProvider(this); } + ~MutableConfigProviderCommonBase() override { subscription_->unbindConfigProvider(this); } // Envoy::Config::ConfigProvider SystemTime lastUpdated() const override { return subscription_->lastUpdated(); } + ApiType apiType() const override { return api_type_; } + +protected: + MutableConfigProviderCommonBase(ConfigSubscriptionCommonBaseSharedPtr&& subscription, + Server::Configuration::FactoryContext& factory_context, + ApiType api_type) + : tls_(factory_context.threadLocal().allocateSlot()), subscription_(subscription), + api_type_(api_type) {} + + ThreadLocal::SlotPtr tls_; + ConfigSubscriptionCommonBaseSharedPtr subscription_; +private: + ApiType api_type_; +}; + +/** + * Provides common mutable (dynamic) config provider functionality required by + * ConfigProvider::ApiType::Full DS APIs. + */ +class MutableConfigProviderBase : public MutableConfigProviderCommonBase { +public: // Envoy::Config::ConfigProvider + // NOTE: This is being promoted to public for internal uses to avoid an unnecessary dynamic_cast + // in the public API (ConfigProvider::config()). ConfigConstSharedPtr getConfig() const override { return tls_->getTyped().config_; } @@ -280,16 +366,20 @@ class MutableConfigProviderImplBase : public ConfigProvider { * @param config supplies the newly instantiated config. */ void onConfigUpdate(const ConfigConstSharedPtr& config) { + if (getConfig() == config) { + return; + } tls_->runOnAllThreads( [this, config]() -> void { tls_->getTyped().config_ = config; }); } protected: - MutableConfigProviderImplBase(ConfigSubscriptionInstanceBaseSharedPtr&& subscription, - Server::Configuration::FactoryContext& factory_context) - : subscription_(subscription), tls_(factory_context.threadLocal().allocateSlot()) {} + MutableConfigProviderBase(ConfigSubscriptionCommonBaseSharedPtr&& subscription, + Server::Configuration::FactoryContext& factory_context, + ApiType api_type) + : MutableConfigProviderCommonBase(std::move(subscription), factory_context, api_type) {} - const ConfigSubscriptionInstanceBaseSharedPtr& subscription() const { return subscription_; } + ~MutableConfigProviderBase() override = default; private: struct ThreadLocalConfig : public ThreadLocal::ThreadLocalObject { @@ -298,9 +388,50 @@ class MutableConfigProviderImplBase : public ConfigProvider { ConfigProvider::ConfigConstSharedPtr config_; }; +}; - ConfigSubscriptionInstanceBaseSharedPtr subscription_; - ThreadLocal::SlotPtr tls_; +/** + * Provides common mutable (dynamic) config provider functionality required by + * ConfigProvider::ApiType::Delta DS APIs. + */ +class DeltaMutableConfigProviderBase : public MutableConfigProviderCommonBase { +public: + // Envoy::Config::ConfigProvider + // This promotes getConfig() to public so that internal uses can avoid an unnecessary dynamic_cast + // in the public API (ConfigProvider::config()). + ConfigConstSharedPtr getConfig() const override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } + + /** + * Non-const overload for use within the framework. + * @return ConfigSharedPtr the config implementation associated with the provider. + */ + virtual ConfigSharedPtr getConfig() PURE; + + /** + * Propagates a delta config update to all workers. + * @param updateCb the callback to run on each worker. + */ + void onConfigUpdate(Envoy::Event::PostCb update_cb) { + tls_->runOnAllThreads(std::move(update_cb)); + } + +protected: + DeltaMutableConfigProviderBase(ConfigSubscriptionCommonBaseSharedPtr&& subscription, + Server::Configuration::FactoryContext& factory_context, + ApiType api_type) + : MutableConfigProviderCommonBase(std::move(subscription), factory_context, api_type) {} + + ~DeltaMutableConfigProviderBase() override = default; + + /** + * Must be called by the derived class' constructor. + * @param initializeCb supplies the initialization callback to be issued for each worker + * thread. + */ + void initialize(ThreadLocal::Slot::InitializeCb initializeCb) { + subscription_->bindConfigProvider(this); + tls_->set(std::move(initializeCb)); + } }; /** @@ -308,9 +439,9 @@ class MutableConfigProviderImplBase : public ConfigProvider { * lifetime of subscriptions and dynamic config providers, along with determining which * subscriptions should be associated with newly instantiated providers. * - * The implementation of this class is not thread safe. Note that ImmutableConfigProviderImplBase - * and ConfigSubscriptionInstanceBase call the corresponding {bind,unbind}* functions exposed by - * this class. + * The implementation of this class is not thread safe. Note that ImmutableConfigProviderBase + * and ConfigSubscriptionCommonBase call the corresponding {bind,unbind}* functions exposed + * by this class. * * All config processing is done on the main thread, so instantiation of *ConfigProvider* objects * via createStaticConfigProvider() and createXdsConfigProvider() is naturally thread safe. Care @@ -333,18 +464,19 @@ class ConfigProviderManagerImplBase : public ConfigProviderManager, public Singl virtual ProtobufTypes::MessagePtr dumpConfigs() const PURE; protected: - using ConfigProviderSet = std::unordered_set; + // Ordered set for deterministic config dump output. + using ConfigProviderSet = std::set; using ConfigProviderMap = std::unordered_map, EnumClassHash>; using ConfigSubscriptionMap = - std::unordered_map>; + std::unordered_map>; ConfigProviderManagerImplBase(Server::Admin& admin, const std::string& config_name); const ConfigSubscriptionMap& configSubscriptions() const { return config_subscriptions_; } /** - * Returns the set of bound ImmutableConfigProviderImplBase-derived providers of a given type. + * Returns the set of bound ImmutableConfigProviderBase-derived providers of a given type. * @param type supplies the type of config providers to return. * @return const ConfigProviderSet* the set of config providers corresponding to the type. */ @@ -362,12 +494,12 @@ class ConfigProviderManagerImplBase : public ConfigProviderManager, public Singl template std::shared_ptr getSubscription(const Protobuf::Message& config_source_proto, Init::Manager& init_manager, - const std::function& subscription_factory_fn) { - static_assert(std::is_base_of::value, - "T must be a subclass of ConfigSubscriptionInstanceBase"); + static_assert(std::is_base_of::value, + "T must be a subclass of ConfigSubscriptionCommonBase"); - ConfigSubscriptionInstanceBaseSharedPtr subscription; + ConfigSubscriptionCommonBaseSharedPtr subscription; const uint64_t manager_identifier = MessageUtil::hash(config_source_proto); auto it = config_subscriptions_.find(manager_identifier); @@ -381,7 +513,7 @@ class ConfigProviderManagerImplBase : public ConfigProviderManager, public Singl bindSubscription(manager_identifier, subscription); } else { // Because the ConfigProviderManagerImplBase's weak_ptrs only get cleaned up - // in the ConfigSubscriptionInstanceBase destructor, and the single threaded nature + // in the ConfigSubscriptionCommonBase destructor, and the single threaded nature // of this code, locking the weak_ptr will not fail. subscription = it->second.lock(); } @@ -392,7 +524,7 @@ class ConfigProviderManagerImplBase : public ConfigProviderManager, public Singl private: void bindSubscription(const uint64_t manager_identifier, - ConfigSubscriptionInstanceBaseSharedPtr& subscription) { + ConfigSubscriptionCommonBaseSharedPtr& subscription) { config_subscriptions_.insert({manager_identifier, subscription}); } @@ -400,8 +532,8 @@ class ConfigProviderManagerImplBase : public ConfigProviderManager, public Singl config_subscriptions_.erase(manager_identifier); } - void bindImmutableConfigProvider(ImmutableConfigProviderImplBase* provider); - void unbindImmutableConfigProvider(ImmutableConfigProviderImplBase* provider); + void bindImmutableConfigProvider(ImmutableConfigProviderBase* provider); + void unbindImmutableConfigProvider(ImmutableConfigProviderBase* provider); // TODO(jsedgwick) These two members are prime candidates for the owned-entry list/map // as in ConfigTracker. I.e. the ProviderImpls would have an EntryOwner for these lists @@ -411,10 +543,10 @@ class ConfigProviderManagerImplBase : public ConfigProviderManager, public Singl Server::ConfigTracker::EntryOwnerPtr config_tracker_entry_; - // See comment for friend classes in the ConfigSubscriptionInstanceBase for more details on the - // use of friends. - friend class ConfigSubscriptionInstanceBase; - friend class ImmutableConfigProviderImplBase; + // See comment for friend classes in the ConfigSubscriptionCommonBase for more details on + // the use of friends. + friend class ConfigSubscriptionCommonBase; + friend class ImmutableConfigProviderBase; }; } // namespace Config diff --git a/test/common/config/config_provider_impl_test.cc b/test/common/config/config_provider_impl_test.cc index 306c09c015a79..3945dc9d79ad1 100644 --- a/test/common/config/config_provider_impl_test.cc +++ b/test/common/config/config_provider_impl_test.cc @@ -4,8 +4,10 @@ #include "common/protobuf/utility.h" #include "test/common/config/dummy_config.pb.h" +#include "test/mocks/config/mocks.h" #include "test/mocks/server/mocks.h" #include "test/test_common/simulated_time_system.h" +#include "test/test_common/utility.h" #include "gmock/gmock.h" #include "gtest/gtest.h" @@ -14,9 +16,11 @@ namespace Envoy { namespace Config { namespace { +using testing::InSequence; + class DummyConfigProviderManager; -class StaticDummyConfigProvider : public ImmutableConfigProviderImplBase { +class StaticDummyConfigProvider : public ImmutableConfigProviderBase { public: StaticDummyConfigProvider(const test::common::config::DummyConfig& config_proto, Server::Configuration::FactoryContext& factory_context, @@ -38,7 +42,7 @@ class StaticDummyConfigProvider : public ImmutableConfigProviderImplBase { test::common::config::DummyConfig config_proto_; }; -class DummyConfigSubscription : public ConfigSubscriptionInstanceBase, +class DummyConfigSubscription : public ConfigSubscriptionInstance, Envoy::Config::SubscriptionCallbacks { public: DummyConfigSubscription(const uint64_t manager_identifier, @@ -47,7 +51,7 @@ class DummyConfigSubscription : public ConfigSubscriptionInstanceBase, ~DummyConfigSubscription() override = default; - // Envoy::Config::ConfigSubscriptionInstanceBase + // Envoy::Config::ConfigSubscriptionCommonBase void start() override {} // Envoy::Config::SubscriptionCallbacks @@ -55,11 +59,11 @@ class DummyConfigSubscription : public ConfigSubscriptionInstanceBase, void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, const std::string& version_info) override { auto config = MessageUtil::anyConvert(resources[0]); - if (checkAndApplyConfig(config, "dummy_config", version_info)) { + if (checkAndApplyConfigUpdate(config, "dummy_config", version_info)) { config_proto_ = config; } - ConfigSubscriptionInstanceBase::onConfigUpdate(); + ConfigSubscriptionCommonBase::onConfigUpdate(); } void onConfigUpdate(const Protobuf::RepeatedPtrField&, const Protobuf::RepeatedPtrField&, const std::string&) override { @@ -79,7 +83,6 @@ class DummyConfigSubscription : public ConfigSubscriptionInstanceBase, private: absl::optional config_proto_; }; - using DummyConfigSubscriptionSharedPtr = std::shared_ptr; class DummyConfig : public ConfigProvider::Config { @@ -87,14 +90,14 @@ class DummyConfig : public ConfigProvider::Config { DummyConfig(const test::common::config::DummyConfig&) {} }; -class DummyDynamicConfigProvider : public MutableConfigProviderImplBase { +class DummyDynamicConfigProvider : public MutableConfigProviderBase { public: DummyDynamicConfigProvider(DummyConfigSubscriptionSharedPtr&& subscription, - ConfigConstSharedPtr initial_config, + const ConfigConstSharedPtr& initial_config, Server::Configuration::FactoryContext& factory_context) - : MutableConfigProviderImplBase(std::move(subscription), factory_context), + : MutableConfigProviderBase(std::move(subscription), factory_context, ApiType::Full), subscription_(static_cast( - MutableConfigProviderImplBase::subscription().get())) { + MutableConfigProviderCommonBase::subscription_.get())) { initialize(initial_config); } @@ -102,7 +105,7 @@ class DummyDynamicConfigProvider : public MutableConfigProviderImplBase { DummyConfigSubscription& subscription() { return *subscription_; } - // Envoy::Config::MutableConfigProviderImplBase + // Envoy::Config::MutableConfigProviderBase ConfigProvider::ConfigConstSharedPtr onConfigProtoUpdate(const Protobuf::Message& config) override { return std::make_shared( @@ -116,8 +119,6 @@ class DummyDynamicConfigProvider : public MutableConfigProviderImplBase { } return &subscription_->config_proto().value(); } - - // Envoy::Config::ConfigProvider std::string getConfigVersion() const override { return ""; } private: @@ -162,22 +163,24 @@ class DummyConfigProviderManager : public ConfigProviderManagerImplBase { } // Envoy::Config::ConfigProviderManager - ConfigProviderPtr createXdsConfigProvider(const Protobuf::Message& config_source_proto, - Server::Configuration::FactoryContext& factory_context, - const std::string&) override { + ConfigProviderPtr + createXdsConfigProvider(const Protobuf::Message& config_source_proto, + Server::Configuration::FactoryContext& factory_context, + const std::string&, + const Envoy::Config::ConfigProviderManager::OptionalArg&) override { DummyConfigSubscriptionSharedPtr subscription = getSubscription( config_source_proto, factory_context.initManager(), [&factory_context](const uint64_t manager_identifier, ConfigProviderManagerImplBase& config_provider_manager) - -> ConfigSubscriptionInstanceBaseSharedPtr { + -> ConfigSubscriptionCommonBaseSharedPtr { return std::make_shared( manager_identifier, factory_context, static_cast(config_provider_manager)); }); ConfigProvider::ConfigConstSharedPtr initial_config; - const MutableConfigProviderImplBase* provider = - subscription->getAnyBoundMutableConfigProvider(); + const auto* provider = static_cast( + subscription->getAnyBoundMutableConfigProvider()); if (provider) { initial_config = provider->getConfig(); } @@ -188,31 +191,38 @@ class DummyConfigProviderManager : public ConfigProviderManagerImplBase { // Envoy::Config::ConfigProviderManager ConfigProviderPtr createStaticConfigProvider(const Protobuf::Message& config_proto, - Server::Configuration::FactoryContext& factory_context) override { + Server::Configuration::FactoryContext& factory_context, + const Envoy::Config::ConfigProviderManager::OptionalArg&) override { return std::make_unique( dynamic_cast(config_proto), factory_context, *this); } + ConfigProviderPtr + createStaticConfigProvider(std::vector>&&, + Server::Configuration::FactoryContext&, const OptionalArg&) override { + ASSERT(false, "this provider does not expect multiple config protos"); + return nullptr; + } }; StaticDummyConfigProvider::StaticDummyConfigProvider( const test::common::config::DummyConfig& config_proto, Server::Configuration::FactoryContext& factory_context, DummyConfigProviderManager& config_provider_manager) - : ImmutableConfigProviderImplBase(factory_context, config_provider_manager, - ConfigProviderInstanceType::Static), + : ImmutableConfigProviderBase(factory_context, config_provider_manager, + ConfigProviderInstanceType::Static, ApiType::Full), config_(std::make_shared(config_proto)), config_proto_(config_proto) {} DummyConfigSubscription::DummyConfigSubscription( const uint64_t manager_identifier, Server::Configuration::FactoryContext& factory_context, DummyConfigProviderManager& config_provider_manager) - : ConfigSubscriptionInstanceBase( + : ConfigSubscriptionInstance( "DummyDS", manager_identifier, config_provider_manager, factory_context.timeSource(), factory_context.timeSource().systemTime(), factory_context.localInfo()) {} class ConfigProviderImplTest : public testing::Test { public: - ConfigProviderImplTest() { + void initialize() { EXPECT_CALL(factory_context_.admin_.config_tracker_, add_("dummy", _)); provider_manager_ = std::make_unique(factory_context_.admin_); } @@ -235,13 +245,15 @@ test::common::config::DummyConfig parseDummyConfigFromYaml(const std::string& ya // subscriptions, config protos and data structures generated as a result of the // configurations (i.e., the ConfigProvider::Config). TEST_F(ConfigProviderImplTest, SharedOwnership) { + initialize(); Init::ExpectableWatcherImpl watcher; factory_context_.init_manager_.initialize(watcher); envoy::api::v2::core::ApiConfigSource config_source_proto; config_source_proto.set_api_type(envoy::api::v2::core::ApiConfigSource::GRPC); ConfigProviderPtr provider1 = provider_manager_->createXdsConfigProvider( - config_source_proto, factory_context_, "dummy_prefix"); + config_source_proto, factory_context_, "dummy_prefix", + ConfigProviderManager::NullOptionalArg()); // No config protos have been received via the subscription yet. EXPECT_FALSE(provider1->configProtoInfo().has_value()); @@ -256,7 +268,8 @@ TEST_F(ConfigProviderImplTest, SharedOwnership) { // Check that a newly created provider with the same config source will share // the subscription, config proto and resulting ConfigProvider::Config. ConfigProviderPtr provider2 = provider_manager_->createXdsConfigProvider( - config_source_proto, factory_context_, "dummy_prefix"); + config_source_proto, factory_context_, "dummy_prefix", + ConfigProviderManager::NullOptionalArg()); EXPECT_TRUE(provider2->configProtoInfo().has_value()); EXPECT_EQ(&dynamic_cast(*provider1).subscription(), @@ -269,7 +282,8 @@ TEST_F(ConfigProviderImplTest, SharedOwnership) { // Change the config source and verify that a new subscription is used. config_source_proto.set_api_type(envoy::api::v2::core::ApiConfigSource::REST); ConfigProviderPtr provider3 = provider_manager_->createXdsConfigProvider( - config_source_proto, factory_context_, "dummy_prefix"); + config_source_proto, factory_context_, "dummy_prefix", + ConfigProviderManager::NullOptionalArg()); EXPECT_NE(&dynamic_cast(*provider1).subscription(), &dynamic_cast(*provider3).subscription()); @@ -305,10 +319,61 @@ TEST_F(ConfigProviderImplTest, SharedOwnership) { .size()); } +// A ConfigProviderManager that returns a mock ConfigProvider. +class DummyConfigProviderManagerMockConfigProvider : public DummyConfigProviderManager { +public: + DummyConfigProviderManagerMockConfigProvider(Server::Admin& admin) + : DummyConfigProviderManager(admin) {} + + ConfigProviderPtr + createXdsConfigProvider(const Protobuf::Message& config_source_proto, + Server::Configuration::FactoryContext& factory_context, + const std::string&, + const Envoy::Config::ConfigProviderManager::OptionalArg&) override { + DummyConfigSubscriptionSharedPtr subscription = getSubscription( + config_source_proto, factory_context.initManager(), + [&factory_context](const uint64_t manager_identifier, + ConfigProviderManagerImplBase& config_provider_manager) + -> ConfigSubscriptionCommonBaseSharedPtr { + return std::make_shared( + manager_identifier, factory_context, + static_cast(config_provider_manager)); + }); + return std::make_unique(std::move(subscription), nullptr, + factory_context); + } +}; + +// Test that duplicate config updates will not trigger creation of a new ConfigProvider::Config. +TEST_F(ConfigProviderImplTest, DuplicateConfigProto) { + InSequence sequence; + // This provider manager returns a MockMutableConfigProviderBase. + auto provider_manager = + std::make_unique(factory_context_.admin_); + envoy::api::v2::core::ApiConfigSource config_source_proto; + config_source_proto.set_api_type(envoy::api::v2::core::ApiConfigSource::GRPC); + ConfigProviderPtr provider = provider_manager->createXdsConfigProvider( + config_source_proto, factory_context_, "dummy_prefix", + ConfigProviderManager::NullOptionalArg()); + auto* typed_provider = static_cast(provider.get()); + DummyConfigSubscription& subscription = + static_cast(typed_provider->subscription()); + // First time issuing a configUpdate(). A new ConfigProvider::Config should be created. + EXPECT_CALL(*typed_provider, onConfigProtoUpdate(_)).Times(1); + Protobuf::RepeatedPtrField untyped_dummy_configs; + untyped_dummy_configs.Add()->PackFrom(parseDummyConfigFromYaml("a: a dynamic dummy config")); + subscription.onConfigUpdate(untyped_dummy_configs, "1"); + // Second time issuing the configUpdate(), this time with a duplicate proto. A new + // ConfigProvider::Config _should not_ be created. + EXPECT_CALL(*typed_provider, onConfigProtoUpdate(_)).Times(0); + subscription.onConfigUpdate(untyped_dummy_configs, "1"); +} + // Tests that the base ConfigProvider*s are handling registration with the // /config_dump admin handler as well as generic bookkeeping such as timestamp // updates. TEST_F(ConfigProviderImplTest, ConfigDump) { + initialize(); // Empty dump first. auto message_ptr = factory_context_.admin_.config_tracker_.config_tracker_callbacks_["dummy"](); const auto& dummy_config_dump = @@ -327,7 +392,8 @@ TEST_F(ConfigProviderImplTest, ConfigDump) { timeSystem().setSystemTime(std::chrono::milliseconds(1234567891234)); ConfigProviderPtr static_config = provider_manager_->createStaticConfigProvider( - parseDummyConfigFromYaml(config_yaml), factory_context_); + parseDummyConfigFromYaml(config_yaml), factory_context_, + ConfigProviderManager::NullOptionalArg()); message_ptr = factory_context_.admin_.config_tracker_.config_tracker_callbacks_["dummy"](); const auto& dummy_config_dump2 = static_cast(*message_ptr); @@ -343,7 +409,8 @@ TEST_F(ConfigProviderImplTest, ConfigDump) { envoy::api::v2::core::ApiConfigSource config_source_proto; config_source_proto.set_api_type(envoy::api::v2::core::ApiConfigSource::GRPC); ConfigProviderPtr dynamic_provider = provider_manager_->createXdsConfigProvider( - config_source_proto, factory_context_, "dummy_prefix"); + config_source_proto, factory_context_, "dummy_prefix", + ConfigProviderManager::NullOptionalArg()); // Static + dynamic config dump. Protobuf::RepeatedPtrField untyped_dummy_configs; @@ -368,12 +435,33 @@ TEST_F(ConfigProviderImplTest, ConfigDump) { )EOF", expected_config_dump); EXPECT_EQ(expected_config_dump.DebugString(), dummy_config_dump3.DebugString()); + + ConfigProviderPtr static_config2 = provider_manager_->createStaticConfigProvider( + parseDummyConfigFromYaml("a: another static dummy config"), factory_context_, + ConfigProviderManager::NullOptionalArg()); + message_ptr = factory_context_.admin_.config_tracker_.config_tracker_callbacks_["dummy"](); + const auto& dummy_config_dump4 = + static_cast(*message_ptr); + MessageUtil::loadFromYaml(R"EOF( +static_dummy_configs: + - dummy_config: { a: another static dummy config } + last_updated: { seconds: 1234567891, nanos: 567000000 } + - dummy_config: { a: a static dummy config } + last_updated: { seconds: 1234567891, nanos: 234000000 } +dynamic_dummy_configs: + - version_info: v1 + dummy_config: { a: a dynamic dummy config } + last_updated: { seconds: 1234567891, nanos: 567000000 } +)EOF", + expected_config_dump); + EXPECT_THAT(expected_config_dump, ProtoEqIgnoreRepeatedFieldOrdering(dummy_config_dump4)); } // Tests that dynamic config providers enforce that the context's localInfo is // set, since it is used to obtain the node/cluster attributes required for // subscriptions. TEST_F(ConfigProviderImplTest, LocalInfoNotDefined) { + initialize(); factory_context_.local_info_.node_.set_cluster(""); factory_context_.local_info_.node_.set_id(""); @@ -381,12 +469,274 @@ TEST_F(ConfigProviderImplTest, LocalInfoNotDefined) { config_source_proto.set_api_type(envoy::api::v2::core::ApiConfigSource::GRPC); EXPECT_THROW_WITH_MESSAGE( provider_manager_->createXdsConfigProvider(config_source_proto, factory_context_, - "dummy_prefix"), + "dummy_prefix", + ConfigProviderManager::NullOptionalArg()), EnvoyException, "DummyDS: node 'id' and 'cluster' are required. Set it either in 'node' config or " "via --service-node and --service-cluster options."); } +class DeltaDummyConfigProviderManager; + +class DeltaDummyConfigSubscription : public DeltaConfigSubscriptionInstance, + Envoy::Config::SubscriptionCallbacks { +public: + using ProtoMap = std::map; + + DeltaDummyConfigSubscription(const uint64_t manager_identifier, + Server::Configuration::FactoryContext& factory_context, + DeltaDummyConfigProviderManager& config_provider_manager); + + // Envoy::Config::ConfigSubscriptionCommonBase + void start() override {} + + // Envoy::Config::SubscriptionCallbacks + void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, + const std::string& version_info) override; + void onConfigUpdate(const Protobuf::RepeatedPtrField&, + const Protobuf::RepeatedPtrField&, const std::string&) override { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; + } + void onConfigUpdateFailed(const EnvoyException*) override { + ConfigSubscriptionCommonBase::onConfigUpdateFailed(); + } + std::string resourceName(const ProtobufWkt::Any&) override { + return "test.common.config.DummyConfig"; + } + + const ProtoMap& protoMap() const { return proto_map_; } + +private: + ProtoMap proto_map_; +}; +using DeltaDummyConfigSubscriptionSharedPtr = std::shared_ptr; + +class ThreadLocalDummyConfig : public ThreadLocal::ThreadLocalObject, + public Envoy::Config::ConfigProvider::Config { +public: + void addProto(const test::common::config::DummyConfig& config_proto) { + protos_.push_back(config_proto); + } + + uint32_t numProtos() const { return protos_.size(); } + +private: + std::vector protos_; +}; + +class DeltaDummyDynamicConfigProvider : public Envoy::Config::DeltaMutableConfigProviderBase { +public: + DeltaDummyDynamicConfigProvider(DeltaDummyConfigSubscriptionSharedPtr&& subscription, + Server::Configuration::FactoryContext& factory_context, + std::shared_ptr dummy_config) + : DeltaMutableConfigProviderBase(std::move(subscription), factory_context, + ConfigProvider::ApiType::Delta), + subscription_(static_cast( + MutableConfigProviderCommonBase::subscription_.get())) { + initialize([&dummy_config](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { + return (dummy_config != nullptr) ? dummy_config : std::make_shared(); + }); + } + + DeltaDummyConfigSubscription& subscription() { return *subscription_; } + + // Envoy::Config::ConfigProvider + ConfigProtoVector getConfigProtos() const override { + ConfigProtoVector proto_vector; + for (const auto& value_type : subscription_->protoMap()) { + proto_vector.push_back(&value_type.second); + } + return proto_vector; + } + std::string getConfigVersion() const override { + return (subscription_->configInfo().has_value()) + ? subscription_->configInfo().value().last_config_version_ + : ""; + } + ConfigConstSharedPtr getConfig() const override { + return std::dynamic_pointer_cast(tls_->get()); + } + + // Envoy::Config::DeltaMutableConfigProviderBase + ConfigSharedPtr getConfig() override { + return std::dynamic_pointer_cast(tls_->get()); + } + + std::shared_ptr getThreadLocalDummyConfig() { + return std::dynamic_pointer_cast(tls_->get()); + } + +private: + DeltaDummyConfigSubscription* subscription_; +}; + +class DeltaDummyConfigProviderManager : public ConfigProviderManagerImplBase { +public: + DeltaDummyConfigProviderManager(Server::Admin& admin) + : ConfigProviderManagerImplBase(admin, "dummy") {} + + // Envoy::Config::ConfigProviderManagerImplBase + ProtobufTypes::MessagePtr dumpConfigs() const override { + auto config_dump = std::make_unique(); + for (const auto& element : configSubscriptions()) { + auto subscription = element.second.lock(); + ASSERT(subscription); + + if (subscription->configInfo()) { + auto* dynamic_config = config_dump->mutable_dynamic_dummy_configs()->Add(); + dynamic_config->set_version_info(subscription->configInfo().value().last_config_version_); + const auto* typed_subscription = + static_cast(subscription.get()); + const DeltaDummyConfigSubscription::ProtoMap& proto_map = typed_subscription->protoMap(); + for (const auto& value_type : proto_map) { + dynamic_config->mutable_dummy_configs()->Add()->MergeFrom(value_type.second); + } + TimestampUtil::systemClockToTimestamp(subscription->lastUpdated(), + *dynamic_config->mutable_last_updated()); + } + } + + return config_dump; + } + + // Envoy::Config::ConfigProviderManager + ConfigProviderPtr + createXdsConfigProvider(const Protobuf::Message& config_source_proto, + Server::Configuration::FactoryContext& factory_context, + const std::string&, + const Envoy::Config::ConfigProviderManager::OptionalArg&) override { + DeltaDummyConfigSubscriptionSharedPtr subscription = + getSubscription( + config_source_proto, factory_context.initManager(), + [&factory_context](const uint64_t manager_identifier, + ConfigProviderManagerImplBase& config_provider_manager) + -> ConfigSubscriptionCommonBaseSharedPtr { + return std::make_shared( + manager_identifier, factory_context, + static_cast(config_provider_manager)); + }); + + auto* existing_provider = static_cast( + subscription->getAnyBoundMutableConfigProvider()); + return std::make_unique( + std::move(subscription), factory_context, + (existing_provider != nullptr) ? existing_provider->getThreadLocalDummyConfig() : nullptr); + } +}; + +DeltaDummyConfigSubscription::DeltaDummyConfigSubscription( + const uint64_t manager_identifier, Server::Configuration::FactoryContext& factory_context, + DeltaDummyConfigProviderManager& config_provider_manager) + : DeltaConfigSubscriptionInstance( + "Dummy", manager_identifier, config_provider_manager, factory_context.timeSource(), + factory_context.timeSource().systemTime(), factory_context.localInfo()) {} + +void DeltaDummyConfigSubscription::onConfigUpdate( + const Protobuf::RepeatedPtrField& resources, + const std::string& version_info) { + if (resources.empty()) { + return; + } + + // For simplicity, there is no logic here to track updates and/or removals to the existing config + // proto set (i.e., this is append only). Real xDS APIs will need to track additions, updates and + // removals to the config set and apply the diffs to the underlying config implementations. + for (const auto& resource_any : resources) { + auto dummy_config = MessageUtil::anyConvert(resource_any); + proto_map_[version_info] = dummy_config; + // Propagate the new config proto to all worker threads. + applyDeltaConfigUpdate([&dummy_config](const ConfigSharedPtr& config) { + auto* thread_local_dummy_config = static_cast(config.get()); + // Per above, append only for now. + thread_local_dummy_config->addProto(dummy_config); + }); + } + + ConfigSubscriptionCommonBase::onConfigUpdate(); + setLastConfigInfo(absl::optional({absl::nullopt, version_info})); +} + +class DeltaConfigProviderImplTest : public testing::Test { +public: + DeltaConfigProviderImplTest() { + EXPECT_CALL(factory_context_.admin_.config_tracker_, add_("dummy", _)); + provider_manager_ = std::make_unique(factory_context_.admin_); + } + + Event::SimulatedTimeSystem& timeSystem() { return time_system_; } + +protected: + Event::SimulatedTimeSystem time_system_; + NiceMock factory_context_; + std::unique_ptr provider_manager_; +}; + +// Validate that delta config subscriptions are shared across delta dynamic config providers and +// that the underlying Config implementation can be shared as well. +TEST_F(DeltaConfigProviderImplTest, MultipleDeltaSubscriptions) { + envoy::api::v2::core::ApiConfigSource config_source_proto; + config_source_proto.set_api_type(envoy::api::v2::core::ApiConfigSource::GRPC); + ConfigProviderPtr provider1 = provider_manager_->createXdsConfigProvider( + config_source_proto, factory_context_, "dummy_prefix", + ConfigProviderManager::NullOptionalArg()); + + // No config protos have been received via the subscription yet. + EXPECT_FALSE(provider1->configProtoInfoVector().has_value()); + + Protobuf::RepeatedPtrField untyped_dummy_configs; + untyped_dummy_configs.Add()->PackFrom(parseDummyConfigFromYaml("a: a dummy config")); + untyped_dummy_configs.Add()->PackFrom(parseDummyConfigFromYaml("a: another dummy config")); + + DeltaDummyConfigSubscription& subscription = + dynamic_cast(*provider1).subscription(); + subscription.onConfigUpdate(untyped_dummy_configs, "1"); + + ConfigProviderPtr provider2 = provider_manager_->createXdsConfigProvider( + config_source_proto, factory_context_, "dummy_prefix", + ConfigProviderManager::NullOptionalArg()); + + // Providers, config implementations (i.e., the ThreadLocalDummyConfig) and config protos are + // expected to be shared for a given subscription. + EXPECT_EQ(&dynamic_cast(*provider1).subscription(), + &dynamic_cast(*provider2).subscription()); + ASSERT_TRUE(provider2->configProtoInfoVector().has_value()); + EXPECT_EQ( + provider1->configProtoInfoVector().value().config_protos_, + provider2->configProtoInfoVector().value().config_protos_); + EXPECT_EQ(provider1->config().get(), + provider2->config().get()); + // Validate that the config protos are propagated to the thread local config implementation. + EXPECT_EQ(provider1->config()->numProtos(), 2); + + // Issue a second config update to validate that having multiple providers bound to the + // subscription causes a single update to the underlying shared config implementation. + subscription.onConfigUpdate(untyped_dummy_configs, "2"); + // NOTE: the config implementation is append only and _does not_ track updates/removals to the + // config proto set, so the expectation is to double the size of the set. + EXPECT_EQ(provider1->config()->numProtos(), 4); + EXPECT_EQ(provider1->configProtoInfoVector().value().version_, + "2"); +} + +// Tests a config update failure. +TEST_F(DeltaConfigProviderImplTest, DeltaSubscriptionFailure) { + envoy::api::v2::core::ApiConfigSource config_source_proto; + config_source_proto.set_api_type(envoy::api::v2::core::ApiConfigSource::GRPC); + ConfigProviderPtr provider = provider_manager_->createXdsConfigProvider( + config_source_proto, factory_context_, "dummy_prefix", + ConfigProviderManager::NullOptionalArg()); + DeltaDummyConfigSubscription& subscription = + dynamic_cast(*provider).subscription(); + const auto time = std::chrono::milliseconds(1234567891234); + timeSystem().setSystemTime(time); + const EnvoyException ex(fmt::format("config failure")); + // Verify the failure updates the lastUpdated() timestamp. + subscription.onConfigUpdateFailed(&ex); + EXPECT_EQ(std::chrono::time_point_cast(provider->lastUpdated()) + .time_since_epoch(), + time); +} + } // namespace } // namespace Config } // namespace Envoy diff --git a/test/common/config/dummy_config.proto b/test/common/config/dummy_config.proto index fcb2749e4f036..ae32e1477e04c 100644 --- a/test/common/config/dummy_config.proto +++ b/test/common/config/dummy_config.proto @@ -25,3 +25,13 @@ message DummyConfigsDump { repeated StaticConfigs static_dummy_configs = 1; repeated DynamicConfigs dynamic_dummy_configs = 2; } + +message DeltaDummyConfigsDump { + message DynamicConfigs { + string version_info = 1; + repeated DummyConfig dummy_configs = 2; + google.protobuf.Timestamp last_updated = 3; + } + + repeated DynamicConfigs dynamic_dummy_configs = 2; +} diff --git a/test/mocks/config/BUILD b/test/mocks/config/BUILD index 4ffbef91dbbfb..287ec6ebf11c2 100644 --- a/test/mocks/config/BUILD +++ b/test/mocks/config/BUILD @@ -16,6 +16,7 @@ envoy_cc_mock( "//include/envoy/config:grpc_mux_interface", "//include/envoy/config:subscription_interface", "//include/envoy/config:xds_grpc_context_interface", + "//source/common/config:config_provider_lib", "//source/common/config:resources_lib", "//source/common/protobuf:utility_lib", "@envoy_api//envoy/api/v2:cds_cc", diff --git a/test/mocks/config/mocks.cc b/test/mocks/config/mocks.cc index 00d70fac699b2..b3e5c73d9caeb 100644 --- a/test/mocks/config/mocks.cc +++ b/test/mocks/config/mocks.cc @@ -45,5 +45,12 @@ MockGrpcMuxCallbacks::MockGrpcMuxCallbacks() { MockGrpcMuxCallbacks::~MockGrpcMuxCallbacks() {} +MockMutableConfigProviderBase::MockMutableConfigProviderBase( + std::shared_ptr&& subscription, + ConfigProvider::ConfigConstSharedPtr, Server::Configuration::FactoryContext& factory_context) + : MutableConfigProviderBase(std::move(subscription), factory_context, ApiType::Full) { + subscription_->bindConfigProvider(this); +} + } // namespace Config } // namespace Envoy diff --git a/test/mocks/config/mocks.h b/test/mocks/config/mocks.h index 7bdf32b42898e..5176655a65395 100644 --- a/test/mocks/config/mocks.h +++ b/test/mocks/config/mocks.h @@ -5,6 +5,7 @@ #include "envoy/config/subscription.h" #include "envoy/config/xds_grpc_context.h" +#include "common/config/config_provider_impl.h" #include "common/config/resources.h" #include "common/protobuf/utility.h" @@ -91,5 +92,19 @@ class MockGrpcStreamCallbacks : public GrpcStreamCallbacks&& subscription, + ConfigProvider::ConfigConstSharedPtr initial_config, + Server::Configuration::FactoryContext& factory_context); + + MOCK_CONST_METHOD0(getConfig, ConfigConstSharedPtr()); + MOCK_METHOD1(onConfigProtoUpdate, ConfigConstSharedPtr(const Protobuf::Message& config_proto)); + MOCK_METHOD1(initialize, void(const ConfigConstSharedPtr& initial_config)); + MOCK_METHOD1(onConfigUpdate, void(const ConfigConstSharedPtr& config)); + + ConfigSubscriptionCommonBase& subscription() { return *subscription_.get(); } +}; + } // namespace Config } // namespace Envoy diff --git a/test/test_common/utility.h b/test/test_common/utility.h index 822c159a870d6..34fdfdd639fb9 100644 --- a/test/test_common/utility.h +++ b/test/test_common/utility.h @@ -209,10 +209,17 @@ class TestUtility { * * @param lhs proto on LHS. * @param rhs proto on RHS. + * @param ignore_repeated_field_ordering if true, repeated field ordering will be ignored. * @return bool indicating whether the protos are equal. */ - static bool protoEqual(const Protobuf::Message& lhs, const Protobuf::Message& rhs) { - return Protobuf::util::MessageDifferencer::Equivalent(lhs, rhs); + static bool protoEqual(const Protobuf::Message& lhs, const Protobuf::Message& rhs, + bool ignore_repeated_field_ordering = false) { + Protobuf::util::MessageDifferencer differencer; + differencer.set_message_field_comparison(Protobuf::util::MessageDifferencer::EQUIVALENT); + if (ignore_repeated_field_ordering) { + differencer.set_repeated_field_comparison(Protobuf::util::MessageDifferencer::AS_SET); + } + return differencer.Compare(lhs, rhs); } /** @@ -250,7 +257,7 @@ class TestUtility { } for (int i = 0; i < lhs.size(); ++i) { - if (!TestUtility::protoEqual(lhs[i], rhs[i])) { + if (!TestUtility::protoEqual(lhs[i], rhs[i], /*ignore_repeated_field_ordering=*/false)) { return false; } } @@ -514,7 +521,22 @@ MATCHER_P(HeaderMapEqualIgnoreOrder, expected, "") { } MATCHER_P(ProtoEq, expected, "") { - const bool equal = TestUtility::protoEqual(arg, expected); + const bool equal = + TestUtility::protoEqual(arg, expected, /*ignore_repeated_field_ordering=*/false); + if (!equal) { + *result_listener << "\n" + << "==========================Expected proto:===========================\n" + << expected.DebugString() + << "------------------is not equal to actual proto:---------------------\n" + << arg.DebugString() + << "====================================================================\n"; + } + return equal; +} + +MATCHER_P(ProtoEqIgnoreRepeatedFieldOrdering, expected, "") { + const bool equal = + TestUtility::protoEqual(arg, expected, /*ignore_repeated_field_ordering=*/true); if (!equal) { *result_listener << "\n" << "==========================Expected proto:===========================\n" @@ -543,7 +565,7 @@ MATCHER_P(Percent, rhs, "") { envoy::type::FractionalPercent expected; expected.set_numerator(rhs); expected.set_denominator(envoy::type::FractionalPercent::HUNDRED); - return TestUtility::protoEqual(expected, arg); + return TestUtility::protoEqual(expected, arg, /*ignore_repeated_field_ordering=*/false); } } // namespace Envoy